#!/bin/bash
# usage: kvm_move.sh VMID kxXX [forceSCSI] [toVMID]
# sshpass recommended (apt-get install sshpass)
# wget -qN 'https://bitbucket.org/!api/2.0/snippets/ruweb/ooyRX/master/files/kvm_move.sh' && chmod 700 kvm_move.sh

VMID=""
TOKX=""
RLIMIT=""
TOSCSI=""
TOVMID=""
DEBUG=""
ONLYDISK=""
PAUSE=""
EXIT=0
for i in "$@";
    do case $i in
        -I=*|--to-vmid=*) TOVMID="${i#*=}"
        ;;
        -I*) TOVMID="${i#-I}"
        ;;
        -L=*|--rate-limit=*) RLIMIT="${i#*=}"
        ;;
        -L*) RLIMIT="${i#-L}"
        ;;
        -S|--force-scsi) TOSCSI=YES
        ;;
        -D|--debug) DEBUG=YES
        ;;
        -O|--only-disk) ONLYDISK=YES
        ;;
        -P|--pause) PAUSE=YES
        ;;
        [0-9]*) [ -n "$VMID" ] && echo "Unknown option: $i" && EXIT=1 || VMID=$i
        ;;
        -*) echo "Unknown option: $i" && EXIT=1
        ;;
        *) [ -n "$TOKX" ] && echo "Unknown option: $i" && EXIT=1 || TOKX=$i
        ;;
    esac
done

([ 0$EXIT -gt 0 ] || [ -z "$VMID" ] || [ -z "$TOKX" ]) && cat <<EOF && exit $EXIT
Usage: $0 VMID kxXX [OPTION]...
Options: 
    -I, --to-vmid=NUM        use specific NUM as VMID on target node even if specified VMID exist
    -S, --force-scsi         switch virtio devices to scsi
    -L, --rate-limit=RATE    limit transfer to RATE bytes per second (option for pv)
    -O, --only-disk          only copy disk (no changes to configs will be made)
    -P, --pause              pause and ask for confirmation before VM shutdown for final snapshot
    -D, --debug              print each command to terminal (set -x)
EOF

curl 'https://bitbucket.org/!api/2.0/snippets/ruweb/ooyRX/master/files/kvm_move.sh' > "$0.tmp"
if ! diff -q "$0.tmp" "$0"; then
    [ -s "$0.tmp" ] && cat "$0.tmp" > "$0" && rm "$0.tmp" && "$0" "$@"
    exit
else
    rm "$0.tmp"
fi

TOZFS=""
FROMZFS=""
ZFSARG="-RLec"

FOUND=$(qm list | egrep -w "^[[:space:]]+$VMID")
[ -z "$FOUND" ] && echo "VMID '$VMID' not found" && exit 1
VMNAME=$(echo "$FOUND" | awk '{print $2}')
DISK=$(egrep "^(virtio|scsi|ide)0:.*vm-$VMID-disk-[0123].*G" "/etc/pve/qemu-server/$VMID.conf" | grep -o "disk-[0123]")
[ -z "$DISK" ] && echo "ERROR: Can't determine main disk file" && exit 1

if [ "$TOKX" != "${TOKX#kx}" ] && [ "$TOKX" = "${TOKX%.*}" ]; then
    KXADDR=${TOKX#kx}
    KXADDR=${KXADDR#0}
    KXADDR="192.168.254.$KXADDR"
    [ "$TOKX" = "${TOKX%.*}" ] && TOKX="$TOKX.deserv.net"
elif [ "$TOKX" != "${TOKX#ky}" ] && [ "$TOKX" = "${TOKX%.*}" ]; then
    KXADDR=${TOKX#ky}
    KXADDR="10.10.11.1$KXADDR"
    [ "$TOKX" = "${TOKX%.*}" ] && TOKX="$TOKX.deserv.net"
else
    [ "$TOKX" = "${TOKX%.*}" ] && TOKX="$TOKX.deserv.net"
    KXADDR=$(host "$TOKX" | head -n1 | awk '/has address/ { print $4 }')
    [ -z "$KXADDR" ] && echo "$TOKX address not found" && exit 1
fi

[ -n "$(which sshpass)" ] || apt-get install sshpass
[ -z "$(which pv)" ] && apt-get install pv

grep -q ": zfs:" "/etc/pve/qemu-server/$VMID.conf" && FROMZFS="1$(egrep "^[[:space:]]+pool[[:space:]]" /etc/pve/storage.cfg)" && FROMZFS=${FROMZFS##*pool }

echo "VMID $VMID ($VMNAME) will be moved to $TOKX ($KXADDR)"
if [ -n "$FROMZFS" ]; then
    zfs list -t all -ro name,volsize,used,logicalused | grep -Ew "$FROMZFS/vm-$VMID-disk-[0-9]|VOLSIZE"
fi
echo "/var/lib/vz/images/$VMID: $(ls -shl "/var/lib/vz/images/$VMID/")"

SSHCMD="ssh -o StrictHostKeyChecking=no -p 2022 $KXADDR"
if [ -n "$(which sshpass)" ]; then
    read -srp "Enter root password for $TOKX: " SSHPASS
    echo
    export SSHPASS
    SSHCMD="sshpass -e $SSHCMD"
else
    echo "WARNING: Consider installing sshpass (apt-get install sshpass) to skip additional password prompts!"
fi

RES=$($SSHCMD "tar czPf backup$(date +%y%m%d%H%M%S).tgz /etc/pve /var/www/httpae/core && printf 'OK' \
    && grep -q '^zfspool: ' /etc/pve/storage.cfg && printf 'z'")
[ "${RES%z}" != "OK" ] && echo "Remote SSH test error. Aborting." && exit 1
[ "${RES#OK}" = "z" ] && TOZFS="1"

[ -n "$DEBUG" ] && set -x

if [ -z "$ONLYDISK" ]; then
    echo "Backing up config data..."
    tar czPf "backup$(date +%y%m%d%H%M%S).tgz" /etc/pve /var/www/httpae/core
fi

echo "Checking target node..."
if [ -z "$TOVMID" ]; then
    TOVMID=$($SSHCMD "NEWID='$VMID'; while [ -e \"/etc/pve/qemu-server/\$NEWID.conf\" ]; do NEWID=\$(expr \$NEWID + 1); done; echo \$NEWID")
    [ "$TOVMID" != "$VMID" ] && echo "New VMID will be used: $TOVMID"
fi
FOUND=$($SSHCMD "touch '/etc/pve/qemu-server/$TOVMID.conf'; grep -H '^$VMNAME:' /etc/pve/priv/shadow.cfg; grep -H ':$VMNAME@pve:' /etc/pve/user.cfg; \
    ls -d '/var/lib/vz/images/$TOVMID/'* 2>/dev/null; [ -s '/etc/pve/qemu-server/$TOVMID.conf' ] && ls -d '/etc/pve/qemu-server/$TOVMID.conf' \
    && qm list | egrep -w '^[[:space:]]+$TOVMID'; \
    [ -e /dev/zvol ] && zfs list -t all -o name,volsize,used,logicalused,creation | grep '/vm-$TOVMID-'")
if [ -n "$FOUND" ]; then
    echo "$FOUND" | grep -w running && echo "VMID $TOVMID already exists and running on target node. Aborting." && exit 1
    printf "WARNING\n%s\nalready exists on $TOKX\n" "$FOUND"
    read -rp "ARE YOU SURE YOU WANT TO CONTINUE? [yN]: " YN
    [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
fi

TAREXCLUDE=""
$SSHCMD "dd if=/dev/null of=/dev/null count=1 conv=sparse 2>/dev/null" && DDSPARSE="conv=sparse" || DDSPARSE=""
if [ -z "$FROMZFS" ]; then
    RATIO=$(ls -ls "/var/lib/vz/images/$VMID/vm-$VMID-$DISK.qcow2" | awk '{ratio=$1/$6*1024*100; printf "%.0f\n",ratio}')
    if [ -n "$DDSPARSE" ]; then
        [ "0$RATIO" -ge 75 ] && TAREXCLUDE="vm-$VMID-$DISK.qcow2"
#    else
#        [ "0$RATIO" -ge 98 ] && TAREXCLUDE="vm-$VMID-$DISK.qcow2"
    fi
fi

MYPID=$$
if [ -z "$(which pv)" ]; then
    BUFCMD="dd" && BUFARG1="bs=1M" && BUFARG2=$BUFARG1
    ([ -n "$FROMZFS" ] || [ -n "$TAREXCLUDE" ]) && (while kill -0 "$MYPID" 2>/dev/null; do sleep 20; killall -q -USR1 dd; done) &
else
    BUFCMD="pv --buffer-size=1m --interval=2 --timer --rate --average-rate --bytes --progress --eta" && BUFARG1="--eta" && BUFARG2=$BUFARG1
    [ -n "$RLIMIT" ] && BUFARG1="--rate-limit=$RLIMIT"
fi

if [ -n "$FROMZFS" ] && [ -n "$TOZFS" ]; then
    for disk in $(grep -o "^[a-z]*[0-9]: zfs:vm-$VMID-disk-[^,-]*" "/etc/pve/qemu-server/$VMID.conf" | awk -F- '{print $4}'); do
        echo "Sending ZFS snapshot to ${TOKX%%.*}/vm-$TOVMID-disk-$disk..." && zfs snapshot "$FROMZFS/vm-$VMID-disk-$disk@prepare"
        [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
        $SSHCMD "zfs send '$ZFSARG' -n '${TOKX%%.*}'" 2>/dev/null
        [ $? -ne 0 ] && BUFARG2="" || BUFARG2=$(zfs send "$ZFSARG" -Pn "$FROMZFS/vm-$VMID-disk-$disk@prepare" 2>/dev/null |grep ^size |cut -f2)
        [ -z "$BUFARG2" ] && ZFSARG="-RLe" && BUFARG2=$(zfs send "$ZFSARG" -Pn "$FROMZFS/vm-$VMID-disk-$disk@prepare" |grep ^size |cut -f2)
        if [ "$BUFCMD" != "dd" ]; then
            [ -n "$BUFARG2" ] && BUFARG2="--size=$BUFARG2" || BUFARG2=$BUFARG1
        else
            BUFARG2=$BUFARG1
        fi
        zfs send "$ZFSARG" "$FROMZFS/vm-$VMID-disk-$disk@prepare" | $BUFCMD "$BUFARG1" "$BUFARG2" | $SSHCMD "zfs recv -Fu '${TOKX%%.*}/vm-$TOVMID-disk-$disk'"
        [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
    done
fi

[ -n "$PAUSE" ] && read -p "Press enter to shutdown VM and proceed to final copy phase "

if echo "$FOUND" | grep -qw "stopped"; then
    WASRUNNING=""
else
    WASRUNNING=YES
    echo "Shutting down VMID \"$VMID\"..."
    qm shutdown "$VMID"
fi
until qm status "$VMID" | grep -qw "stopped"; do
    echo "VMID \"$VMID\" is still running. Please shut it down..."
    sleep 5
done
date

cp "/etc/pve/qemu-server/$VMID.conf" "/etc/pve/qemu-server/$VMID.conf.new"
if [ -z "$ONLYDISK" ]; then
    mv -v "/etc/pve/qemu-server/$VMID.conf" "/etc/pve/qemu-server/$VMID.conf~"

    echo "Moving PVE user..."
    STR=$(grep "^$VMNAME:" /etc/pve/priv/shadow.cfg)
    if [ -n "$STR" ]; then
        [ "$($SSHCMD "echo '$STR' >> /etc/pve/priv/shadow.cfg && echo OK")" != "OK" ] && echo "Error adding shadow.cfg string. Aborting." && exit 1
        [ "$($SSHCMD "mv /etc/pve/user.cfg /etc/pve/user.cfg~ && echo 'user:$VMNAME@pve:1:0::::::' >> /etc/pve/user.cfg && cat /etc/pve/user.cfg~ >> /etc/pve/user.cfg && echo 'acl:1:/vms/$TOVMID:$VMNAME@pve:PVEVMUserRUWEB:' >> /etc/pve/user.cfg && perl -pi -e 's/^group:iso:/group:iso:$VMNAME\@pve,/' /etc/pve/user.cfg && echo OK")" != "OK" ] && echo "Error adding user.cfg strings. Aborting." && exit 1
        mv /etc/pve/priv/shadow.cfg /etc/pve/priv/shadow.cfg~ && grep -v "^$VMNAME:" /etc/pve/priv/shadow.cfg~ > /etc/pve/priv/shadow.cfg
        mv /etc/pve/user.cfg /etc/pve/user.cfg~ && grep -v ":$VMNAME@pve:" /etc/pve/user.cfg~ > /etc/pve/user.cfg && perl -pi -e "s/(,|:)$VMNAME\@pve,?/\$1/" /etc/pve/user.cfg
    elif [ -z "$FROMZFS" ] || [ -z "$TOZFS" ]; then
        read -rp "Error: shadow.cfg string not found. Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
    else
        echo "Error: shadow.cfg string not found!"
    fi
fi

echo "Copying disk images..."
SIZE=$(egrep "(virtio|scsi|ide)0.*G" "/etc/pve/qemu-server/$VMID.conf.new")
SIZE=${SIZE##*size=}
SIZE=${SIZE%%G*}

(while kill -0 "$MYPID" 2>/dev/null; do sleep 20; killall -q -USR1 tar || break; done) &
(cd "/var/lib/vz/images/$VMID" && tar -cSvf- --totals=SIGUSR1 --exclude="$TAREXCLUDE" -- * | $SSHCMD "mkdir -p '/var/lib/vz/images/$TOVMID' \
    && cd '/var/lib/vz/images/$TOVMID' && tar -xpf-")
[ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
[ "$TOVMID" != "$VMID" ] && $SSHCMD "cd '/var/lib/vz/images/$TOVMID' \
    && for i in 0 1 2 3 4 5 6 7 8 9; do [ -e \"vm-$VMID-disk-\$i.qcow2\" ] && mv -v \"vm-$VMID-disk-\$i.qcow2\" \"vm-$TOVMID-disk-\$i.qcow2\"; done"

if [ -n "$TAREXCLUDE" ] && [ -z "$FROMZFS" ]; then
    echo "vm-$VMID-$DISK.qcow2 (dd $DDSPARSE)"
    if [ "$BUFCMD" != "dd" ]; then
        BUFARG2=$(ls -l /var/lib/vz/images/$VMID/vm-$VMID-$DISK.qcow2 | awk '{print $5}')
        [ -n "$BUFARG2" ] && BUFARG2="--size=$BUFARG2" || BUFARG2=$BUFARG1
    fi
    dd if="/var/lib/vz/images/$VMID/vm-$VMID-$DISK.qcow2" bs=1M status=noxfer | $BUFCMD "$BUFARG1" "$BUFARG2" | $SSHCMD "dd of='/var/lib/vz/images/$TOVMID/vm-$TOVMID-$DISK.qcow2' bs=1M $DDSPARSE"
    [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
fi

if [ -n "$FROMZFS" ]; then
    if [ -z "$TOZFS" ]; then
        echo "Copying ZFS volume to RAW..."
        dd if="/dev/zvol/$FROMZFS/vm-$VMID-$DISK" bs=1M status=noxfer | $BUFCMD "$BUFARG1" "$BUFARG2" | $SSHCMD "dd of='/var/lib/vz/images/$TOVMID/vm-$VMID-$DISK.raw' bs=1M $DDSPARSE"
        [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
        echo "Converting RAW to QCOW2..."
        $SSHCMD "qemu-img convert -t writeback -p -o preallocation=metadata -f raw -O qcow2 '/var/lib/vz/images/$TOVMID/vm-$VMID-$DISK.raw' \
            '/var/lib/vz/images/$TOVMID/vm-$TOVMID-$DISK.qcow2' && rm '/var/lib/vz/images/$TOVMID/vm-$VMID-$DISK.raw'"
        [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
        perl -pi -e 's#^(virtio|scsi)0: zfs:vm-(\d+)-disk-(\d)#${1}0: local:$2/vm-$2-disk-$3.qcow2#' "/etc/pve/qemu-server/$VMID.conf.new"
        perl -pi -e 's/^(virtio|scsi)(0: local.*),cache=[a-z]+(.*)/$1$2,cache=writethrough$3/' "/etc/pve/qemu-server/$VMID.conf.new"
    else
#        echo "Copying ZFS volume..."
#        zfs send "$FROMZFS/vm-$VMID-$DISK" | dd bs=1M | $SSHCMD "zfs recv '${TOKX%%.*}/vm-$TOVMID-$DISK' && zfs destroy '${TOKX%%.*}/vm-$TOVMID-$DISK@--head--'"
        for disk in $(grep -o "^[a-z]*[0-9]: zfs:vm-$VMID-disk-[^,]*" "/etc/pve/qemu-server/$VMID.conf.new" | awk -F- '{print $4}'); do
            zfs snapshot "$FROMZFS/vm-$VMID-disk-$disk@final"
            [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
            [ -n "$ONLYDISK" ] && [ -n "$WASRUNNING" ] && echo "Starting VM back..." && qm start "$VMID"
            echo "Sending incremental ZFS update..." \
                && zfs send -vRi "$FROMZFS/vm-$VMID-disk-$disk@prepare" "$FROMZFS/vm-$VMID-disk-$disk@final" | $SSHCMD "zfs recv -vu '${TOKX%%.*}/vm-$TOVMID-disk-$disk' \
                && zfs destroy '${TOKX%%.*}/vm-$TOVMID-disk-$disk@prepare' && zfs destroy '${TOKX%%.*}/vm-$TOVMID-disk-$disk@final'"
            [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
        done
    fi
elif [ -n "$TOZFS" ]; then
        echo "Converting QCOW2 to ZFS volume..."
        $SSHCMD "zfs create -s -V ${SIZE}G '${TOKX%%.*}/vm-$TOVMID-$DISK' && sleep 5 \
            && qemu-img convert -t writeback -p -n -f qcow2 -O raw '/var/lib/vz/images/$TOVMID/vm-$TOVMID-$DISK.qcow2' \
            'zeroinit:/dev/zvol/${TOKX%%.*}/vm-$TOVMID-$DISK' && rm '/var/lib/vz/images/$TOVMID/vm-$TOVMID-$DISK.qcow2'"
        [ $? -ne 0 ] && read -rp "Something wrong? Continue? [yN]: " YN && [ "$YN" != "y" ] && [ "$YN" != "Y" ] && echo "Aborted." && exit 1
        perl -pi -e 's#^(virtio|scsi)0: local:(\d++)/vm-\d++-disk-(\d).qcow2#${1}0: zfs:vm-$2-disk-$3#' "/etc/pve/qemu-server/$VMID.conf.new"
        perl -pi -e 's/^(virtio|scsi)(0: zfs.*),cache=[a-z]+(.*)/$1$2,cache=none$3/' "/etc/pve/qemu-server/$VMID.conf.new"
        perl -pi -e 's/^(virtio|scsi)(0: zfs.*),format=qcow2(.*)/$1$2$3/' "/etc/pve/qemu-server/$VMID.conf.new"
        grep -q '^scsihw: ' "/etc/pve/qemu-server/$VMID.conf.new" || echo "scsihw: virtio-scsi-pci" >> "/etc/pve/qemu-server/$VMID.conf.new"
fi

if [ -n "$ONLYDISK" ]; then
    echo -e "\a"
    echo "Starting VM on $TOKX..."
    $SSHCMD "qm start '$TOVMID'"
    if [ -n "$FROMZFS" ]; then
        echo "Do not forget to destroy snapshots:"
        printf "# zfs destroy %s/vm-%s-%s@prepare; zfs destroy %s/vm-%s-%s@final\n" "$FROMZFS" "$VMID" "$DISK" "$FROMZFS" "$VMID" "$DISK"
    fi
    exit 0
fi

echo "Moving VM config..."
perl -ne "
        if ('$TOSCSI' ne '') {
            /^(bootdisk|ide\d|virtio\d|scsi\d)/ && !/media=cdrom/ && s/(virtio|ide)(\d)/scsi\$2/;
            if (/^scsi\d/) { s/,discard=[a-z]+//; s/(.*)/\$1,discard=on/; }
        }
        m/^(args|ide|virtio|scsi)/ && s#(:|-)$VMID(,|/|-)#\${1}$TOVMID\${2}#g;
        if (m/^(ide|scsi)1: local/) { s/,cache=[a-z]+//; s/(.*)/\${1},cache=writethrough/; }
        print;
    " "/etc/pve/qemu-server/$VMID.conf.new" \
    | $SSHCMD "cat - > /etc/pve/qemu-server/$TOVMID.conf"
rm "/etc/pve/qemu-server/$VMID.conf.new"
mv -v "/var/lib/vz/images/$VMID" "/var/lib/vz/images/$VMID~"
[ -n "$FROMZFS" ] && zfs rename "$FROMZFS/vm-$VMID-$DISK" "$FROMZFS/vm-$VMID-$DISK.moved"

echo "Moving IP and template data..."
if [ ! -e /var/www/httpae/core/httpae.conf ]; then
for FILE in /var/www/httpae/core/nameip /var/www/httpae/core/nametempl; do
    STR=$(grep "^$VMNAME " $FILE)
    if [ -z "$STR" ]; then
        echo "Error: $VMNAME not found in $FILE"
    else
        [ "$($SSHCMD "echo '$STR' >> '$FILE' && echo OK")" != "OK" ] && echo "Error adding string to $TOKX:$FILE"
        [ "$FILE" = "/var/www/httpae/core/nameip" ] && [ -s /var/www/httpae/core/iptables.rules ] && for IPP in $STR; do
            STR=$(grep " $IPP " /var/www/httpae/core/iptables.rules)
            if [ -n "$STR" ]; then
                [ "$($SSHCMD "echo '$STR' >> /var/www/httpae/core/iptables.rules && echo OK")" != "OK" ] \
                    && echo "Error adding string to $TOKX:/var/www/httpae/core/iptables.rules"
                mv /var/www/httpae/core/iptables.rules /var/www/httpae/core/iptables.rules~ \
                    && grep -v " $IPP " /var/www/httpae/core/iptables.rules~ > /var/www/httpae/core/iptables.rules
            fi
        done
    fi
    mv "$FILE" "$FILE~" && grep -v "^$VMNAME " "$FILE~" > "$FILE"
done
STR=$(grep "^$VMNAME " /var/www/httpae/core/nameip~ | awk '{print $1,$2}')
else
    STR=$(/var/www/httpae/core/confvm.pl list "$VMNAME")
    if [ -z "$STR" ]; then 
        echo "Error: confvm.pl list $VMNAME is empty!"
    else
        echo "$STR" |$SSHCMD "/var/www/httpae/core/confvm.pl add" || echo "Error: confvm.pl add returned non-zero!"
        /var/www/httpae/core/confvm.pl delete "$VMNAME" || echo "Error: confvm.pl delete returned non-zero!"
        STR=$(echo "$STR" |grep "^[[:space:]]*ip[[:space:]]")
        IP=$(echo "$STR" |awk '{print $2}')
        [ -s /var/www/httpae/core/iptables.rules ] && for IPP in $STR; do
            [ "$IPP" = "ip" ] && continue
            STR=$(grep " $IPP " /var/www/httpae/core/iptables.rules)
            if [ -n "$STR" ]; then
                [ "$($SSHCMD "echo '$STR' >> /var/www/httpae/core/iptables.rules && echo OK")" != "OK" ] \
                    && echo "Error adding string to $TOKX:/var/www/httpae/core/iptables.rules"
                mv /var/www/httpae/core/iptables.rules /var/www/httpae/core/iptables.rules~ \
                    && grep -v " $IPP " /var/www/httpae/core/iptables.rules~ > /var/www/httpae/core/iptables.rules
            fi
        done
        STR="$VMNAME $IP"
    fi
fi

echo "Starting VM on $TOKX..."
$SSHCMD "qm start '$TOVMID'"

echo "Restarting IPTABLES on $TOKX..."
$SSHCMD "/var/www/httpae/core/startiptables.pl"
date

echo "Restarting local IPTABLES..."
/var/www/httpae/core/startiptables.pl

printf "\nПеренос %s
Здравствуйте.
Ваш VDS %s был перенесен на другую ноду.
Новая ссылка для входа в Proxmox: https://%s:8006/\n" "$STR" "$STR" "$TOKX"

printf "\nDo not forget to update VM node number inside billing system and remove /var/lib/vz/images/%s~/" "$VMID"
[ -n "$FROMZFS" ] && printf " and %s/vm-%s-%s.moved volume" "$FROMZFS" "$VMID" "$DISK"
echo -e "\a"

$SSHCMD "/var/www/httpae/core/crqmlistfile.pl"
/var/www/httpae/core/crqmlistfile.pl