Source

aws_zfs_tools / backup / zfs-send.sh

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
#! /bin/bash -x

# Chip Schweiss - chip.schweiss@wustl.edu
#
# Copyright (C) 2012  Chip Schweiss

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

cd $( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
. ../zfs-tools-init.sh


if [ "x$zfs_logfile" != "x" ]; then
    logfile="$zfs_logfile"
else
    logfile="$default_logfile"
fi

if [ "x$zfs_report" != "x" ]; then
    report_name="$zfs_report"
else
    report_name="$default_report_name"
fi


# Must support multiple options
#   Local source and destination
#   Remote destination
#     Use bbcp
#     Encrypt traffic
#     Compress traffic
#   Use mbuffer (local or remote)
#   Replication stream
#   Property reset
#   Flat file target
#   SHA256 sum

result=
verify='true'
remote_ssh=
target_fifo=
target_fifos=
source_fifo=
local_fifos=

die () {
    error "$1"
    if [ "$tmpdir" != "" ]; then
        rm -rf $tmpdir
    fi
    exit 1
}

# show function usage
show_usage() {
    echo
    echo "Usage: $0 -s {source_zfs_folder} -t {target_zfs_folder}"
    echo "  [-f {first_snap}]   First snapshot.  Defaults to 'origin'"
    echo "  [-l {last_snap}]    Last snapshot.  Defaults to latest snapshot."
    echo "  [-h host]           Send to a remote host.  Defaults to via SSH."
    echo "  [-m]                Use mbuffer."
    echo "  [-b n]              Use BBCP, n connections.  "
    echo "     [-e]             Encrypt traffic w/ openssl.  Only for BBCP."
    echo "  [-g n]              Compress with gzip level n."
    echo "  [-z n]              Compress with LZ4.  Specify 1 for standard LZ4.  Specify 4 - 9 for LZ4HC compression level."
    echo "  [-r]                Use a replication stream"
    echo "  [-p {prop_string} ] Reset properties on target"
    echo "  [-F]                Target is a flat file.  No zfs receive will be used."
    echo "  [-k {file} ]        Generate a md5 sum.  Store it in {file}."
    echo "  [-L]                Overide default log file location."
    echo "  [-R]                Overide default report name."
}

# Minimum number of arguments needed by this program
MIN_ARGS=4

if [ "$#" -lt "$MIN_ARGS" ]; then
    show_usage
    exit 1
fi

source_folder=
target_folder=
target_pool=
first_snap=
last_snap=
remote_host=
mbuffer_use='false'
bbcp_streams=0
bbcp_encrypt='false'
gzip=0
lz4=0
replicate='false'
target_prop=
flat_file='false'
gen_chksum=

while getopts s:t:f:l:h:mbeg:zrpFk:L:R: opt; do
    case $opt in
        s)  # Source ZFS folder
            source_folder="$OPTARG"
            debug "zfs_send: Source folder: $source_folder"
            ;;
        t)  # Target ZFS folder or flat file
            target_folder="$OPTARG"
            debug "zfs_send: Target folder: $target_folder"
            ;;
        f)  # First snapshot
            first_snap="$OPTARG"
            debug "zfs_send: first_snap:    $first_snap"
            ;;
        l)  # Last snapshot
            last_snap="$OPTARG"
            debug "zfs_send: last_snap:     $last_snap"
            ;;
        h)  # Remote host
            remote_host="$OPTARG"
            debug "zfs_send: Remote host:   $remote_host"
            ;;
        m)  # Use mbuffer
            mbuffer_use='true'
            debug "zfs_send: Using mbuffer"
            ;;
        b)  # Use BBCP
            bbcp_streams="$OPTARG"
            debug "zfs_send: Using BBCP, $bbcp_streams connections."
            ;;
        e)  # Encrypt BBCP traffic
            bbcp_encrypt='true'
            debug "zfs_send: Encrypting BBCP traffic"
            ;;
        g)  # Compress with gzip
            gzip="$OPTARG"
            debug "zfs_send: Gzip compression level $gzip"
            ;;
        z)  # Compress with LZ4
            lz4="$OPTARG"
            case $lz4 in
                1) 
                    debug "zfs_send: Using LZ4 standard" ;;
                [4-9])
                    debug "zfs_send: Using LZ4HC level $lz4" ;;
                *)
                    die "zfs_send: Invalid LZ4 specified" ;;
            esac
            ;;
        r)  # Use a replication stream
            replicate='true'
            debug "zfs_send: Using a replication stream."
            ;;
        p)  # Reset properties on target
            target_prop="-o $OPTARG"
            debug "zfs_send: resetting target properties to: $target_prop"
            ;;
        F)  # Target is a flat file
            flat_file='true'
            debug "zfs_send: Flat file target"
            ;;
        k)  # Generate a md5 checksum
            gen_chksum="$OPTARG"
            debug "zfs_send: Generate MD5 sum in file $gen_chksum"
            ;;
        L)  # Overide default log file
            log_file="$OPTARG"
            debug "zfs_send: Log file set to $log_file"
            ;;
        R)  # Overide default report name
            report_name="$OPTARG"
            debug "zfs_send: Report name set to $report_name"
            ;;   
        ?)  # Show program usage and exit
            show_usage
            exit 0
            ;;
        :)  # Mandatory arguments not specified
            die "zfs_send: Option -$OPTARG requires an argument."
            ;;
    esac
done

tmpdir=$TMP/zfs_send_$$
remote_tmp=/tmp/zfs_send_$$

mkdir $tmpdir

###
#
# Verify all input is valid
#
###

if [ "$source_folder" == "" ]; then
    die "zfs_send: no source folder specified"
fi

if [ "$target_folder" == "" ]; then
    die "zfs_send: not target specified"
fi


if [ "$first_snap" == "" ]; then
    first_snap='origin'
    debug "zfs_send: first_snap no specified, set to origin"
fi

if [ "$flat_file" == 'false' ]; then
    # Split into pool / folder
    target_pool=`echo $target_folder | awk -F "/" '{print $1}'`
fi

re='^[0-9]+$'
if ! [[ $gzip =~ $re ]] ; then
   die "zfs_send: -g expects a number between 0 and9"
fi

if ! [[ $lz4 =~ $re ]] ; then
    if [ [ $lz4 -gt 9 ] || [ $lz4 -lt 4 ] ]; then
        die "zfs_send: -z expects a number between 4 and 9"
    fi
fi

###
# Verify remote host
###


if [ "$remote_host" != "" ]; then
    $timeout 30s $ssh root@${remote_host} mkdir $remote_tmp
    result=$?
    if [ $result -ne 0 ]; then
        error "zfs_send: Cannot connect to remote host at root@${remote_host}"
        verify='fail'
    else
        debug "zfs_send: Remote host connection verified."
    fi
else
    if [ "$bbcp_streams" != "0" ]; then
        error "zfs_send: Cannot use bbcp for local jobs"
    fi
fi

###
# Verify source folder
###

zfs list $source_folder &> /dev/null
result=$?
if [ $result -ne 0 ]; then
    error "zfs_send: Source zfs folder $source_folder not found."
    verify='fail'
fi

###
# Verify target folder / file
##

if [ "$remote_host" == "" ]; then
    # Local test
    if [ "$flat_file" == 'false' ]; then
        if [ "$replicate" == 'false' ]; then
            zfs list $target_folder &> /dev/null
            result=$?
            if [ $result -ne 0 ]; then
                error "zfs_send: Replicate not specified however target folder $target_folder does not exist."
                verify='fail'
            fi
        else
            # Verify pool exists
            zfs list $target_pool &> /dev/null
            result=$?
            if [ $result -ne 0 ]; then
                error "zfs_send: Replicate specified however target pool $target_pool does not exist."
                verify='fail'
            fi
        fi
    else # Flat file
        touch $target_folder &> /dev/null
        result=$?
        if [ $result -ne 0 ]; then
            error "zfs_send: Cannot create flat file $target_folder"
            verify='fail'
        else
            rm $target_folder
        fi
    fi
else
    # Remote test
    remote_ssh="$ssh root@$remote_host"
    if [ "$flat_file" == 'false' ]; then
        if [ "$replicate" == 'false' ]; then
            $timeout 2m $remote_ssh zfs list $target_folder &> /dev/null
            result=$?
            if [ $result -ne 0 ]; then
                error "zfs_send: Replicate not specified however target folder $target_folder does not exist on host $remote_host"
                verify='fail'
            fi
        else
            # Verify pool exists
            $timeout 2m $ssh $remote_ssh zfs list $target_pool &> /dev/null
            result=$?
            if [ $result -ne 0 ]; then
                error "zfs_send: Replicate specified however target pool $target_pool does not exist on host $remote_host"
                verify='fail'
            fi
        fi
    else # Flat file
        $timeout 30s $remote_ssh touch $target_folder &> /dev/null
        result=$?
        if [ $result -ne 0 ]; then
            error "zfs_send: Cannot create flat file $target_folder on host $remote_host"
            verify='fail'
        else
            $remote_ssh rm $target_folder
        fi
    fi
fi

if [ "$verify" == 'fail' ]; then
    die "zfs_send: Input validation failed.  Aborting."
fi    

###
#
# Setup named pipes for each transport type
#
###

remote_fifo () {
    $timeout 1m $remote_ssh mkfifo ${remote_tmp}/${1} || \
        die "zfs_send: Could not setup remote fifo $1 on host $remote_host"
    target_fifos="${remote_tmp}/${1} $target_fifos"
    echo -n "${remote_tmp}/${1}"
}

local_fifo () {
    mkfifo $1 || \
        die "zfs_send: Could not setup fifo ${tmpdir}/${1}"
    local_fifos="${tmpdir}/${1} $local_fifos"
    echo -n "${tmpdir}/${1}"
}


#TODO: Build from target to source connecting fifos as we build


##
# zfs receive or flat file
##

if [ "$flat_file" == 'true' ]; then
    # To flat file
    if [ "$remote_host" == "" ]; then
        target_fifo=`local_fifo flat_file.in`
        ( cat $target_fifo 1> $flat_file 2> $tmpdir/flat_file.error ; echo $? > $tmpdir/flat_file.errorlevel ) &       
    else
        target_fifo=`remote_fifo flat_file.in`
        $remote_ssh "nohup cat $target_fifo 1> $flat_file 2> $remote_tmp/flat_file.error &"
    fi
else
    # To zfs receive
    if [ "$remote_host" == "" ]; then
        # Local
        target_fifo=`local_fifo zfs_receive.in`
        ( cat $target_fifo | zfs receive -F -vu ${target_prop} ${target_folder} \
            2> $tmpdir/zfs_receive.error ; echo $? > $tmpdir/zfs_receive.errorlevel ) &
    else
        # Remote
        target_fifo=`remote_fifo zfs_receive.in`
        $remote_ssh "( nohup cat $target_fifo | zfs receive -F -vu ${target_prop} ${target_folder} \
            2> $remote_tmp/zfs_receive.error ; echo $? > $remote_tmp/zfs_receive.errorlevel ) &"
    fi
fi

##
# to Glacier
##


##
# mbuffer - Receive end
##

if [ "$mbuffer_use" == 'true' ]; then
    # Target end
    if [ [ "$flat_file" == 'false' ] && [ "$remote_host" != "" ] ]; then
            target_mbuffer_fifo=`remote_fifo target.mbuffer.in`
            $remote_ssh "( nohup mbuffer -q -s 128k -m 128M --md5 -l $remote_tmp/target.mbuffer.log \
                -i $target_mbuffer_fifo -o $target_fifo 1>/dev/null \
                2> $remote_tmp/target.mbuffer.error \
                < /dev/null ; echo $? > $remote_tmp/target.mbuffer.errorlevel ) &"
            target_fifo="$target_mbuffer_fifo"
    fi
fi

##
# gzip - Decompress
##

if [ [ "$gzip" -ne 0 ] && [ "$flat_file" == 'false' ] && [ "$remote_host" != "" ] ]; then
    target_gzip_fifo=`remote_fifo target.gzip.in`
    $remote_ssh "( nohup gunzip < $target_gzip_fifo 1> $target_fifo 2> $remote_tmp/zfs_target.gzip.error ; \
        echo $? > $remote_tmp/zfs_target.gzip.errorlevel ) &"
    target_fifo="$target_gzip_fifo"
fi

##
# LZ4     
##

#TODO:  Need and LZ4 binary    


##
# OpenSSL Decrypt
##

if [ "$bbcp_encrypt" == 'true' ]; then
    bbcp_key="$tmpdir/bbcp.key"
    remote_bbcp_key="$remote_tmp/bbcp.key"
    # Generate ssl key 
    pwgen -s 63 1 > $bbcp_key
    # Push key to remote
    scp $bbcp_key root@${remote_host}:${remote_bbcp_key}

    # Open FIFO
    target_ssl_fifo=`remote_fifo target_ssl_in`

    # Start openssl
    $remote_ssh "( nohup openssl aes-256-cbc -d -pass file:$bbcp_key <$target_ssl_fifo 1> $target_fifo \
        2> $remote_tmp/target_ssl.error ; echo $? > $remote_tmp/target_ssl.errorlevel ) &"

    target_fifo="$target_ssl_fifo"
fi


##
# BBCP
##

if [ "$bbcp_streams" -ne 0 ]; then
    # Source FIFO
    target_bbcp_fifo=`local_fifo bbcp_fifo`
    ( $bbcp -s $bbcp_streams --progress 300 --pipe i $target_bbcp_fifo root@${remote_host}:${target_fifo} 1> $tmpdir/bbcp.log \
        2> $tmpdir/bbcp.error < /dev/null ; echo $? > $tmpdir/bbcp.errorlevel ) &
fi
    





##
# SSH
##

##
# OpenSSL Encrypt
##

##
# gpg Encrypt
##

##
# LZ4
##

##
# gzip
##

##
# mbuffer - Send end
##

##
# zfs send
##