From a441072ba89343c95655fec7e8a177d3521963e8 Mon Sep 17 00:00:00 2001 From: Jared Hancock Date: Wed, 5 Apr 2017 16:33:21 -0500 Subject: [PATCH] amendme: Add support for REMOTE_BACKUPS This allows the remote host to cooperate with the PUSH process and create backups on the remote host itself, for files which will be overwritten by the PUSH. This is similar to the concept introduced in the inline-pull code, and includes support for BACKUP_EXCLUDE and BACKUP settings on the remote host. --- bin/bitpocket | 152 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 110 insertions(+), 42 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index 8cec63b..88ef29b 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -20,6 +20,7 @@ SLOW_SYNC_FILE="$TMP_DIR/slow" RSYNC_RSH="ssh" BACKUPS=true BACKUP_EXCLUDE=() +REMOTE_BACKUPS=false # Default command-line options and such COMMANDS=() @@ -48,6 +49,7 @@ if [ -n "$REMOTE_HOST" ]; then else REMOTE_RUNNER="bash -c" REMOTE="$REMOTE_PATH" + REMOTE_BACKUPS=false fi REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp" @@ -97,6 +99,9 @@ BACKUPS=true # These files are *not* revisioned. These use bash wildcard patterns instead of # rsync filter patterns BACKUP_EXCLUDE=( '*.tmp' '*.temp' '~\$*' '*.TMP' ) +# Make revisions of files on the REMOTE_HOST in the push phase. NOTE: This +# requires `bitpocket` in the PATH on the remote host +REMOTE_BACKUPS=false ## SSH command with options for connecting to \$REMOTE # RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa" @@ -143,7 +148,6 @@ function pull() { echo "# >> Saving current state and backing up files (if needed)" cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after" - touch "$TMP_DIR/pull-delete" # Create a duplicate of STDOUT for logging of backed-up files, and use fd#4 # for logging of deleted files, which need to be sorted @@ -153,58 +157,73 @@ function pull() { # Determine what will be fetched from server and make backup # copies of any local files to be deleted or overwritten. # Order of includes/excludes/filters is EXTREMELY important - cat "$TMP_DIR/local-add-del" \ - | rsync --dry-run \ - -auzxi --delete --exclude "/$DOT_DIR" --exclude-from=- \ + rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \ + --exclude-from="$TMP_DIR/local-add-del" \ $RSYNC_OPTS $USER_RULES $REMOTE/ . \ - | while read line; do - filename=$(sed "s:^\S*\s*::" <<< "$line" | sed 's:\d96:\\\`:g') - if [[ "$line" =~ ^[ch\<\>][fd]|^\*deleting ]]; then - operation=${line%% *} - if should_backup "$filename"; then - [[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \ - || mkdir -p "$DOT_DIR"/backups/$TIMESTAMP - cp --parents -p --reflink=auto "$filename" \ - "$DOT_DIR"/backups/$TIMESTAMP || die "BACKUP" - echo " B $filename" >&3 - fi - # rsync does not support DELETE via --files-from, so delete inline here - # after the backup was made - if [[ "$operation" == "*deleting" ]]; then - # Only delete locally if deleted remotely - if grep -q -x "/$filename" "$TMP_DIR/remote-del"; then - echo " | *deleting $filename" >&3 - echo "/$filename" >&4 - [[ -d "$filename" ]] && rmdir "$filename" || rm "$filename" - continue - fi - elif [[ "$operation" =~ \+\+\+\+$ ]]; then - # Mark as added locally - echo "/$filename" >> "$TMP_DIR/tree-after" - fi - fi - # Drop trailing slash from folders so the contents are not recursively - # pulled - if [[ -d "$filename" && ${filename: -1} == '/' ]]; then - filename="${filename:0:-1}" - fi - # Sync the file. Use a NULL byte delimiter - printf '%s\0' "$filename" - done \ + | backup_inline \ | rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \ $REMOTE/ . \ | sed "s/^/ | /" || die "PULL" + [[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \ + && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" + # Close extra file descriptors exec 4>&- exec 3>&- +} - [[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \ - && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" +function backup_inline() { + # Read file names from STDIN, and, if they exist locally, create a backup + # of each of the files. If they are requested to be deleted (indicated by + # the tag at the line beginning), then they are deleted here. Write the + # filenames to STDOUT which need to be synchronized by `rsync` + while read line + do + filename=$(sed "s:^\S*\s*::" <<< "$line" | sed 's:\d96:\\\`:g') + if [[ "$line" =~ ^[ch\<\>][fd]|^\*deleting ]] + then + operation=${line%% *} + if should_backup "$filename" + then + [[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \ + || mkdir -p "$DOT_DIR"/backups/$TIMESTAMP + cp --parents -p --reflink=auto "$filename" \ + "$DOT_DIR"/backups/$TIMESTAMP || die "BACKUP" + echo " B $filename" >&3 + fi + # rsync does not support DELETE via --files-from, so delete inline here + # after the backup was made + if [[ "$operation" == "*deleting" ]] + then + # Only delete locally if deleted remotely + if grep -q -x "/$filename" "$TMP_DIR/remote-del" + then + # Delete here rather than syncing + echo " | *deleting $filename" >&3 + echo "/$filename" >&4 + [[ -d "$filename" ]] && rmdir "$filename" || rm "$filename" + continue + fi + elif [[ "$operation" =~ \+\+\+\+$ ]] + then + # Mark as added locally + echo "/$filename" >> "$TMP_DIR/tree-after" + fi + fi + # Drop trailing slash from folders so the contents are not recursively + # pulled + if [[ -d "$filename" && ${filename: -1} == '/' ]]; then + filename="${filename:0:-1}" + fi + # Sync the file. Use a NULL byte delimiter + printf '%s\0' "$filename" + done } function push() { # Actual push + # Send new and updated, remotely remove files deleted locally # Order of includes/excludes/filters is EXTREMELY important echo @@ -217,6 +236,52 @@ function push() { | sed "s/^/ | /" || die "PUSH" } +function remote_pull() { + # Similar to the pull(), except the rsync output it sent to an rsync + # session on the remote server. Bitpocket is used on the remote side + # so that backups can be captured before the files are overwritten + # by the push sync. + # + # Here, the files to sync are sent to STDIN, piped through + # backup_inline, and piped through an rsync process to fetch the files + + exec 3>&1 + exec 4>/dev/null + + # Send new and updated, remotely remove files deleted locally + # Order of includes/excludes/filters is EXTREMELY important + backup_inline \ + | rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \ + $REMOTE/ . \ + | sed "s/^/ | /" \ + || die "REMOTE-PULL" + + [[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \ + && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" + + # Close extra file descriptors + exec 4>&- + exec 3>&- +} + +function push_remote() { + # Works with remote_pull() to send the files which would be sent by the + # push() method, but allows for a backup on the remote side. + + echo + echo "# Pushing changes to server (with backups)" + + rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \ + --exclude-from="$TMP_DIR/remote-del" \ + $USER_RULES . $REMOTE/ \ + | $REMOTE_RUNNER " + cd \"$REMOTE_PATH\" + bitpocket remote-pull + " \ + | sed "s/^/ | /" \ + || die "REMOTE-PUSH" +} + function analyse { # Check what has changed touch "$STATE_DIR/tree-prev" @@ -284,7 +349,7 @@ function sync { fi pull - push + [[ $REMOTE_BACKUPS == true ]] && push_remote || push if [[ ${OPTIONS[pretend]} == false ]]; then # Save after-sync state @@ -505,7 +570,7 @@ function parseargs() { COMMANDS+=($1) ARGS+=($2 "$3") shift; shift;; - sync|init|pack|cron|log|list|help) + sync|init|pack|cron|log|list|help|remote-pull) COMMANDS+=($1);; *) echo "!!! Invalid command: $1";; esac @@ -527,4 +592,7 @@ case ${COMMANDS[0]} in list) list;; help) usage;; sync) sync;; + # Cooperate with push_remote to fetch files to be pushed with support for + # local backups in the BACKUPS/ folder + remote-pull) remote_pull;; esac