Skip to content

Commit

Permalink
amendme: Add support for REMOTE_BACKUPS
Browse files Browse the repository at this point in the history 
This allows the remote host to cooperate with the PUSH process and create
backups on the remote host itself, for files which will be overwritten by the
PUSH.

This is similar to the concept introduced in the inline-pull code, and includes
support for BACKUP_EXCLUDE and BACKUP settings on the remote host.
  • Loading branch information
Jared Hancock committed Jul 11, 2017
1 parent f28a683 commit 859f76b
Showing 1 changed file with 106 additions and 39 deletions.
145 changes: 106 additions & 39 deletions bin/bitpocket
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ SLOW_SYNC_FILE="$TMP_DIR/slow"
RSYNC_RSH="ssh"
BACKUPS=true
BACKUP_EXCLUDE=()
REMOTE_BACKUPS=false

# Default command-line options and such
COMMANDS=()
Expand Down Expand Up @@ -48,6 +49,7 @@ if [ -n "$REMOTE_HOST" ]; then
else
REMOTE_RUNNER="bash -c"
REMOTE="$REMOTE_PATH"
REMOTE_BACKUPS=false
fi

REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"
Expand Down Expand Up @@ -97,6 +99,9 @@ BACKUPS=true
# These files are *not* revisioned. These use bash wildcard patterns instead of
# rsync filter patterns
BACKUP_EXCLUDE=( '*.tmp' '*.temp' '~\$*' '*.TMP' )
# Make revisions of files on the REMOTE_HOST in the push phase. NOTE: This
# requires `bitpocket` in the PATH on the remote host
REMOTE_BACKUPS=false
## SSH command with options for connecting to \$REMOTE
# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa"
Expand Down Expand Up @@ -143,7 +148,6 @@ function pull() {
echo "# >> Saving current state and backing up files (if needed)"

cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after"
touch "$TMP_DIR/pull-delete"

# Create a duplicate of STDOUT for logging of backed-up files, and use fd#4
# for logging of deleted files, which need to be sorted
Expand All @@ -153,55 +157,69 @@ function pull() {
# Determine what will be fetched from server and make backup
# copies of any local files to be deleted or overwritten.
# Order of includes/excludes/filters is EXTREMELY important
cat "$TMP_DIR/local-add-del" \
| rsync --dry-run \
-auzxi --delete --exclude "/$DOT_DIR" --exclude-from=- \
rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \
--exclude-from="$TMP_DIR/local-add-del" \
$RSYNC_OPTS $USER_RULES $REMOTE/ . \
| while read line; do
filename=$(sed "s:^\S*\s*::" <<< "$line" | sed 's:\d96:\\\`:g')
if [[ "$line" =~ ^[ch\<\>][fd]|^\*deleting ]]; then
operation=${line%% *}
if should_backup "$filename"; then
[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
|| mkdir -p "$DOT_DIR"/backups/$TIMESTAMP
cp --parents -p --reflink=auto "$filename" \
"$DOT_DIR"/backups/$TIMESTAMP || die "BACKUP"
echo " B $filename" >&3
fi
# rsync does not support DELETE via --files-from, so delete inline here
# after the backup was made
if [[ "$operation" == "*deleting" ]]; then
echo " | *deleting $filename" >&3
echo "/$filename" >&4
[[ -d "$filename" ]] && rmdir "$filename" || rm "$filename"
continue
elif [[ "$operation" =~ \+\+\+\+$ ]]; then
# Mark as added locally
echo "/$filename" >> "$TMP_DIR/tree-after"
fi
fi
# Drop trailing slash from folders so the contents are not recursively
# pulled
if [[ -d "$filename" && ${filename: -1} == '/' ]]; then
filename="${filename:0:-1}"
fi
# Sync the file. Use a NULL byte delimiter
printf '%s\0' "$filename"
done \
| backup_inline \
| rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \
$REMOTE/ . \
| sed "s/^/ | /" || die "PULL"

[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
&& echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"

# Close extra file descriptors
exec 4>&-
exec 3>&-
}

[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
&& echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"
function backup_inline() {
# Read file names from STDIN, and, if they exist locally, create a backup
# of each of the files. If they are requested to be deleted (indicated by
# the tag at the line beginning), then they are deleted here. Write the
# filenames to STDOUT which need to be synchronized by `rsync`
while read line
do
filename=$(sed "s:^\S*\s*::" <<< "$line" | sed 's:\d96:\\\`:g')
if [[ "$line" =~ ^[ch\<\>][fd]|^\*deleting ]]
then
operation=${line%% *}
if should_backup "$filename"
then
[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
|| mkdir -p "$DOT_DIR"/backups/$TIMESTAMP
cp --parents -p --reflink=auto "$filename" \
"$DOT_DIR"/backups/$TIMESTAMP || die "BACKUP"
echo " B $filename" >&3
fi
# rsync does not support DELETE via --files-from, so delete inline here
# after the backup was made
if [[ "$operation" == "*deleting" ]]
then
# Delete here rather than syncing
echo " | *deleting $filename" >&3
echo "/$filename" >&4
[[ -d "$filename" ]] && rmdir "$filename" || rm "$filename"
continue
elif [[ "$operation" =~ \+\+\+\+$ ]]
then
# Mark as added locally
echo "/$filename" >> "$TMP_DIR/tree-after"
fi
fi
# Drop trailing slash from folders so the contents are not recursively
# pulled
if [[ -d "$filename" && ${filename: -1} == '/' ]]; then
filename="${filename:0:-1}"
fi
# Sync the file. Use a NULL byte delimiter
printf '%s\0' "$filename"
done
}

function push() {
# Actual push

# Send new and updated, remotely remove files deleted locally
# Order of includes/excludes/filters is EXTREMELY important
echo
Expand All @@ -213,6 +231,52 @@ function push() {
| sed "s/^/ | /" || die "PUSH"
}

function remote_pull() {
# Similar to the pull(), except the rsync output it sent to an rsync
# session on the remote server. Bitpocket is used on the remote side
# so that backups can be captured before the files are overwritten
# by the push sync.
#
# Here, the files to sync are sent to STDIN, piped through
# backup_inline, and piped through an rsync process to fetch the files

exec 3>&1
exec 4>/dev/null

# Send new and updated, remotely remove files deleted locally
# Order of includes/excludes/filters is EXTREMELY important
backup_inline \
| rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \
$REMOTE/ . \
| sed "s/^/ | /" \
|| die "REMOTE-PULL"

[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
&& echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"

# Close extra file descriptors
exec 4>&-
exec 3>&-
}

function push_remote() {
# Works with remote_pull() to send the files which would be sent by the
# push() method, but allows for a backup on the remote side.

echo
echo "# Pushing changes to server (with backups)"

rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \
--exclude-from="$TMP_DIR/remote-del" \
$USER_RULES . $REMOTE/ \
| $REMOTE_RUNNER "
cd \"$REMOTE_PATH\"
bitpocket remote-pull
" \
| sed "s/^/ | /" \
|| die "REMOTE-PUSH"
}

function analyse {
# Check what has changed
touch "$STATE_DIR/tree-prev"
Expand Down Expand Up @@ -280,7 +344,7 @@ function sync {
fi

pull
push
[[ $REMOTE_BACKUPS == true ]] && push_remote || push

if [[ ${OPTIONS[pretend]} == false ]]; then
# Save after-sync state
Expand Down Expand Up @@ -501,7 +565,7 @@ function parseargs() {
COMMANDS+=($1)
ARGS+=($2 "$3")
shift; shift;;
sync|init|pack|cron|log|list|help)
sync|init|pack|cron|log|list|help|remote-pull)
COMMANDS+=($1);;
*) echo "!!! Invalid command: $1";;
esac
Expand All @@ -523,4 +587,7 @@ case ${COMMANDS[0]} in
list) list;;
help) usage;;
sync) sync;;
# Cooperate with push_remote to fetch files to be pushed with support for
# local backups in the BACKUPS/ folder
remote-pull) remote_pull;;
esac

0 comments on commit 859f76b

Please sign in to comment.