#!/bin/bash # borg-backup.sh # Script to run regularly to backup a Jean-Cloud machine # # This will create a separate borg repo for every item in the BORG_REPOS variable # And in each location specified in the BORG_HOSTS variable # Use the file borg-conf.env to set these. # # If it finds an item in the BORG_REPOS that isn't yet a borg repository on one # of the BORG_HOSTS, it will init a new repo there. # # Dependencies: # packages: borg > 1.4 # scripts: /usr[/local]/bin/driglibash-base # files: /data/borg/config/borg-conf.env # /data/borg/config/.borgexclude # Cheatsheet: # ${#array[@]} number of elements in array # ${array[@]} each element in array (separate words) # ${array[i]} i-th element in array if test -s /usr/local/bin/driglibash-base -a -r /usr/local/bin/driglibash-base ; then . /usr/local/bin/driglibash-base elif test -s /usr/bin/driglibash-base -a -r /usr/bin/driglibash-base ; then . /usr/bin/driglibash-base else die "Could'nt source driglibash. See https://github.com/adrian-amaglio/driglibash/" fi BORG_ENV="/data/borg/config/borg-conf.env"; test -s "$BORG_ENV" && test -r "$BORG_ENV" || die "Couldn't find \"$BORG_ENV\" configuration file!" . "$BORG_ENV" mkdir -p "$BORG_BASE_DIR" "$BORG_CACHE_DIR" "$BORG_CONFIG_DIR" "$BORG_TMPDIR" "$BORG_SECURITY_DIR" "$BORG_SECURITY_DIR/passphrases" "$BORG_SECURITY_DIR/repokeys" function init_repo() { # args : # $1 : host (local path or ssh where the borg repo is stored) # $2 : path (local dir(s) to be saved in the repo) # $3 : name of the repo on (remote) host # $4 : unique alias to identiy the host test "$verbosity" -gt 0 && echo "init_repo( $1 \\ $2 \\ $3)" mkdir -p "$BORG_SECURITY_DIR/passphrases/$4/" mkdir -p "$BORG_SECURITY_DIR/repokeys/$4/" #create passphrase LC_ALL=C tr -dc A-Za-z0-9 "$BORG_SECURITY_DIR/passphrases/$4/$3" export BORG_PASSPHRASE=$(cat "$BORG_SECURITY_DIR/passphrases/$4/$3") #init repo test "$verbosity" -gt 1 && echo "borg init ${verbosity:+"--progress"} --make-parent-dirs -e repokey "$1/$3"" test "$verbosity" -gt 3 && read -p " Continue ?" run borg init ${verbosity:+"--progress"} --make-parent-dirs -e repokey "$1/$3" #create first entry test "$verbosity" -gt 1 && echo "borg create ${verbosity:+"--progress"} ${BORG_EXCLUDE_FILE:+"--exclude-from $BORG_EXCLUDE_FILE"} "$1/$3"::"init-$(date +%Y-%m-%d_%H-%M-%S)" "$2"" test "$verbosity" -gt 3 && read -p " Continue ?" run borg create ${verbosity:+"--progress"} ${BORG_EXCLUDE_FILE:+--exclude-from "$BORG_EXCLUDE_FILE"} "$1/$3"::"init-$(date +%Y-%m-%d_%H-%M-%S)" "$2" #export repokey in case of repo catastrophic loss test "$verbosity" -gt 1 && echo "borg key export "$1/$3" "$BORG_SECURITY_DIR/repokeys/$3"" test "$verbosity" -gt 3 && read -p " Continue ?" run borg key export "$1/$3" "$BORG_SECURITY_DIR/repokeys/$4/$3" #TODO These keys should be backuped somewhere } for alias in "${!host_mode[@]}" ; do # Begin parameter validation test -n "${host_repo_dir["$alias"]}" && test -d "${host_repo_dir[$alias]}" || pathchk -p -P "${host_repo_dir["$alias"]}" 2>/dev/null && mkdir -p "${host_repo_dir[$alias]}" || die "Config error! Host $alias : "${host_repo_dir["$alias"]}" isn't a valid repo dir." if test "${host_mode[$alias]}" = "local" ; then host="${host_repo_dir[$alias]}" elif test "${host_mode[$alias]}" = "ssh" ; then test -n "${host_user["$alias"]}" && echo "${host_user["$alias"]}" | grep -q -E "^[a-z_][a-z0-9_-]*$" || die "Config error! Host $alias : ${host_user["$alias"]} isn't a valid username." test -z ${host_host["$alias"]} && die "Config error! Host $alias : you must provide a host in ssh mode!" check_host=false # IPv4 regexp echo ${host_host["$alias"]} | grep -q -E "^([0-2]?[0-9]{1,2}\.){3}[0-2]?[0-9]{1,2}$" && check_host=true # IPv6 regexp echo ${host_host["$alias"]} | grep -q -E "^(((([a-f]|[0-9]){1,4})|:):){6}([a-f]|[0-9]){1,4}$" && check_host=true # URL regexp echo ${host_host["$alias"]} | grep -q -E "^[0-9a-zA-Z]([-.\w]*[0-9a-zA-Z])*\.[a-z]{2,5}$" && check_host=true test "$check_host" = true || die "Config error! Host $alias : ${host_host["$alias"]} isn't a valid host (expected IPv4, IPv6 or URL)." test -n "${host_port["$alias"]}" && test "${host_port["$alias"]}" -gt 2>/dev/null 0 && test "${host_port["$alias"]}" -le 65536 || die "Config error! Host $alias : "${host_port["$alias"]}" isn't a valid port." # End parameter validation # Parameter expansion lvl: I was not ready for this. host="ssh://${host_user["$alias"]:+${host_user["$alias"]}@}\ ${host_host["$alias"]:+${host_host["$alias"]}}\ ${host_port["$alias"]:+:${host_port["$alias"]}}\ ${host_repo_dir["$alias"]:+${host_repo_dir["$alias"]}}" # super-secret-back-door elif test "${host_mode[$alias]}" = "iknowwhatimdoing" ; then host="${host_host["$alias"]}" else die "Config error! Host $alias : unrecognized mode ${host_mode[$alias]}" fi test "$verbosity" -gt 0 && section "$alias: $host" for repo in "${local_repos[@]}" ; do test "$verbosity" -gt 0 && section "$repo" # we use a python-like name for the repo: reponame=$(echo "$repo" | tr "/" ".") #Check that the repo exists (we could be backuping a new service) check_repo_exists=false; if test "${host_mode[$alias]}" = "ssh" ; then export BORG_PASSPHRASE=$(cat "$BORG_SECURITY_DIR/passphrases/$alias/$reponame") && borg list "$host/$reponame" > /dev/null && check_repo_exists=true || "Could'nt open repo $reponame at host $host. Creating it." fi test "${host_mode[$alias]}" = "local" && test -d "$host/$reponame" && test -s "$host/$reponame/README" && grep -q "This is a Borg Backup repository." "$host/$reponame/README" && check_repo_exists=true #TODO: this doesn't check if a distant repo exists if $check_repo_exists = true ; then #it's okay, repo exists, start the normal backup test -s "$BORG_SECURITY_DIR/passphrases/$alias/$reponame" && export BORG_PASSPHRASE=$(cat "$BORG_SECURITY_DIR/passphrases/$alias/$reponame") || die "Couldn't get passphrase for repo $alias/$repo from file: $BORG_SECURITY_DIR/passphrases/$alias/$reponame" test $verbosity -gt 1 && echo "borg create ${verbosity:+"--progress"} ${BORG_EXCLUDE_FILE:+--exclude-from "$BORG_EXCLUDE_FILE"} --compression obfuscate,115,auto,zstd,20 "$host/$reponame"::"$reponame-$(date +%Y-%m-%d_%H-%M-%S)" "$repo"" test $verbosity -gt 3 && read -p " Continue ?" run borg create ${verbosity:+"--progress"} ${BORG_EXCLUDE_FILE:+--exclude-from "$BORG_EXCLUDE_FILE"} --compression obfuscate,115,auto,zstd,20 "$host/$reponame"::"$reponame-$(date +%Y-%m-%d_%H-%M-%S)" "$repo" #TODO Check that zstd lvl 20 compression is not too cpu-intensive, could be reduced (or use lz4) (see borg help benchmark) # Global retention parameters hourly=${BORG_KEEP_HOURLY[all]:+"--keep-hourly=${BORG_KEEP_HOURLY[all]} "} daily=${BORG_KEEP_DAILY[all]:+"--keep-daily=${BORG_KEEP_DAILY[all]} "} weekly=${BORG_KEEP_WEEKLY[all]:+"--keep-weekly=${BORG_KEEP_WEEKLY[all]} "} monthly=${BORG_KEEP_MONTHLY[all]:+"--keep-monthly=${BORG_KEEP_MONTHLY[all]} "} yearly=${BORG_KEEP_YEARLY[all]:+"--keep-yearly=${BORG_KEEP_YEARLY[all]} "} test $verbosity -gt 2 && echo "Global retention policy : $hourly $daily $weekly $monthly $yearly" # Per-host retention parameters test -n "${BORG_KEEP_HOURLY["$alias"]}" && hourly="--keep-hourly=${BORG_KEEP_HOURLY["$alias"]}" test -n "${BORG_KEEP_DAILY["$alias"]}" && daily="--keep-daily=${BORG_KEEP_DAILY["$alias"]}" test -n "${BORG_KEEP_WEEKLY["$alias"]}" && weekly="--keep-weekly=${BORG_KEEP_WEEKLY["$alias"]}" test -n "${BORG_KEEP_MONTHLY["$alias"]}" && monthly="--keep-monthly=${BORG_KEEP_MONTHLY["$alias"]}" test -n "${BORG_KEEP_YEARLY["$alias"]}" && yearly="--keep-yearly=${BORG_KEEP_YEARLY["$alias"]}" test $verbosity -gt 2 && echo "$alias retention policy : $hourly $daily $weekly $monthly $yearly" test $verbosity -gt 1 && echo "borg prune ${verbosity:+"--progress"} --list --glob-archives \"$reponame*\" $hourly $daily $weekly $monthly $yearly \"$host/$reponame\"" test $verbosity -gt 3 && read -p " Continue ?" run borg prune ${verbosity:+"--progress"} --list --glob-archives \"$reponame*\" $hourly $daily $weekly $monthly $yearly "$host/$reponame" else #If repo doesn't exist, create it init_repo "$host" "$repo" "$reponame" "$alias" fi done done