#!/bin/bash

# Daire Byrne - 20/09/08
# This script does lustre backups to /net/epsilon.
# Backups can be "full" or "incremental" and optionally 
# syncronise deletes. To be scheduled by a cronjob.

CHUNKS=60
HOURS=72
DATE=`date +%Y%m%d`
RAND=$RANDOM
E2SCAN="/usr/sbin/e2scan"
MDT_DEV="/dev/lustre/mdt1"
LIST="modified"
MDS_BACK="mds1e"
COMM_NEW="-2"
COMM_DEL="-1"
ROOT=""

export LC_COLLATE="en_US.UTF-8"

usage() {
    echo "Usage:  `basename $0` {-F|-H|-I|-R} [ options ] profile-name"
    echo "Modes:  -F: full backup"
    echo "        -H: hardlink (implies -F)"
    echo "        -I: incremental"
    echo "        -R: restore"
    echo "Options:"
    echo "        -h help:    this usage message"
    echo "        -d deletes: syncronise deletes"
    echo "        -r root:    subdir to consider (default /)"
    echo "        -c chunks:  number of copy splits (default 60)"
    echo "        -t time:    modified files since <hours> ago (default 72)"
    echo "        -n dryrun:  dry-run rsyncs and skip deletes"
    echo "Examples:"
    echo "        `basename $0` -F -d iota       # do a full sync of iota"
    echo "        `basename $0` -I -t168 iota    # copy last week of modified files"
    echo "        `basename $0` -H iota          # backup with hardlinks"
    echo "        `basename $0` -R -r film iota  # restore latest \"film\" to iota"
    exit
}

setup() {
    [ "$1" = "" ] && usage

    while getopts "FHIRdhnr:c:t:" options; do
      case $options in
        F ) FULL=1;;
        H ) HARD=1;;
        I ) INCR=1;;
        R ) REST=1;;
        d ) DELS=1;;
        n ) DRYR=1;;
        r ) ROOT=$OPTARG;;
        t ) HOURS=$OPTARG;;
        c ) CHUNKS=$OPTARG;;
        h ) usage;;
        \? ) usage
             exit 1;;
        * ) usage
             exit 1;;
      esac
    done
    shift $(($OPTIND - 1))
    PROFILE=$1

    [ -z $FULL ] && [ -z $INCR ] && [ -z $REST ] && [ -z $HARD ] && usage

    PATH_SRC="/net/${PROFILE}/${ROOT}"
    PATH_DST="/net/epsilon/${PROFILE}/${ROOT}"
    HARD_SRC="/net/epsilon/${PROFILE}/${ROOT}"
    HARD_DST="/net/epsilon/${PROFILE}.${DATE}"

    if [ "$PROFILE" = "alpha" ]; then
        MDS_MAIN="mds1a"
        MDT_DEV="/dev/lustre/mdt1a"
    elif [ "$PROFILE" = "delta" ]; then
        MDS_MAIN="mds1d"
    elif [ "$PROFILE" = "iota" ]; then
        MDS_MAIN="mds1i"
        EXCLUDES='tmp logs common/fcq_logs avid_in avid_out film/*/*/work/*/maya/rib film/*/*/work/*/maya/rmantex'
    else
       echo -e "unknown profile! - $PROFILE\n"
       exit
    fi

    RSYNC_OPTS="$PATH_SRC $PATH_DST"
    MODTIME=`date --date="$HOURS hours ago" +%s`
    OUTDIR="/net/soft/systems/flbackups/${PROFILE}.${DATE}.${RAND}"
    LOCKFILE=/var/run/flbackup.$PROFILE.lock

    [[ $FULL ]] && LIST="$MDS_MAIN"
    [[ $HARD ]] && LIST="$MDS_MAIN" && FULL=1 && RSYNC_OPTS="-H --link-dest=$HARD_SRC $PATH_SRC $HARD_DST"
    [[ $DRYR ]] && RSYNC_OPTS="--dry-run $RSYNC_OPTS"
    [[ $REST ]] && RSYNC_OPTS="-u $PATH_DST $PATH_SRC" && ROOT="${PROFILE}/${ROOT}" 
    [[ $REST ]] && PATH_DST="$PATH_SRC" && MODTIME=0 && COMM_NEW="-1" && COMM_DEL="-2" 
}

die() {
    echo "  * error - " $@ >&2
    exit 1
}

log() {
    LOGSTAMP=`date +"["%d-%m-%Y" "%R":"%S"]"`
    echo -ne $LOGSTAMP "${1}\n"
}

checks() {
    log "doing preflight checks ..."
    [ "`hostname -s`" = "lust1e-mgmt" ] || die "must run on lust1e!"
    [ -e $LOCKFILE ] && die "lockfile already exists!"
    [ -e /var/run/rsync-${PROFILE}.lock ] && die "syncer lockfile already exists!"
    [ -e /opt/sge/default/common/settings.sh ] || die "can't find SGE environment!"

    mkdir -p $PATH_DST || die "couldn't create backup destination directory!"
    [ $HARD ] && mkdir -p $HARD_DST || die "couldn't create backup destination directory!"
    [ -d ${PATH_SRC} ] || die "${PATH_SRC}/ path doesn't exist!"
    mkdir -m 0777 -p $OUTDIR || die "couldn't create output directory!"

    log "output: $OUTDIR"
    log " done.\n"
}

build_excludes() {
    if [ ! "$EXCLUDES" = "" ]; then
        for x in $EXCLUDES; do
            RSYNC_EXCLUDES="$RSYNC_EXCLUDES --exclude=\"${x}/**\""
        done
    else
        RSYNC_EXCLUDES=""
    fi
}

e2scan_sync() {
    log "doing comparison e2scans ..."
    ssh $MDS_MAIN "$E2SCAN -l -D -C "/ROOT/${ROOT}" -N 0 ${MDT_DEV} | sort -S 2G -o ${OUTDIR}/files.${MDS_MAIN}" &
    ssh $MDS_BACK "$E2SCAN -l -D -C "/ROOT/${PROFILE}/${ROOT}" -N 0 /dev/lustre/mdt1e | sort -S 4G -o ${OUTDIR}/files.${MDS_BACK}" &
    wait || die "one or more e2scan processes failed!"
    log " done.\n"
}

e2scan_modified() {
    [[ $INCR ]] && log "doing $HOURS hour incremental e2scan ..."
    [[ $FULL ]] && log "doing full e2scan ..." && MODTIME=0
    [[ $REST ]] && log "doing restore e2scan ..." && HACK="e" && MDS_MAIN="${MDS_BACK}"

    ssh $MDS_MAIN "$E2SCAN -l -D -C \"/ROOT/${ROOT}\" -N $MODTIME ${MDT_DEV}${HACK} | grep -x -v \".\" > ${OUTDIR}/files.${LIST}" & 
    wait || die "e2scan process failed!"
    MODFILES=`cat $OUTDIR/files.${LIST} | wc -l`; log "$MODFILES modified files."
    log " done.\n"
}

comm_sync() {
    log "generating new and delete file lists ..."
    comm $COMM_NEW -3 $OUTDIR/files.${MDS_MAIN} $OUTDIR/files.${MDS_BACK} > $OUTDIR/files.new &
    comm $COMM_DEL -3 $OUTDIR/files.${MDS_MAIN} $OUTDIR/files.${MDS_BACK} | grep -x -v "." | tac > $OUTDIR/files.deletes &
    wait || die "one or more comm processes failed!"
    NEWFILES=`cat $OUTDIR/files.new | wc -l`; log "$NEWFILES new files."
    log " done.\n"
}

split_chunks() {
    log "splitting files.$LIST list into $CHUNKS chunks ..."
    FILES=`cat ${OUTDIR}/files.${LIST} | wc -l`
    LINES=`echo ${FILES}/${CHUNKS} | bc`
    csplit -s -n1 -f ${OUTDIR}/files. ${OUTDIR}/files.$LIST ${LINES} \{`expr ${CHUNKS} - 2`\} || die "couldn't split files!"
    mv ${OUTDIR}/files.0 ${OUTDIR}/files.${CHUNKS} || die "couldn't split files!"
    log " done.\n"
}

do_deletes() {
    DELETES=`cat ${OUTDIR}/files.deletes | wc -l`
    log "doing $DELETES deletes in $PATH_DST ..."
    cd $PATH_DST || die "couldn't access destination directory!"
    [ $DELETES -gt 0 ] && [ -z $DRYR ] && sed 's/^/"/;s/**$/"/' ${OUTDIR}/files.deletes | xargs rm -rf -v > ${OUTDIR}/deletes.log
    [ $DELETES -gt 0 ] && [ $DRYR ] && awk '{print "skipping delete: ", $0}' ${OUTDIR}/files.deletes
    log " done.\n"
}

submit_sge() {
    source /opt/sge/default/common/settings.sh
    log "submitting SGE copy jobs ..."
    cat > ${OUTDIR}/qsub.in <<- END
	#!/bin/sh

	#\$ -N rsync-${PROFILE}
	#\$ -t 1-${CHUNKS}
	#\$ -j y
	#\$ -r y
	#\$ -o ${OUTDIR}
	#\$ -q systems.q
	#\$ -l hostname='rr*,arch=lx24-amd64'
	#\$ -sync y

	echo "running on \$HOSTNAME ..."
	echo "-------------------------------"
	if [ -d /net/${PROFILE}/.snapshots ] && [ -d /net/epsilon/${PROFILE} ]; then	
	    rsync -avx --stats --files-from=${OUTDIR}/files.\$SGE_TASK_ID $RSYNC_EXCLUDES $RSYNC_OPTS
	else
	    echo "could not access source or destination path on this machine!";echo
	    exit 99
	fi

	RSYNC_EXIT=\$?
	echo "RSYNC_EXIT=\$RSYNC_EXIT"

	EXIT=0

	ERROR_COUNT=\`grep -c '^rsync error' \$SGE_STDOUT_PATH\`
	RSYNC_DONE=\`grep -c '^total size is' \$SGE_STDOUT_PATH\`

	[ \$RSYNC_EXIT -ne 0 ] && [ \$RSYNC_EXIT -ne 24 ] && \
	  echo "error state exit code from rsync" ##&& EXIT=99
	[ \$ERROR_COUNT -ne 0 ] && \
     	  echo "found \$ERROR_COUNT instances of error in rsync log" ##&& EXIT=99
	[ \$RSYNC_DONE -eq 0 ] && \
	  echo "rsync didn't finish!" && EXIT=99
	exit \$EXIT
END
    qsub ${OUTDIR}/qsub.in
    log " done.\n"
}

cleanup_files() {
    log "removing temp files ..."
    rm -f $OUTDIR/files.modified $OUTDIR/files.$MDS_MAIN $OUTDIR/files.$MDS_BACK
    rm -f $LOCKFILE
    log " done.\n"
}

########
# Main #
setup $@
checks
build_excludes

touch $LOCKFILE
# Backup with deletes
if [[ "$DELS" = "1" ]]; then
    e2scan_sync
    comm_sync
    do_deletes
    if [[ "$INCR" = "1" ]]; then
        e2scan_modified
    fi
# Backup without deletes
else
    e2scan_modified
fi
split_chunks
submit_sge
cleanup_files