* Backup Scripts in bash and batch

Dec 062011
 
 December 6, 2011  Posted by jason at 1:40 pm backup , , , , , , , , , , ,  No Responses »

Here is a bash script that I use to do simple and transparent backups of all sorts of data.
I use it primarily to backup to multiple external hard drives, plugged in via USB. It uses regular mount or gnome volume manager. It also sends logs via email using “mail” and a local MTA like postfix. Please modify to fit your needs. Leave a comment if you want. Its also here http://jasonschaefer.com/stuff/jsbk.sh.txt

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163

#!/bin/bash
##jsbk.sh 12-07-11
##email: jason [at] schaeferconsulting [dot] com
##505.570.1114
 
########################## USER EDIT BEGINS #############################
## Email to, comma separated list for multiple recipients. host needs to be able to relay local mail. postfix recommended.
to=""
## Set "yes" to rely on gnome volume manager (/media/[drivename]). Leave blank for manual mount (requires root priv. and will mount to /mnt)
automount="yes"
## Name of drive. If you have multiple backup drives, they should be named the same.
drivename="mybackupdrive"
## Name of directory to write backups in. Leave blank to write to the root of backup drive.
bkdir="mybackups"
## Cleanup backup files when drive reaches this percent
cleanafter="99" #percent
## When backup is full, delete how many days of backups?
deletedaysold="30" #days
## If you are in need of assistance, please contact [the following string]
support="Schaefer Consulting at 505.570.1114"
## To bypass the backup commands, for debug purposes. yes/no
bypassbk="no"
backup_commands () {
########################## START BACKUP COMMANDS ########################
echo "Running backup commands now..."   >> $log 2>&1
## Enter backup line's here i.e., cp, ssh, rsync ####
## append "||check" to the end of the command to verify if the command ran successfully
## exmpl:    /bin/tar zcvf $bklocation/`date +%F`.tar.gz --exclude media/* /home ||check 2>> $log
## exmpl:    /bin/cp -rvu /home/ $bklocation/new-n-changed ||check 2>> $log
## exmpl:    /usr/bin/rsync -av --delete --stats --exclude media/* /home/ $bklocation/rsync-mirror ||check 2>> $log
## exmpl:    /usr/bin/pg_dump -Z 9 -p 5432 -h hostname -U username dbname > $bklocation/dbname_`date +%F_%T`.sqldump.gz ||check 2>> $log
##			         postgres auth uses ~/.pgpass containing localhost:5432:db:dbuser:dbpass
## BACKUP over SMB
## exmpl:		 cd $bklocation/backup-destination && /usr/bin/smbget -URq smb://user:password@hostname/share 2>> $log
 
 
########################### END BACKUP COMMANDS #########################
########################### USER EDIT ENDS ##############################
########################### CRON REFERENCE ##############################
## setup to run on a schedule with cron
## minutes (0-59) | hours (0-23) | day of month (1-31) | month (1-12) | day of week (0-6 or Sun-Sat 0,6) | command
## 0 2 * * * /usr/local/bin/jsbk.sh
## will run at 2:00am every day 
## 0 2 1,15 * * /usr/local/bin/jsbk.sh
## will run at 2:00am 1st and 15th of the month
}
#if you want to see the log on your screen, use log="/dev/stdout" instead
log="/tmp/jsbk.backup.log"
#log="/dev/stdout"
subject="Backup Log from `hostname`"
#mountpoint, for easy reference later
if [ "$automount" = "yes" ]; then
  mp="/media/$drivename"
  else
  mp="/mnt"
fi
#mount point plus backup dir, for easy reference in backup commands
bklocation="$mp/$bkdir"
#determine percent that drive is full
percentfull=`df -h |grep sfmgbackup |awk '{print $5}' |sed 's:%::g'`
starttime=`date +%s`
echo $bklocation
if [ -d "$bklocation" ];then 
  echo bklocation=\"$bklocation\" exists, moving on
  else
  echo bklocation=\"$bklocation\" is not a directory. stripping bkdir, will write files root of $mp
  bkdir=""
fi
trap caughtsig 1 2 3 6
caughtsig () {
  echo "******************** Caught INTERUPT SIG 1,2,3, or 6. **********************" >> $log 2>&1
  exit 1
}
#check if backup is still running or exited without cleanup.
if [ -f $log ]; then
  echo "log ($log) exists, this indicates that $0 may be running or exited uncleanly. verify with ps aux |grep $0 and if not running, delete $log." >> $log 2>&1
  error
  emailend
fi
check () {
  echo "ERROR: A command FAILED!" >> $log 2>&1
  failed="failed"
}
error () {
  echo >> $log 2>&1
  echo -e "\t\t * Backup is NOT working!" >> $log 2>&1
  echo -e "\t\t * Contact technician for service." >> $log 2>&1
  echo -e "\t\t * Keep this message for diagnoses" >> $log 2>&1
  echo -e "\t\t * Backup Failed on `date +%c`"   >> $log 2>&1
  echo -e "\t Diagnoses:" >> $log 2>&1
  #mount >> $log 2>&1
  echo >> $log 2>&1
  df -h >> $log 2>&1
  echo >> $log 2>&1
  echo Mount Point: $mp   >> $log 2>&1
}
emailend () {
  echo >> $log 2>&1
  #calculate running time of backup
  endtime=`date +%s`
  elapsed=$(($endtime - $starttime))
  printf "Running time: "%dh:%dm:%ds"\n" $(($elapsed/3600)) $(($elapsed%3600/60)) $(($elapsed%60)) >> $log 2>&1    
  echo "---" >> $log 2>&1
  echo "If you are in need of assistance, please contact $support" >> $log 2>&1
  echo >> $log 2>&1
  if [ -z "$automount" ]; then
    umount $mp >> $log 2>&1 || { echo -e "Error: Removing hard drive failed\n";} >> $log 2>&1
  fi
  echo "Emailing logs..." >> $log 2>&1
  if [ ! -z "$to" ];then
    /bin/cat $log | /usr/bin/mail -v -s "$subject" $to 
    else
    /bin/mv $log /tmp/jsbk.last.log
    fi
  exit
}
[ "$automount" != yes ] && [ ! -z "$automount" ] && echo ERROR: automount=\"$automount\" is not valid. >> $log 2>&1 && error && emailend
[ -z "$drivename" ] && echo ERROR: drivename is not declared >> $log 2>&1 && error && emailend
[ -z "$bkdir" ] && echo fyi, bkdir is not declared
[ -z "$support" ] && echo fyi, support string is empty
#begin
echo "BACKUP LOG"   >> $log 2>&1
echo "server: `hostname`" >> $log 2>&1
echo -e "Backup BEGAN on `date +%c` \n"   >> $log 2>&1
if [ ! -d "$mp" ];then
  echo "Mount point $mp does not exist!" >> $log 2>&1
if [ "$automount" = "yes" ]; then echo "be sure drivename=\"$drivename\" is correct and gnome volume mounting is working. Or switch to automount=\"no\"" >> $log 2>&1; fi
if [ "$automount" = "no" ]; then echo "check that $mp exists" >> $log 2>&1; fi
echo "Also, be sure the drive is plugged in or turned on." >> $log 2>&1
error
emailend
fi
if [ "$automount" = "no" ]; then
  #mount drive manually at /mnt
  mount -L $drivename $mp >> $log 2>&1 || { echo "ERROR: mounting hard drive failed. Be sure drivename=\"\" is correct. If drive is already mounted this error can be ignored. Also, be sure the drive is plugged in or turned on.";} >> $log 2>&1
fi
echo -e "\n +++ START LOG +++ \n" >> $log 2>&1
if [ "$bypassbk" = "yes" ]; then
  touch $bklocation/bypassbk.test
  echo "Bypassing all backup commands for debug purposes" >> $log 2>&1
  emailend
else
echo "Checking if drive has reached $percentcleanafter% full" >> $log 2>&1
if [ $percentfull -gt $percentcleanafter ];then
echo "Drive has exceeded $percentcleanafter% and all backups older than $deletedaysold will be deleted" >> $log 2>&1
find $bklocation/* -mtime +$deletedaysold -exec rm -fr {} \;  >> $log 2>&1
else
echo "backup drive storage has not exceeded $percentcleanafter, proceeding as usual. Drive currently has $percentfull% used" >> $log 2>&1
fi
  #user backup commands function
  backup_commands
fi
echo -e "\n +++ END LOG +++ \n" >> $log 2>&1
#check to see if $failed equals zero, if it is then all commands were successfull. If not, something failed.
if [ "$failed" = failed ]; then
  echo "ERROR: A problem was detected!" >> $log 2>&1
  error
  emailend
else
  echo -e "\t SUCCESS: Backup Complete!" >> $log 2>&1
  echo -e "\t\t " >> $log 2>&1
  emailend
fi

Here is my batch script I use for MS Windows boxen. Batch scripting is nasty but surprisingly this script has come in handy many a time. Here is a direct link http://jasonschaefer.com/stuff/jsbkwin.bat.txt

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

@echo off
echo jsbkwin 12-07-11
:: INSTRUCTIONS:
:: Save this file to it. Remove the .txt from the name.
:: Download blat http://www.blat.net/ and copy the 3 files (blat.exe, blat.dll, blat.lib) into c:\winnt or c:\windows
:: Ideally, blat will just send the mail. But more and more ISP's are blocking this. You can request that your ISP allow smtp or send on another port or use a remote mail server.
:: Use http://www.stunnel.org/ to make secure (tls) connections for sending mail. See example stunnel.conf at bottom
:: Also, http://www.cygwin.com/ for rsync, ssh and unix2dos on windows.
 
:: Edit file according to your needs and run.
:: This can be run automatically with "at" or "scheduled tasks",
:: You can uncomment the pause's for debugging.
 
::uncomment options as needed (connecting to outside MTA, for example)
set to=""
set from=""
set smtp_server="127.0.0.1"
::set smtp_port="4465"
::set username=""
::set password=""
set subject="Backup logs from %computername%"
set log="jsbkwin_log.txt"
 
::File extensions to be excluded, uncomment and modify as needed.
::echo .avi >> exclude.txt
::echo .mp3 >> exclude.txt
echo exclude.txt >> exclude.txt
 
::XCOPY help
:: /s   copied recursively except empty dirs
:: /e   copies empty dirs
:: /v   verifies file writes
:: /y   answers yes with out prompt
:: /i   assumes dirs are dirs (duhh)
:: /h   copies hidden and system files
:: /d   copies new or changed files
:: /EXCLUDE:[file]    files or strings to exclude
::  >> %log% 2>&1     redirects standard out and error and appends to log
 
echo.
echo Press "X" at top right of window to cancel.
echo backup screen will be blank throughout backup process. Check log for
echo activity and results.
echo log location: %log%
::uncomment "pause" for running manual or troubleshooting
::pause
echo.
 
echo ----------BEGIN---------- >> %log%
echo Version: %version% >> %log%
date /t >> %log% 2>&1
time /t >> %log% 2>&1
echo.
echo Backed up files:
echo.
 
echo running rsync-mirror backup now >> %log% 2>&1
::example
:: C:\cygwin\bin\rsync -rut --delete --stats "/cygdrive/c/DATA" "/cygdrive/e/daily/rsync-mirror" >> %log% 2>&1
:: C:\cygwin\bin\rsync -rut --delete --stats "/cygdrive/c/MT0" "/cygdrive/e/daily/MT0" >> %log% 2>&1
::echo. >> %log% 2>&1
::echo. >> %log% 2>&1
::echo running new-n-changed backup now >> %log% 2>&1
::echo. >> %log% 2>&1
:: xcopy /s/e/v/y/i/h/d/EXCLUDE:exclude.txt "c:\DATA" "e:\daily\new-n-changed" >> %log% 2>&1
 
echo.
date /t >> %log% 2>&1
time /t >> %log% 2>&1
echo ----------END---------- >> %log%
echo.
 
C:\cygwin\bin\unix2dos.exe %log%
 
echo mail log
blat %log% -to %to% -subject %subject% -hostname %computername% -server %smtp_server% -f %from% -port %smtp_port% -u %username% -pw %password%
 
del exclude.txt
del %log%
echo.
echo BACKUP COMPLETE
::uncomment "pause" for troubleshooting
::pause
 
::stunnel.conf example:
::client = yes
::debug = debug
::[pop3s]
::accept = 127.0.0.1:9995
::connect = pop3s.myisp.com:995
::[imaps]
::accept = 127.0.0.1:9993
::delay = yes
::connect = imaps.myisp.com:993
::[smtps]
::accept = 127.0.0.1:4465
::connect = smtps.myisp.com:465