Difference between revisions of "Smart-Backup-Source"
(Created page with '{{Admon/important| Warning! | This script is still a work in progress, there are no known bugs, but the script will change over time.}} = Smart Backup Source = * This script is…') |
(→Smart Backup Source) |
||
Line 13: | Line 13: | ||
</pre> | </pre> | ||
− | * Source code of | + | * Source code of smart-bk.py |
<pre> | <pre> |
Latest revision as of 14:43, 7 August 2013
Smart Backup Source
- This script is still a work in progress
- To show/track which version is posted here
commit b6eebe327175c1ca8e444f15561402e4655b1388 Author: Andrew Oatley-Willis <andrew.oatley-willis@senecacollege.ca> Date: Wed Aug 7 14:35:36 2013 -0400
- Source code of smart-bk.py
#!/usr/bin/env python # Andrew Oatley-Willis # Smart backup script # Should be used to make intelligent backups of systems so systems are not overloaded # Should be made to be simple to use and configure import datetime import optparse import pysftp import sys import urllib2 import string import os import sqlite3 as lite # Create/remove a schedule, get schedule information class schedule: def __init__(self): self.logdir = '/var/log/smart-bk/' self.database = '/data/smart-bk/schedule.db' self.updateTime() self.updateSchedules() # Update time def updateTime(self): # Get the date and time and store it self.now = str(datetime.datetime.now()).split(' ') self.year, self.month, self.day = self.now[0].split('-') self.hours, self.minutes, self.seconds = self.now[1].split(':') self.hours, self.minutes = int(self.hours), int(self.minutes) # Update schedules def updateSchedules(self): # Get database and put it in lists self.schedule, self.queue, self.running = self.listSchedule() # Get busy hosts and ids and schedules self.busyhosts = [] self.busyids = [] self.busyschedules = [] for item in self.schedule: for run in self.running: if item[0] == run[0]: if item not in self.busyschedules: self.busyschedules.append(item) if item[0] not in self.busyids: self.busyids.append(item[0]) if item[4] not in self.busyhosts: self.busyhosts.append(item[4]) if item[5] not in self.busyhosts: self.busyhosts.append(item[5]) # Get free hosts and ids and schedules self.freehosts = [] self.freeids = [] self.freeschedules = [] for item in self.schedule: if item not in self.busyschedules and item not in self.freeschedules: self.freeschedules.append(item) if item[0] not in self.busyids and item[0] not in self.freeids: self.freeids.append(item[0]) if item[4] not in self.busyhosts: self.freehosts.append(item[4]) if item[5] not in self.busyhosts: self.freehosts.append(item[5]) # Get queue hosts and ids and schedules self.queuehosts = [] self.queueids = [] self.queueschedules = [] for item in self.schedule: for queue in self.queue: if item[0] == queue[0]: if item not in self.queueschedules: self.queueschedules.append(item) if item[0] not in self.queueids: self.queueids.append(item[0]) if item[4] not in self.queuehosts: self.queuehosts.append(item[4]) if item[5] not in self.queuehosts: self.queuehosts.append(item[5]) # When you print object def __str__(self): return self.prettySchedule() # Create a new schedule def newSchedule(self, time, backuptype, sourcehost, desthost, sourcedir, destdir, sourceuser, destuser): output = self.day, time, backuptype, sourcehost, desthost, sourcedir, destdir, sourceuser, destuser self.writeLog(output) try: con = lite.connect(self.database) cur = con.cursor() cur.execute('INSERT INTO Schedule(day, time, type, source_host, dest_host, source_dir, dest_dir, source_user, dest_user) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)', (self.day, time, backuptype, sourcehost, desthost, sourcedir, destdir, sourceuser, destuser)) con.commit() con.close() except lite.Error, e: if con: con.rollback() con.close() output = 'Error: ' + e.args[0] self.writeLog(output) exit() # Give a schedule id and delete that schedule def removeSchedule(self, scheduleid): output = 'Removing scheduleid = ' + str(scheduleid) + ' from all tables' self.writeLog(output) try: con = lite.connect(self.database) cur = con.cursor() cur.execute('DELETE FROM Queue WHERE scheduleid = ?;', [scheduleid]) cur.execute('DELETE FROM Running WHERE scheduleid = ?;', [scheduleid]) cur.execute('DELETE FROM Schedule WHERE id = ?;', [scheduleid]) con.commit() con.close() except lite.Error, e: if con: con.rollback() con.close() output = 'Error: ' + e.args[0] self.writeLog(output) exit() # Output the schedule in a list def listSchedule(self): schedule = [] queue = [] running = [] try: con = lite.connect(self.database) with con: cur = con.cursor() case = cur.execute('SELECT * FROM Schedule') rows = cur.fetchall() for row in rows: # id, day, time, type, sourcehost, desthost, sourcedir, destdir, sourceuser, destuser schedule.append([row[0], row[1].strip(), row[2].strip(), row[3].strip(), row[4].strip(), row[5].strip(), row[6].strip(), row[7].strip(), row[8].strip(), row[9].strip()]) case = cur.execute('SELECT * FROM Queue') rows = cur.fetchall() for row in rows: # scheduleid, queuetime queue.append([row[0], row[1].strip()]) case = cur.execute('SELECT * FROM Running') rows = cur.fetchall() for row in rows: # scheduleid, starttime running.append([row[0], row[1].strip()]) return schedule, queue, running except lite.Error, e: if con: con.rollback() con.close() output = 'Error: ' + e.args[0] self.writeLog(output) exit() # Make the schedule look pretty and output it def prettySchedule(self): # Schedule print "\n\t" * 10 + "-[Schedule]-" print "-" * 100 print "id" + "|" + "day" + "|" + "time" + "|" + "type" + "|" + "source host" + "|" + "dest host" + "|" + "source dir" + "|" + "dest dir" + "|" + "source user" + "|" + "dest user" print "-" * 100 for item in self.schedule: print str(item[0]) + "|" + item[1] + "|" + item[2] + "|" + item[3] + "|" + item[4] + "|" + item[5] + "|" + item[6] + "|" + item[7] + "|" + item[8] + "|" + item[9] print "-" * 100 # Queue print "\n" * 10 + "-[Queue]-" print "-" * 100 print "|" + "schedule id" + "|" + "queue time" + "|" print "-" * 100 for item in self.queue: print "|" + str(item[0]) + "|" + item[1] + "|" print "-" * 100 # Running print "\n" * 10 + "-[Running]-" print "-" * 100 print "|" + "schedule id" + "|" + "start time" + "|" print "-" * 100 for item in self.running: print "|" + str(item[0]) + "|" + item[1] + "|" print "-" * 100 return "" # Check current time and time on schedules, add to queue if time passed def queueSchedules(self): self.updateSchedules() time = (self.hours * 60 * 60) + (self.minutes * 60) output = "Checking all schedules for expired times" self.writeLog(output) for item in self.schedule: self.updateSchedules() shours, sminutes = item[2].split(':') shours, sminutes = int(shours), int(sminutes) schedtime = (shours * 60 * 60) + (sminutes * 60) lastday = item[1] scheduleid = item[0] if lastday == self.day: continue if scheduleid in self.queueids: continue if scheduleid in self.busyids: continue # If the scheduled time has passed, move schedule into queue if time >= schedtime: output = 'Adding scheduleid = ' + str(scheduleid) + ' to queue' self.writeLog(output) try: # Add 0 for strings 1, 2, 3 to 01, 02, 03 - Important for minutes 12:03 looks like 12:3 if len(str(self.minutes)) == 1: self.minutes = '0' + self.minutes con = lite.connect(self.database) cur = con.cursor() cur.execute('INSERT INTO Queue(scheduleid, queuetime) VALUES(?, ?);', (scheduleid, str(self.hours) + ':' + str(self.minutes))) cur.execute('UPDATE schedule SET day=? where id=?;', (str(self.day), scheduleid)) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() return True def queueSchedule(self, scheduleid): output = 'Adding scheduleid = ' + scheduleid + ' to queue' self.writeLog(output) if scheduleid in self.queueids: return False try: con = lite.connect(self.database) cur = con.cursor() cur.execute('INSERT INTO Queue(scheduleid, queuetime) VALUES(?, ?);', (scheduleid, str(self.hours) + ':' + str(self.minutes))) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() return True def expireSchedule(self, scheduleid): output = 'Marking scheduleid = ' + scheduleid + ' as expired' self.writeLog(output) try: con = lite.connect(self.database) cur = con.cursor() cur.execute('UPDATE Schedule SET day=? where id = ?;', (str(int(self.day)-1), scheduleid)) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() return True # Delete a single queue def removeQueue(self, scheduleid): output = 'Deleting scheduleid = ' + scheduleid + ' from queue' self.writeLog(output) try: con = lite.connect(self.database) cur = con.cursor() cur.execute('DELETE FROM Queue WHERE scheduleid = ?', (scheduleid)) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() # Delete a single run def removeRunning(self, scheduleid): output = 'Deleting scheduleid = ' + scheduleid + ' from running' self.writeLog(output) try: con = lite.connect(self.database) cur = con.cursor() cur.execute('DELETE FROM Running WHERE scheduleid = ?', (scheduleid)) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() # Add a new running instance and make sure one isn't already running def newRunning(self, scheduleid): output = 'Adding scheduleid = ' + scheduleid + ' to running' self.writeLog(output) if scheduleid in self.queueids: return False try: con = lite.connect(self.database) cur = con.cursor() cur.execute('INSERT INTO running(scheduleid, starttime) VALUES(?, ?);', (scheduleid, str(self.hours) + ':' + str(self.minutes))) con.commit() except lite.Error, e: if con: con.rollback() output = 'Error: ' + e.args[0] self.writeLog(output) exit() finally: if con: con.close() return True # Find all hosts in queue, find which one needs to be run first, move hosts to running if no conflicts def startBackup(self): self.updateSchedules() hosts = [] if not self.queueschedules: return False for row in self.queueschedules: self.updateSchedules() # Easy to use variables for backups self.scheduleid = str(row[0]) self.backuptype = row[3] self.sourcehost = row[4] self.desthost = row[5] self.sourcedir = row[6] self.destdir = row[7] self.sourceuser = row[8] self.destuser = row[9] # Check if this backup is already running if row in self.busyschedules: output = 'Busy scheduleid = ' + self.scheduleid + ' already running' self.writeLog(output) continue if self.sourcehost in self.busyhosts or self.desthost in self.busyhosts: output = 'Busy scheduleid = ' + self.scheduleid + ' busy hosts = ', self.busyhosts self.writeLog(output) continue # Check hosts for connectivity if not self.connectHost(self.sourcehost, self.sourceuser): output = 'Unavailable host = ' + self.sourcehost + ' user = ' + self.sourceuser + ' no connection' self.writeLog(output) continue if not self.connectHost(self.desthost, self.destuser): output = 'Unavailable host = ' + self.desthost + ' user = ' + self.destuser + ' no connection' self.writeLog(output) continue if self.sourcedir == self.destdir: output = 'Warning sourcedir = ' + self.sourcedir + ' destdir = ' + self.destdir + ' overwriting directories with same name' self.writeLog(output) hosts.append(self.sourcehost) hosts.append(self.desthost) self.removeQueue(self.scheduleid) self.newRunning(self.scheduleid) # Start the backup here if self.backuptype == 'rsync': success = self.performRsync() elif self.backuptype == 'dbdump': success = self.performDbdump() elif self.backuptype == 'archive': success = self.performArchive() else: output = 'Error: unknown backup type = ' + self.backuptype + ' in schedule' self.writeLog(output) # Finish backup here self.removeRunning(self.scheduleid) if success: output = 'Success: ', row self.writeLog(output) else: output = 'Failed: ', row self.writeLog(output) return True # Log everything def writeLog(self, output): if isinstance(output, basestring): print str(output) else: for line in output: print str(line).strip() log = "" if len(str(self.minutes)) == 1: self.minutes = '0' + str(self.minutes) try: log = open(self.logdir + 'smart-bk-' + str(self.year) + '-' + str(self.month) + '-' + str(self.day) + '-' + str(self.hours) + '-' + str(self.minutes) + '.log', 'a+') if isinstance(output, basestring): log.write(str(output)+'\n') else: for line in output: log.write(str(line)) except Exception, e: print "Error: " + str(e) pass finally: if log: log.close() # Connect to the hosts, return True if success or False if not successful def connectHost(self, host, user): try: #response=urllib2.urlopen('http://'+host,timeout=1) srv = pysftp.Connection(host=host, username=user, log=True) srv.close() return True #except urllib2.URLError as err:pass except:pass return False # Check disk space of partition/lv where directory resides def availableSpace(self, user, host, directory): output = "" errors = "" srv = "" try: srv = pysftp.Connection(host=host, username=user, log=True) output = "df " + directory + " | awk '{print $4}' | grep '^[0-9]*$'" self.writeLog(output) output = srv.execute("df " + directory + " | awk '{print $4}' | grep '^[0-9]*$'") self.writeLog(output) except Exception, e: if output: self.writeLog(output) errors = "Error: " + str(e) self.writeLog(errors) pass finally: if srv: srv.close() if errors: return False return output # Check total space that this directory uses def usedSpace(self, user, host, directory): output = "" errors = "" srv = "" try: srv = pysftp.Connection(host=host, username=user, log=True) output = "du -s " + directory + " | awk '{print $1}' | grep '^[0-9]*$'" self.writeLog(output) output = srv.execute("du -s " + directory + " | awk '{print $1}' | grep '^[0-9]*$'") self.writeLog(output) except Exception, e: if output: self.writeLog(output) errors = "Error: " + str(e) self.writeLog(errors) pass finally: if srv: srv.close() if errors: return False return output # Backup is complete, clean, log, and email results def performRsync(self): output = "" errors = "" srv = "" try: srv = pysftp.Connection(host=self.sourcehost, username=self.sourceuser, log=True) output = 'sudo rsync -aHAXEvz --exclude "lost+found" ' + self.sourcedir + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir self.writeLog(output) output = srv.execute('sudo rsync -aHAXEvz --exclude "lost+found" ' + self.sourcedir + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir + ';echo $?') self.writeLog(output) if output[-1].strip() != '0': errors = 'Error: command returned a non-zero exit status' self.writeLog(errors) srv.close() except Exception, e: if output: self.writeLog(output) errors = "Error: " + str(e) self.writeLog(errors) pass finally: if srv: srv.close() if errors: return False return True # Perfom a archive def performArchive(self): tarfile = '/tmp/archive-' + str(self.year) + '-' + str(self.month) + '-' + str(self.day) + '-' + str(self.hours) + '-' + str(self.minutes) + '.tar.bz' output = "" errors = "" srv = "" try: srv = pysftp.Connection(host=self.sourcehost, username=self.sourceuser, log=True) output = 'sudo tar -cpjvf ' + tarfile + ' ' + self.sourcedir self.writeLog(output) output = srv.execute('sudo tar -cpjvf ' + tarfile + ' ' + self.sourcedir + ';echo $?') self.writeLog(output) if output[-1].strip() != '0': errors = 'Error: command returned a non-zero exit status' self.writeLog(errors) output = 'scp ' + tarfile + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir self.writeLog(output) output = srv.execute('scp ' + tarfile + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir + ';echo $?') self.writeLog(output) if output[-1].strip() != '0': errors = 'Error: command returned a non-zero exit status' self.writeLog(errors) srv.close() except Exception, e: if output: self.writeLog(output) errors = "Error: " + str(e) self.writeLog(errors) pass finally: if srv: srv.close() if errors: return False return True # Perform a dbdump on koji... Should probably change this to support a custom db name def performDbdump(self): kojifile = '/tmp/kojidb-' + str(self.year) + '-' + str(self.month) + '-' + str(self.day) + '-' + str(self.hours) + '-' + str(self.minutes) + '.sql' output = "" errors = "" srv = "" try: srv = pysftp.Connection(host=self.sourcehost, username=self.sourceuser, log=True) output = 'pg_dump koji > ' + kojifile self.writeLog(output) output = srv.execute('pg_dump koji > ' + kojifile + ';echo $?') self.writeLog(output) if output[-1].strip() != '0': errors = 'Error: command returned a non-zero exit status' self.writeLog(errors) output = 'scp ' + kojifile + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir self.writeLog(output) output = srv.execute('scp ' + kojifile + ' ' + self.destuser + '@' + self.desthost + ':' + self.destdir + ';echo $?') self.writeLog(output) if output[-1].strip() != '0': errors = 'Error: command returned a non-zero exit status' self.writeLog(errors) except Exception, e: if output: self.writeLog(output) errors = "Error: " + str(e) self.writeLog(errors) pass finally: if srv: srv.close() if errors: return False return True def main(): # Create command line options desc = """The smart backup scheduler program %prog is used to run backups from computer to computer. %prog does this by adding and removing schedules from a schedule database. Once added to the schedule database, %prog should be run with '--queue' in order to intelligently add hosts to a queue and start running backups. It is recommended to run this as a cron job fairly often, more fequently depending on the number of schedules.""" parser = optparse.OptionParser(description=desc, usage='Usage: %prog [options]') parser.add_option('-q', '--queue', help='queue schedules and start backups', dest='queue', default=False, action='store_true') parser.add_option('-a', '--add', help='add new schedule at specific time', dest='add', default=False, action='store_true') parser.add_option('-s', '--show', help='show the schedule and host info', dest='show', default=False, action='store_true') parser.add_option('-r', '--remove', help='remove existing schedule', dest='remove', default=False, action='store_true') parser.add_option('--remove-queue', help='remove existing schedule from queue', dest='removequeue', default=False, action='store_true') parser.add_option('--remove-run', help='remove existing schedule from running', dest='removerun', default=False, action='store_true') parser.add_option('--expire', help='expire the day in schedule', dest='expire', default=False, action='store_true') parser.add_option('--add-queue', help='add a single schedule to queue', dest='addqueue', default=False, action='store_true') parser.add_option('--sid', help='specify schedule id for removing schedules', dest='sid', default=False, action='store', metavar="scheduleid") parser.add_option('--time', help='specify the time to run the backup', dest='time', default=False, action='store', metavar="18:00") parser.add_option('--backup-type', help='archive, pg_dump, rsync', dest='backuptype', default=False, action='store', metavar="type") parser.add_option('--source-host', help='specify the source backup host', dest='sourcehost', default=False, action='store', metavar="host") parser.add_option('--source-dir', help='specify the source backup dir', dest='sourcedir', default=False, action='store', metavar="dir") parser.add_option('--source-user', help='specify the source user', dest='sourceuser', default=False, action='store', metavar="user") parser.add_option('--dest-host', help='specify the destination backup host', dest='desthost', default=False, action='store', metavar="host") parser.add_option('--dest-dir', help='specify the destination backup dir', dest='destdir', default=False, action='store', metavar="dir") parser.add_option('--dest-user', help='specify the destination user', dest='destuser', default=False, action='store', metavar="user") parser.add_option('--log-dir', help='specify the directory to save logs', dest='logdir', default=False, action='store', metavar="dir") (opts, args) = parser.parse_args() # No options entered if len(sys.argv[1:]) == 0: parser.print_help() exit(-1) # Option switches if opts.time: time = opts.time if opts.sid: scheduleid = opts.sid if opts.backuptype: backuptype = opts.backuptype if opts.sourcehost: sourcehost = opts.sourcehost if opts.desthost: desthost = opts.desthost if opts.sourcedir: sourcedir = opts.sourcedir if opts.destdir: destdir = opts.destdir if opts.sourceuser: sourceuser = opts.sourceuser if opts.destuser: destuser = opts.destuser # Option dependencies if opts.remove and not opts.sid: print "Option remove requires option sid" parser.print_help() exit(-1) if opts.expire and not opts.sid: print "Option expire requires option sid" parser.print_help() exit(-1) if opts.removerun and not opts.sid: print "Option remove-run requires option sid" parser.print_help() exit(-1) if opts.removequeue and not opts.sid: print "Option remove-queue requires option sid" parser.print_help() exit(-1) if opts.addqueue and not opts.sid: print "Option add-queue requires option sid" parser.print_help() exit(-1) if opts.add: if not opts.time or not opts.backuptype or not opts.sourcehost or not opts.desthost or not opts.sourcedir or not opts.destdir or not opts.sourceuser or not opts.destuser: print "Option add requires option time, backup-type, source-host, dest-host, source-dir, dest-dir, source-user, dest-user" parser.print_help() exit(-1) # Weird use cases if opts.add and opts.remove: parser.print_help() exit(-1) # Start program scheduler = schedule() if opts.logdir: scheduler.logdir = opts.logdir if opts.show: # Displays pretty output of schedule, queue, and running tables scheduler = schedule() print scheduler elif opts.add: # Adds a schedule to the schedule table scheduler.newSchedule(time, backuptype, sourcehost, desthost, sourcedir, destdir, sourceuser, destuser) elif opts.remove: # Removes a single schedule from the schedules, removes all instances from queue and running scheduler.removeSchedule(scheduleid) elif opts.removerun: # Removes a single schedule from the queue scheduler.removeRunning(scheduleid) elif opts.removequeue: # Removes a single schedule from the queue scheduler.removeQueue(scheduleid) elif opts.removequeue: # Removes a single schedule from the queue scheduler.removeQueue(scheduleid) elif opts.expire: # Expires day in a schedule scheduler.expireSchedule(scheduleid) elif opts.addqueue: # Adds a single schedule to queue scheduler.queueSchedule(scheduleid) elif opts.queue: # Searches and add all schedules not run today to queue, then moves them to running scheduler.queueSchedules() scheduler.startBackup() if __name__ == '__main__': main()