|
|
| (3 intermediate revisions by the same user not shown) |
| Line 1: |
Line 1: |
| − | {{Admon/important| Warning! | This script has barely been tested as of this posting. May be riddled with bugs or may not function as expected.}}
| |
| | | | |
| − | = Pidora Update Source =
| |
| − |
| |
| − | * This script was kinda rushed, probably need to go back over it and fix some things
| |
| − |
| |
| − | ** Complete testing of all functions in this script has not been completed
| |
| − |
| |
| − | * To show/track which version is posted here
| |
| − |
| |
| − | <pre>
| |
| − | commit 7e5e19709b8ffb3f06ea7287da5a058445123fd3
| |
| − | Author: Andrew Oatley-Willis <andrew.oatley-willis@senecacollege.ca>
| |
| − | Date: Fri Jun 28 11:46:44 2013 -0400
| |
| − |
| |
| − | Bugs fixed
| |
| − | - Stop rsync if packages are unsigned
| |
| − |
| |
| − | Added descriptions
| |
| − | - Added mash, sign, and rsync descriptions
| |
| − | - Output to show when a process has started and ended
| |
| − | </pre>
| |
| − |
| |
| − | * Source code of pidora-update.py
| |
| − |
| |
| − | <pre>
| |
| − | #!/usr/bin/env python
| |
| − | # Andrew Oatley-Willis
| |
| − | # Multi-purpose tool written in python for linux
| |
| − | # This script will allow for a single safeguarded process to run: sigul, mash, and rsync to pidora.ca
| |
| − | # It will do a sanity check on everything that is happening and prevent manual errors that could occur
| |
| − | # Every configuration is customizable on the command line with well named options
| |
| − | import optparse
| |
| − | import pysftp
| |
| − | import sys
| |
| − | import urllib2
| |
| − | import getpass
| |
| − | import crypt
| |
| − | import random
| |
| − | import re
| |
| − | import string
| |
| − | import subprocess
| |
| − | import os
| |
| − |
| |
| − | class tools:
| |
| − | def __init__(self):
| |
| − | # Default configuration values
| |
| − | self.sigulhost = "england.proximity.on.ca"
| |
| − | self.mashhost = "japan.proximity.on.ca"
| |
| − | self.rsynchost = "pidora.proximity.on.ca"
| |
| − | self.siguluser = "agreene"
| |
| − | self.mashuser = "root"
| |
| − | self.rsyncuser = "pidorapr"
| |
| − | self.mashdir = "/usr/local/bin/mash-pidora"
| |
| − | self.kojitags = ['f18-updates', 'f18-rpfr-updates', 'f18-updates-testing', 'f18-rpfr-updates-testing']
| |
| − | self.email = "andrew.oatley-willis@senecacollege.ca"
| |
| − | self.auto = False
| |
| − | self.logdir = "/var/log/pidora-smr/"
| |
| − | self.logfile = "output"
| |
| − |
| |
| − | # Create command line options
| |
| − | parser = optparse.OptionParser()
| |
| − | parser = optparse.OptionParser(usage='Usage: %prog [options]')
| |
| − | parser.add_option('-i', '--info', help='check machine status and configuration', dest='status', default=False, action='store_true')
| |
| − | parser.add_option('-a', '--all', help='sign, mash, rsync', dest='everything', default=False, action='store_true')
| |
| − | parser.add_option('-s', '--sign', help='sign all packages in listed tag', dest='sign', default=False, action='store_true')
| |
| − | parser.add_option('-m', '--mash', help='start a mash run', dest='mash', default=False, action='store_true')
| |
| − | parser.add_option('-r', '--rsync', help='perform a rsync of the mash repos', dest='rsync', default=False, action='store_true')
| |
| − | parser.add_option('-l', '--list-unsigned', help='list unsigned rpms', dest='listunsigned', default=False, action='store_true')
| |
| − | parser.add_option('--auto', help='enables logging and emails logs', dest='auto', default=self.auto, action='store_true')
| |
| − | parser.add_option('--koji-tag', help='specify the koji tag to sign', dest='kojitag', default=False, action='store')
| |
| − | parser.add_option('--email', help='specify the email to send logs to', dest='email', default=False, action='store', metavar=self.email)
| |
| − | parser.add_option('--sigul-user', help='specify the user for sigul', dest='siguluser', default=self.siguluser, action='store', metavar=self.siguluser)
| |
| − | parser.add_option('--sigul-host', help='specify the host for sigul', dest='sigulhost', default=self.sigulhost, action='store', metavar=self.sigulhost)
| |
| − | parser.add_option('--mash-user', help='specify the user for mash', dest='mashuser', default=self.mashuser, action='store', metavar=self.mashuser)
| |
| − | parser.add_option('--mash-host', help='specify the host for mash', dest='mashhost', default=self.mashhost, action='store', metavar=self.mashhost)
| |
| − | parser.add_option('--rsync-user', help='specify the user for rsync', dest='rsyncuser', default=self.rsyncuser, action='store', metavar=self.rsyncuser)
| |
| − | parser.add_option('--rsync-host', help='specify the host for rsync', dest='rsynchost', default=self.rsynchost, action='store', metavar=self.rsynchost)
| |
| − | parser.add_option('--log-dir', help='specify a logging directory', dest='logdir', default=self.logdir, action='store', metavar=self.logdir)
| |
| − | parser.add_option('--log-file', help='specify a log file name', dest='logfile', default=self.logfile, action='store', metavar=self.logfile)
| |
| − | (opts, args) = parser.parse_args()
| |
| − |
| |
| − | # Check number of arguments and check for option switches
| |
| − | if len(sys.argv[1:]) == 0:
| |
| − | parser.print_help()
| |
| − | exit(-1)
| |
| − | if opts.kojitag:
| |
| − | self.kojitags = [opts.kojitag]
| |
| − | if opts.sigulhost:
| |
| − | self.sigulhost = opts.sigulhost
| |
| − | if opts.mashhost:
| |
| − | self.mashhost = opts.mashhost
| |
| − | if opts.rsynchost:
| |
| − | self.rsynchost = opts.rsynchost
| |
| − | if opts.siguluser:
| |
| − | self.siguluser = opts.siguluser
| |
| − | if opts.mashuser:
| |
| − | self.mashuser = opts.mashuser
| |
| − | if opts.rsyncuser:
| |
| − | self.rsyncuser = opts.rsyncuser
| |
| − | if opts.email:
| |
| − | self.email = opts.email
| |
| − | if opts.auto:
| |
| − | self.auto = opts.auto
| |
| − | if opts.logdir:
| |
| − | self.logdir = opts.logdir
| |
| − | if opts.logfile:
| |
| − | self.logfile = self.logdir + opts.logfile
| |
| − |
| |
| − | # Check for a few strange situations with options
| |
| − | self.signmash = False
| |
| − | self.signrsync = False
| |
| − | self.mashrsync = False
| |
| − | if opts.sign and opts.mash and opts.rsync:
| |
| − | opts.sign = False
| |
| − | opts.mash = False
| |
| − | opts.rsync = False
| |
| − | opts.everything = True
| |
| − | elif opts.sign and opts.mash:
| |
| − | opts.sign = False
| |
| − | opts.mash = False
| |
| − | self.signmash = True
| |
| − | elif opts.sign and opts.rsync:
| |
| − | opts.sign = False
| |
| − | opts.rsync = False
| |
| − | self.signrsync = True
| |
| − | elif opts.mash and opts.rsync:
| |
| − | opts.mash = False
| |
| − | opts.rsync = False
| |
| − | self.mashrsync = True
| |
| − |
| |
| − | # Create lists of successful and failed hosts
| |
| − | mhosts, mfail = self.get_status(self.mashhost, self.mashuser)
| |
| − | shosts, sfail = self.get_status(self.sigulhost, self.siguluser)
| |
| − | rhosts, rfail = self.get_status(self.rsynchost, self.rsyncuser)
| |
| − | self.hosts = mhosts + shosts + rhosts
| |
| − | self.fhosts = mfail + sfail + rfail
| |
| − |
| |
| − | # Start the main tasks
| |
| − | if opts.status:
| |
| − | print self.info()
| |
| − | elif self.sigulhost not in self.hosts: # Check connection with sigul host
| |
| − | self.email_exit('[Error]\nCannot connect to sigul: failed hosts: \n' + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − | elif opts.listunsigned:
| |
| − | print 'Unsigned packages: ', self.kojitags
| |
| − | self.checksign()
| |
| − | exit(0)
| |
| − | elif not opts.sign and not opts.mash and not opts.rsync and not opts.everything:
| |
| − | parser.print_help()
| |
| − | exit(-1)
| |
| − | elif opts.sign:
| |
| − | self.sign()
| |
| − | self.email_exit('[Success]\nSign for pidora complete', subject='pidora-smr - success')
| |
| − | elif self.mashhost not in self.hosts: # Check connection with mash host
| |
| − | self.email_exit('[Error]\nCannot connect to mash: failed hosts: \n' + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − | elif self.checksign():
| |
| − | print 'Unsigned packages: ', self.kojitags
| |
| − | self.checksign()
| |
| − | print 'Cannot mash or rsync if packages are not signed'
| |
| − | exit(0)
| |
| − | elif opts.mash:
| |
| − | self.mash()
| |
| − | self.email_exit('[Success]\nMash for pidora complete', subject='pidora-smr - success')
| |
| − | elif self.rsynchost not in self.hosts: # Check connection with rsync host
| |
| − | self.email_exit('[Error]\nCannot connect to rsync: failed hosts: \n' + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − | elif opts.rsync:
| |
| − | self.rsync()
| |
| − | self.email_exit('[Success]\nRsync for pidora complete', subject='pidora-smr - success')
| |
| − | elif opts.everything:
| |
| − | self.sign()
| |
| − | self.mash()
| |
| − | self.rsync()
| |
| − | self.email_exit('[Success]\nSign, mash, rsync for pidora complete', subject='pidora-smr - success')
| |
| − |
| |
| − | # Email text and subject, written a little bit crazy...
| |
| − | def sendemail(self, subject, text):
| |
| − | arg = '-s "' + subject + '" "' + self.email + '"'
| |
| − | output = subprocess.check_output(['echo "' + str(text) + '" |mail ' + str(arg)], shell=True)
| |
| − |
| |
| − | def logging(self, logme):
| |
| − | try:
| |
| − | os.mkdirs(directory)
| |
| − | except OSError: pass
| |
| − |
| |
| − | # Display all configuration data + hosts status
| |
| − | def info(self, infotype='all'):
| |
| − | if infotype == 'all':
| |
| − | info = ['\n[Connection]\nsigulhost = ' + self.sigulhost,
| |
| − | 'siguluser = ' + self.siguluser,
| |
| − | 'mashhost = ' + self.mashhost,
| |
| − | 'mashuser = ' + self.mashuser,
| |
| − | 'rsynchost = ' + self.rsynchost,
| |
| − | 'rsyncuser = ' + self.rsyncuser,
| |
| − | '\n[General]\nauto = ' + str(self.auto),
| |
| − | 'mashdir = ' + self.mashdir,
| |
| − | 'kojitags = ' + str(self.kojitags),
| |
| − | 'email = ' + self.email,
| |
| − | '\nlogdir = ' + self.logdir,
| |
| − | 'logfile = ' + self.logfile,
| |
| − | '\n[Hosts]\nworking hosts: ' + str(self.hosts),
| |
| − | 'failed hosts: ' + str(self.fhosts) + '\n']
| |
| − | elif infotype == 'sign':
| |
| − | info = ['\n[Connection]\nsigulhost = ' + self.sigulhost,
| |
| − | 'siguluser = ' + self.siguluser,
| |
| − | '\n[General]\nauto = ' + str(self.auto),
| |
| − | 'kojitags = ' + str(self.kojitags),
| |
| − | 'logdir = ' + self.logdir,
| |
| − | 'logfile = ' + self.logfile,
| |
| − | '\n[Hosts]\nworking hosts: ' + str(self.hosts),
| |
| − | 'failed hosts: ' + str(self.fhosts) + '\n']
| |
| − | elif infotype == 'mash':
| |
| − | info = ['\n[Connection]\nmashhost = ' + self.mashhost,
| |
| − | 'mashuser = ' + self.mashuser,
| |
| − | '\n[General]\nauto = ' + str(self.auto),
| |
| − | 'mashdir = ' + self.mashdir,
| |
| − | 'kojitags = ' + str(self.kojitags),
| |
| − | '\nlogdir = ' + self.logdir,
| |
| − | 'logfile = ' + self.logfile,
| |
| − | '\n[Hosts]\nworking hosts: ' + str(self.hosts),
| |
| − | 'failed hosts: ' + str(self.fhosts) + '\n']
| |
| − | elif infotype == 'rsync':
| |
| − | info = ['\n[Connection]\nrsynchost = ' + self.rsynchost,
| |
| − | 'rsyncuser = ' + self.rsyncuser,
| |
| − | 'mashdir = ' + self.mashdir,
| |
| − | '\n[General]\nauto = ' + str(self.auto),
| |
| − | 'kojitags = ' + str(self.kojitags),
| |
| − | '\nlogdir = ' + self.logdir,
| |
| − | 'logfile = ' + self.logfile,
| |
| − | '\n[Hosts]\nworking hosts: ' + str(self.hosts),
| |
| − | 'failed hosts: ' + str(self.fhosts) + '\n']
| |
| − | return '\n'.join(info)
| |
| − |
| |
| − | # Display text and exit or send an email and exit
| |
| − | def email_exit(self, text, subject=False, errors=0):
| |
| − | if self.auto and subject:
| |
| − | self.sendemail(subject, text)
| |
| − | exit(errors)
| |
| − | else:
| |
| − | print text
| |
| − | exit(errors)
| |
| − |
| |
| − | # Rsync to the repo directory
| |
| − | def rsync(self):
| |
| − | print '\n== Start: Rsync ==\n'
| |
| − | self.checkmash()
| |
| − | srv = pysftp.Connection(host=self.rsynchost, username=self.rsyncuser, log=True)
| |
| − | output = srv.execute('/home/pidorapr/bin/rsync-japan; echo $? > /home/pidorapr/.rsync-japan-exit-status')
| |
| − | for line in output:
| |
| − | print line.strip()
| |
| − | output = srv.execute('cat /home/pidorapr/.rsync-japan-exit-status')
| |
| − | srv.close()
| |
| − | if str(output.strip()) != '0':
| |
| − | self.email_exit('[Error]\nRsync failed stopping program\nExit status = ' + str(output.strip()) + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − |
| |
| − |
| |
| − |
| |
| − | # Check if hosts are online and can establish connection, return lists of failed and succesful hosts
| |
| − | def get_status(self, host, username):
| |
| − | hostname = []
| |
| − | fhost = []
| |
| − | check = self.connect(host, username)
| |
| − | if check:
| |
| − | hostname.append(host)
| |
| − | else:
| |
| − | fhost.append(host)
| |
| − | return (hostname, fhost)
| |
| − |
| |
| − | # Connect to the hosts, return True or False
| |
| − | def connect(self, host, username):
| |
| − | try:
| |
| − | response=urllib2.urlopen('http://'+host,timeout=1)
| |
| − | srv = pysftp.Connection(host=host, username=username, log=True)
| |
| − | srv.close()
| |
| − | return True
| |
| − | except urllib2.URLError as err:pass
| |
| − | except:pass
| |
| − | return False
| |
| − |
| |
| − | # Start a signing run across a designated tag
| |
| − | def sign(self):
| |
| − | print '\n== Start: Sign run ==\n'
| |
| − | print 'Koji tags marked for signing:'
| |
| − | for tag in self.kojitags:
| |
| − | print tag.strip()
| |
| − | print '\nEnter sigul key passphrase:'
| |
| − | password = getpass.getpass()
| |
| − | for tag in self.kojitags:
| |
| − | print "Signing packages in tag: " + tag
| |
| − | print "Packages found: "
| |
| − | print self.checksign()
| |
| − | tempfile1 = crypt.crypt(str(random.random()), "pidora" ) + '.log'
| |
| − | tempfile = tempfile1.replace("/", "")
| |
| − | tempdir = '~/.pidora/'
| |
| − | srv = pysftp.Connection(host=self.sigulhost, username=self.siguluser, log=True)
| |
| − | errors = srv.execute('mkdir ' + tempdir + ' 2>/dev/null')
| |
| − | errors = srv.execute('touch ' + tempdir + tempfile + '2>/dev/null')
| |
| − | output = srv.execute('~/.sigul/sigulsign_unsigned.py -v --password=' + password + ' --write-all --tag=' + tag + " pidora-18 2>" + tempdir + tempfile)
| |
| − | errors = srv.execute('cat ' + tempdir + tempfile)
| |
| − | srv.close()
| |
| − | # Scan through output and find errors! If errors are found, stop program and spit out error warnings
| |
| − | outputs = output + errors
| |
| − | errors = []
| |
| − | for output in outputs:
| |
| − | print output.strip()
| |
| − | if re.search('^ERROR:.*$', output):
| |
| − | errors.append(output)
| |
| − | if errors:
| |
| − | self.email_exit('[Error]\nError signing stopping program\n' + str(errors) + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − |
| |
| − | # Check koji for unsigned packages, returns True if unsigned rpms are found
| |
| − | def checksign(self):
| |
| − | check = False
| |
| − | for tag in self.kojitags:
| |
| − | srv = pysftp.Connection(host=self.sigulhost, username=self.siguluser, log=True)
| |
| − | output = srv.execute("~/.sigul/sigulsign_unsigned.py --just-list --tag=" + tag + " pidora-18")
| |
| − | srv.close()
| |
| − | for rpm in output:
| |
| − | print rpm.strip()
| |
| − | if rpm.strip() != "":
| |
| − | check = "unsigned rpms found"
| |
| − | if check:
| |
| − | return True
| |
| − |
| |
| − | # Run mash and search through the log file for failed mash errors
| |
| − | def mash(self):
| |
| − | print '\n== Start: Mash run ==\n'
| |
| − | srv = pysftp.Connection(host=self.mashhost, username=self.mashuser, log=True)
| |
| − | output = srv.execute('/usr/local/bin/mashrun-pidora-18')
| |
| − | srv.close()
| |
| − | self.checkmash()
| |
| − |
| |
| − | def checkmash(self):
| |
| − | errors = []
| |
| − | srv = pysftp.Connection(host=self.mashhost, username=self.mashuser, log=True)
| |
| − | output = srv.execute('cat /mnt/koji/mash/pidora-mash-latest/mash.log')
| |
| − | srv.close()
| |
| − | for line in output:
| |
| − | if re.search('^mash failed .*$', line):
| |
| − | errors.append(line.strip())
| |
| − | if errors:
| |
| − | self.email_exit('[Error]\nmash failed on repo stopping program\n' + str(errors) + self.info(), subject='pidora-smr - failed', errors=1)
| |
| − |
| |
| − |
| |
| − |
| |
| − | if __name__ == '__main__':
| |
| − | tools()
| |
| − |
| |
| − | </pre>
| |
