Run tests.py

#!c:/Python24/python.exe
#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is standalone Firefox Windows performance test.
#
# The Initial Developer of the Original Code is Google Inc.
# Portions created by the Initial Developer are Copyright (C) 2006
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Elizabeth Chak <elichak@gmail.com> (Seneca@York)
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

"""Runs extension performance tests.
   This file runs Ts (startup time) and Tp (page load time) tests
   for an extension with different profiles.    
"""

__author__ = 'elichak@gmail.com (Liz Chak)'


import time
import syck
import sys
import re
import pb
import report
import paths
import tp
import ts
import os
from os.path import join, getsize
import stat
import string
import urlparse 
#import killableprocess #Python 2.3

# Number of times to run startup test (Ts)
TS_NUM_RUNS = 5

# Number of times the page load test (Tp) loads each page in the test.
TP_NUM_CYCLES = 7

# Resolution of counter sample data for page load test (Tp), in seconds
# (For example, if TP_RESOLUTION=1, sample counters every 1 second
TP_RESOLUTION = 1

#killProcess = killableprocess.Popen


"""
 Since python doesn't have 'switch' statements, this class
 aims to duplicate C's original switch functionality and
 structure with reasonable accuracy
"""
class switch(object):
    def __init__(self, value):
        self.value = value
        self.fall = False

    def __iter__(self):
        #Return the match method once, then stop
        yield self.match
        raise StopIteration
    
    def match(self, *args):
        #Indicate whether or not to enter a case suite
        if self.fall or not args:
            return True
        elif self.value in args: 
            self.fall = True
            return True
        else:
            return False

"""
 This handles any exception
"""
     
def ExpHandler(*posargs):

    def nestedhandler(func,exptuple, *pargs, **kwargs):
        #Function that creates a nested exception handler from the passed exception tuple

        exp, handler = exptuple[0]
        try:
            if len(exptuple)==1:
                func(*pargs, **kwargs)
            else:
                nestedhandler(func,exptuple[1:], *pargs, **kwargs)
        except exp, e:
            if handler:
                handler(e)
            else:
                print e.__class__.__name__,':',e
                sys.stdout.flush()
    def wrapper(f):
        t = tuple(item for item in posargs[0]
            if issubclass(item,Exception)) or (Exception,)
        def newfunc(*pargs, **kwargs):
            try: 
                f(*pargs, **kwargs)
            except t, e:
                # Only inform the user that the exception occured
                print e.__class__.__name__,':',e
                sys.stdout.flush()
        return newfunc
    return wrapper

#@ExpHandler((KeyError, Exception))
def test_file(filename):
  """Runs the Ts and Tp tests on the given config file and generates a report.
     To change the preferences that are set on the profiles that are 
     tested (config.yaml), edit the arrays in the main function below.
  Args:
    @type     string
    @param    filename: the name of the file to run the tests on
    
    @type     boolean
    @return   returns whether the test was successful or not 
  """
  
  test_configs = []
  test_names = []
  title = ''
  filename_prefix = ''
  testDone = False
  
  """Reads in the profile info from the YAML config file
     Validates the profile items and terminates if the YAML
     config file doesn't valildate. If there isn't a given
     YAML filename, the program will terminate.
  """
  
  try:
	  if not filename:
	      error = "Error... " + filename + " doesn't exist, please create it in the same directory!" 
	      sys.exit(error)  #why doesn't this exit the program?       
	  else:    
	      print "checking... reading %s" %filename
	      sys.stdout.flush()
	      if not os.path.exists(filename):
		 error = "Error... " + filename + " doesn't exist, please create it in the same directory!" 
		 sys.exit(error)  #why doesn't this exit the program?
	      else:
		config_file = open(filename, 'r')
		yaml = syck.load(config_file)
		config_file.close()

		print "checking... Values in %s" %filename
		sys.stdout.flush()
		for item in yaml:

			c = item
			for case in switch(c):
				print "checking... Value of item in %s: %s" %(filename, item)
				if case('filename'):
					filename_prefix = yaml[item]
					print "checking... filename: %s" %yaml[item]
					break
				if case('title'): # note the * for unpacking as arguments
					title = yaml[item]
					print "checking... title: %s" %yaml[item]
					break
				if case(): # normal argument passing style also applies
					print "checking... Values of preferences, extensions, firefox"
					new_config = [yaml[item]['preferences'],
						      yaml[item]['extensions'],
						      yaml[item]['firefox']]
					print "checking... preferences: %s" %yaml[item]['preferences']
					print "checking... extensions: %s" %yaml[item]['extensions']
					print "checking... firefox: %s" %yaml[item]['firefox']
					if not os.path.exists(yaml[item]['firefox']):
						error = "\nError... Please check that the firefox path in "+filename+" points to the desired instance: " + yaml[item]['firefox']
						sys.exit(error)  #why doesn't this exit the program?
					test_configs.append(new_config)
					test_names.append(item)
					break
		print "processing... Performance Testing is starting"
  except IOError:
                return 0
	        IO_error = "\nError... Please check that your file is valid:" + filename
	  	sys.exit(IO_error) #why doesn't this exit the program?
  config_file.close()

  from time import sleep
      
  progbar = pb.progressbarClass(4) 
  
  # Run startup time test
  progbar.progress(1)
  ts_times = ts.RunStartupTests(paths.BASE_PROFILE_DIR,
				test_configs,
				TS_NUM_RUNS)
  progbar.progress(2)
  # Run page load test.  For possible values of counters argument, see
  # http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
  (tp_times, tp_counters) = tp.RunPltTests(paths.BASE_PROFILE_DIR,
					   test_configs,
					   TP_NUM_CYCLES,
					   ['Private Bytes', 'Working Set', '% Processor Time'],
					   TP_RESOLUTION)

  #Generate a report of the merits of the Performance Testing
  #YAML profile items that are passed into GenerateReport 
  #are: title, filename and other test items: firefox, extensions 
  #and preferences
  progbar.progress(3)
  report.GenerateReport(title,
			filename_prefix,
			test_names,
			ts_times,
			tp_times,
			tp_counters,
			TP_RESOLUTION)
  progbar.progress(4)

  return 1
  
#@ExpHandler((IOError, Exception))
def checkPaths():
  # Checks if the following paths exist (initialized in paths.py)
  # The paths in the paths.py file may be modifiled to direct to 
  # the desired locations.
  
  validates = False
  path_dirs = [paths.BASE_PROFILE_DIR, paths.REPORTS_DIR] 
  test_files =[paths.INIT_URL, paths.TP_URL, paths.TS_URL]
  
  # Checks if the paths in paths.py exist. If it doesn't exist, the user will
  # be warned and it will ask the user if he/she wants the missing directory
  # to be created
   
  for dir in path_dirs:
    #head, tail = os.path.split(dir)
    head = dir
    if not os.path.exists(head): 
	print "Warning... Dir doesn't exist: %s" % head
	sys.stdout.flush()
    	resp = raw_input("Do you want the directory to be created (y/n)? ")
    	
    	# For some reason, it complains when I use string.lowercase, therefore, I have to check 
    	# the user input in a more inefficient way. If the user chooses Y/y, a missing dir will
    	# be created. However, if the user chooses N/n, an error message is thrown to let the user
    	# know of the missing directory so that he/she can create it manually. Program should exit
    	# but it doesn't with sys.exit. Why???
    	while not (resp == "y") or not (resp == "Y") or not (resp == "n") or not (resp == "N"):
    	    if (resp == "y") or (resp == "Y"): 
    	        print "creating dir..."
    		dirname = head
    		os.mkdir(dirname)
    		print "Dir %s is created" % dirname
    		break
    	    elif (resp == "n") or (resp == "N"):	
                #killProcess.kill()
		error = "\nError... Please check that" + head + " is created or exist or change path in paths.py to correct path" 
		sys.exit(error) #why doesn't this exit the program?
		break
	    else:
    	        resp = raw_input("Do you want the directory to be created (y/n)? ")
	        sys.stdout.flush()
	        
    elif os.path.exists(head):
	print "checking... Exist: Dir %s" % head
	sys.stdout.flush()

  # Checks if contents of base_profile_dir exist. Initially, if contents of the base_profile_dir doesn't exist,
  # it will throw a ZeroDivision error. Since this module's exception is captured by the ExpHandler module,
  # it throws a more appopriate error message. However, it should terminate the program, but it doesn't for
  # some reason. sys.exit doesn't seem to work!!!
  validatePaths = True
  if os.path.isdir(paths.BASE_PROFILE_DIR):
	for root, dirs, files in os.walk(paths.BASE_PROFILE_DIR):
	    print root, "consumes",
	    print sum(getsize(join(root, name)) for name in files),
	    print "bytes in", len(files), "non-directory files"
            if 'CVS' in dirs:
                 dirs.remove('CVS')  # don't visit CVS directories 
        if sum(getsize(join(root, name)) for name in files) == 0:
            #killProcess.kill()
            validatePaths = False
            error = "\nError... Please check that you have files in " + paths.BASE_PROFILE_DIR
            sys.exit(error)  #why doesn't this exit the program?
            
  elif not os.path.isdir(paths.BASE_PROFILE_DIR):
       #killProcess.kill()
       validatePaths = False
       error = "\nError... Please check that " + paths.BASE_PROFILE_DIR + " exist"
       sys.exit(error) # it doesn't exit the program, why?
  sys.stdout.flush()
  #validates the file urls
  validateURLS = True
  for f in test_files:
        #Converts file urls to file paths to check if the paths initialized 
        #in paths.py are valid. (INIT_URL, TS_URL, TP_URL)
	file_url = f
	file_path = file_url[8:]   
	file_path = file_path.replace('/', '\\') 
	file_path = re.split("[?]", file_path)
  	
	if not os.path.exists(file_path[0]):
		error = "\nError... Please check if the path in paths.py for " + file_path[0] + " is right"
		validateURLS = False
	else:
		print "checking... Exist: %s" %file_path[0]
		sys.stdout.flush()
		validateURLS = True
  if not validateURLS:
        sys.exit(error) # it doesn't exit the program, why?
        
  if validateURLS and validatePaths:
        return 1
  else:
        return 0
  return 0
  
if __name__=='__main__':  	
  testComplete = False
  filesValidate = False
  
  # Check if paths in paths.py exist
  filesValidate = checkPaths()

  # Read in each config file and run the tests on it.
  if filesValidate:
      for i in range(1, len(sys.argv)):
          testComplete = test_file(sys.argv[i])
  elif not filesValidate:
      print "Files didn't validate!"
  if testComplete:  
      print "Performance testing is done... check your results in %s" %paths.REPORTS_DIR
  elif not testComplete:
      print "Performance testing failed"