Run tests.py

#!c:/Python24/python.exe
#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is standalone Firefox Windows performance test.
#
# The Initial Developer of the Original Code is Google Inc.
# Portions created by the Initial Developer are Copyright (C) 2006
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Elizabeth Chak <elichak@gmail.com> (Seneca@York)
#   Annie Sullivan <annie.sullivan@gmail.com> (original developer of the 
#                                              Performance Testing framework)
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****



"""This file runs extension performance tests.

   It runs Ts (startup time) and Tp (page load time) tests
   for an extension with different profiles.
   
   Before this file is written, there were many problems with 
   configuring the framework. Users were thrown with errors that
   took a long time to debug and solve. This file faciliates a
   user in configuring a framework with minimal frustration and 
   time. It tries to eliminate as many user errors as possible, errors
   that users are prone to making while configuring the framework.
   Informative messages are shown to the user whenever they make
   an accidental error while configuring the framework. 
   
   Comments:
   At first the program wouldn't exit on sys.exit()
   Rob Campbell suggested that I look into Ben Smedberg's killableprocess.py.
   It doesn't work with this because it's using Python 2.3
   Haven't figured out how to work around the exceptions about ctypes library
   when I try to use killableprocess.py
"""

__author__ = 'elichak@gmail.com (Liz Chak)'


import syck
import sys
import re
import pb
import report
import paths
import tp
import ts
import os
#import killableprocess #Python 2.3

# Number of times to run startup test (Ts)
TS_NUM_RUNS = 5

# Number of times the page load test (Tp) loads each page in the test.
TP_NUM_CYCLES = 7

# Resolution of counter sample data for page load test (Tp), in seconds
# (For example, if TP_RESOLUTION=1, sample counters every 1 second
TP_RESOLUTION = 1

#killProcess = killableprocess.Popen


####
# Since python doesn't have 'switch' statements, this class
# aims to duplicate C's original switch functionality and
# structure with reasonable accuracy
####
class switch(object):
    def __init__(self, value):
        self.value = value
        self.fall = False

    def __iter__(self):
        #Return the match method once, then stop
        yield self.match
   
    def match(self, *args):
        #Indicate whether or not to enter a case suite
        if self.fall or not args:
            return True
        elif self.value in args: 
            self.fall = True
            return True
        else:
            return False

####
# This is a helper module that handles any exception
####
     
def ExpHandler(*posargs):

    def nestedhandler(func,exptuple, *pargs, **kwargs):
        #Function that creates a nested exception handler from the passed exception tuple

        exp, handler = exptuple[0]
        try:
            if len(exptuple)==1:
                func(*pargs, **kwargs)
            else:
                nestedhandler(func,exptuple[1:], *pargs, **kwargs)
        except exp, e:
            if handler:
                handler(e)
            else:
                print e.__class__.__name__,':',e
                sys.stdout.flush()
    def wrapper(f):
        t = tuple(item for item in posargs[0]
            if issubclass(item,Exception)) or (Exception,)
        def newfunc(*pargs, **kwargs):
            try: 
                f(*pargs, **kwargs)
            except t, e:
                # Only inform the user that the exception occured
                print e.__class__.__name__,':',e
                sys.stdout.flush()
        return newfunc
    return wrapper

####
#  Runs the Ts and Tp tests on the given config file and generates a report.
#  The file is validated to ensure that the config file is valid. However, it
#  doesn't check if all the profile items are valid. Only partial validation was
#  done. The firefox profile is checked to ensure that it is a valid location
#  of the browser that is used for testing.
#
#  To change the preferences that are set on the profiles that are 
#  tested (config.yaml), you may edit the arrays in the main function below.
#  Args:
#    @type     string
#    @param    filename: the name of the config file where the profile of the 
#              test is stored
#    
#    @type     boolean
#    @return   returns whether the test was successful or not 
#
#    @type     dict
#    @param    yaml[item]: the list of profile item values from the yaml file
#
#    @type     list of yaml[item] dict values
#    @param    test_configs
#
#    @type     list
#    @param    test_names; profile names; in our case, title, filename and other 
#              test profile items: firefox, extensions   
#
####
#@ExpHandler((KeyError, Exception)) #wouldn't return value from module if this is not commented out
def test_file(filename):
  
  test_configs = []
  test_names = []
  title = ''
  filename_prefix = ''
  testDone = False
  
  """Reads in the profile info from the YAML config file
     Validates the profile items and terminates if the YAML
     config file doesn't valildate. If there isn't a given
     YAML filename, the program will terminate.
  """
  
  try:
	  if not filename:
	      error = "Error... " + filename + " doesn't exist, please create it in the same directory!" 
	      sys.exit(error)  #why doesn't this exit the program?       
	  else:    
	      print "checking... reading %s" %filename
	      sys.stdout.flush()
	      if not os.path.exists(filename):
		 error = "Error... " + filename + " doesn't exist, please create it in the same directory!" 
		 sys.exit(error)  #why doesn't this exit the program?
	      else:
		config_file = open(filename, 'r')
		yaml = syck.load(config_file)
		config_file.close()

		print "checking... Values in %s" %filename
		sys.stdout.flush()
		for item in yaml:

			c = item
			for case in switch(c):
				print "checking... Value of item in %s: %s" %(filename, item)
				if case('filename'):
					filename_prefix = yaml[item]
					print "checking... filename: %s" %yaml[item]
					break
				if case('title'): # note the * for unpacking as arguments
					title = yaml[item]
					print "checking... title: %s" %yaml[item]
					break
				if case(): # normal argument passing style also applies
					print "checking... Values of preferences, extensions, firefox"
					new_config = [yaml[item]['preferences'],
						      yaml[item]['extensions'],
						      yaml[item]['firefox']]
					print "checking... preferences: %s" %yaml[item]['preferences']
					print "checking... extensions: %s" %yaml[item]['extensions']
					print "checking... firefox: %s" %yaml[item]['firefox']
					if not os.path.exists(yaml[item]['firefox']):
						error = "\nError... Please check that the firefox path in "+filename+" points to the desired instance: " + yaml[item]['firefox']
						sys.exit(error)  #why doesn't this exit the program?
					test_configs.append(new_config)
					test_names.append(item)
					break
		print "processing... Performance Testing is starting"
  except IOError:
                return 0
	        IO_error = "\nError... Please check that your file is valid:" + filename
	  	sys.exit(IO_error) #why doesn't this exit the program?
  config_file.close()

  from time import sleep
  
  # Instantiating a progress bar object to show the progress each time a profile is used for testing
  # With that said, the progress bar will show again if the user runs more than 1 config file
  progbar = pb.progressbarClass(4) #4 stages
  
  # Run startup time test
  progbar.progress(1) #progress bar stage 1 completed
  ts_times = ts.RunStartupTests(paths.BASE_PROFILE_DIR,
				test_configs,
				TS_NUM_RUNS)
  progbar.progress(2) #progress bar stage 2 completed
  
  # Run page load test.  For possible values of counters argument, see
  # http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
  (tp_times, tp_counters) = tp.RunPltTests(paths.BASE_PROFILE_DIR,
					   test_configs,
					   TP_NUM_CYCLES,
					   ['Private Bytes', 'Working Set', '% Processor Time'],
					   TP_RESOLUTION)

  #Generate a report of the merits of the Performance Testing
  #YAML profile items that are passed into GenerateReport 
  #are: title, filename and other test items: firefox, extensions 
  #and preferences
  progbar.progress(3) #progress bar stage 3 completed
  report.GenerateReport(title,
			filename_prefix,
			test_names,
			ts_times,
			tp_times,
			tp_counters,
			TP_RESOLUTION)
			
  progbar.progress(4) #progress bar stage 4 completed

  return 1
  
####
#  
# Checks the paths in paths.py to see if they are pointing to the right location.
# If it doesn't, the user will be shown informative messages to fix the problem.
# In the case of a missing directory, the user will be prompted if he/she would like
# the directory to be created. It also checks if the base profile directory have
# any contents or else the program will throw a nasty ZeroDivision Error. File paths
# that uses the file url has to be parsed before being checked.
# 
# Args:
#    @type     list
#    @param    path_dirs: a list of the dir path constant values in paths.py 
# 
#    @type     list
#    @param    test_files: a list of the file path constant values in paths.py (file urls)
#
####

#@ExpHandler((IOError, Exception)) #wouldn't return value from module if this is not commented out
def checkPaths():
  # Checks if the following paths exist (initialized in paths.py)
  # The paths in the paths.py file may be modifiled to direct to 
  # the desired locations.
  
  validates = False
  path_dirs = [paths.BASE_PROFILE_DIR, paths.REPORTS_DIR] 
  test_files =[paths.INIT_URL, paths.TP_URL, paths.TS_URL]
  
  # Checks if the paths in paths.py exist. If it doesn't exist, the user will
  # be warned and it will ask the user if he/she wants the missing directory
  # to be created
   
  for dir in path_dirs:
    #head, tail = os.path.split(dir)
    head = dir
    if not os.path.exists(head): 
	print "Warning... Dir doesn't exist: %s" % head
	sys.stdout.flush()
    	resp = raw_input("Do you want the directory to be created (y/n)? ")
    	
    	while resp not in ("y", "Y", "n", "N"):
    	    resp = raw_input("Do you want the directory to be created (y/n)? ")
    	    sys.stdout.flush()
    	if resp in ("y", "Y"): 
    	    print "creating dir..."
    	    dirname = head
    	    os.mkdir(dirname)
    	    print "Dir %s is created" % dirname
    	    break
    	elif resp in ("n", "N"):
    	   #killProcess.kill()
    	   error = "\nError... Please check that" + head + " is created or exist or change path in paths.py to correct path" 
    	   sys.exit(error) #why doesn't this exit the program?
    	   break
	        
    elif os.path.exists(head):
	print "checking... Exist: Dir %s" % head
	sys.stdout.flush()

  # Checks if contents of base_profile_dir exist. Initially, if contents of the base_profile_dir doesn't exist,
  # it will throw a ZeroDivision error. Since this module's exception is captured by the ExpHandler module,
  # it throws a more appopriate error message. However, it should terminate the program, but it doesn't for
  # some reason. sys.exit doesn't seem to work!!!
  validatePaths = True
  if os.path.isdir(paths.BASE_PROFILE_DIR):
	for root, dirs, files in os.walk(paths.BASE_PROFILE_DIR):
	    print root, "consumes",
	    print sum(os.path.getsize(os.path.join(root, name)) for name in files),
	    print "bytes in", len(files), "non-directory files"
            if 'CVS' in dirs:
                 dirs.remove('CVS')  # don't visit CVS directories 
        if sum(os.path.getsize(os.path.join(root, name)) for name in files) == 0:
            #killProcess.kill()
            validatePaths = False
            error = "\nError... Please check that you have files in " + paths.BASE_PROFILE_DIR
            sys.exit(error)  #why doesn't this exit the program?
            
  elif not os.path.isdir(paths.BASE_PROFILE_DIR):
       #killProcess.kill()
       validatePaths = False
       error = "\nError... Please check that " + paths.BASE_PROFILE_DIR + " exist"
       sys.exit(error) # it doesn't exit the program, why?
  sys.stdout.flush()
  
  #validates the file urls
  validateURLS = True
  for f in test_files:
        #Converts file urls to file paths to check if the paths initialized 
        #in paths.py are valid. (INIT_URL, TS_URL, TP_URL)
	file_url = f
	file_path = file_url[8:]   
	file_path = file_path.replace('/', '\\') 
	file_path = re.split("[?]", file_path)
  	
	if not os.path.exists(file_path[0]):
		error = "\nError... Please check if the path in paths.py for " + file_path[0] + " is right"
		validateURLS = False
	else:
		print "checking... Exist: %s" %file_path[0]
		sys.stdout.flush()
  if not validateURLS:
        sys.exit(error) # it doesn't exit the program, why?
        
  return validateURLS and validatePaths

####
#
# Main function:
#    It will validate the required files, directories and configuration before performing the 
#    Firefox Performance Testing. If the required configuration files/dirs don't validate, users
#    cannot start Firefox Performance Testing until the error is fixed.
#    User may list as many space-seperated config file arguments as possible when they run this file 
#    Tells the user when the performance testing is starting, completed or failed/halted.
#
####
if __name__=='__main__':  	
  testComplete = False
  filesValidate = False
  
  # Check if paths in paths.py exist
  filesValidate = checkPaths()

  # Read in each config file and run the tests on it.
  if filesValidate:
      for i in range(1, len(sys.argv)):
          testComplete = test_file(sys.argv[i])
  elif not filesValidate:
      print "Files didn't validate!"
  if testComplete:  
      print "Performance testing is done... check your results in %s" %paths.REPORTS_DIR
  elif not testComplete:
      print "Performance testing failed"