#!c:/Python24/python.exe
#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is standalone Firefox Windows performance test.
#
# The Initial Developer of the Original Code is Google Inc.
# Portions created by the Initial Developer are Copyright (C) 2006
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Elizabeth Chak <elichak@gmail.com> (Seneca@York)
# Annie Sullivan <annie.sullivan@gmail.com> (original developer of the
# Performance Testing framework)
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****
"""This file runs extension performance tests.
It runs Ts (startup time) and Tp (page load time) tests
for an extension with different profiles.
Before this file is written, there were many problems with
configuring the framework. Users were thrown with errors that
took a long time to debug and solve. This file faciliates a
user in configuring a framework with minimal frustration and
time. It tries to eliminate as many user errors as possible, errors
that users are prone to making while configuring the framework.
Informative messages are shown to the user whenever they make
an accidental error while configuring the framework.
Comments:
At first the program wouldn't exit on sys.exit()
Rob Campbell suggested that I look into Ben Smedberg's killableprocess.py.
It doesn't work with this because it's using Python 2.3
Haven't figured out how to work around the exceptions about ctypes library
when I try to use killableprocess.py
"""
__author__ = 'elichak@gmail.com (Liz Chak)'
import syck
import sys
import re
import pb
import report
import paths
import tp
import ts
import os
#import killableprocess #Python 2.3
# Number of times to run startup test (Ts)
TS_NUM_RUNS = 5
# Number of times the page load test (Tp) loads each page in the test.
TP_NUM_CYCLES = 7
# Resolution of counter sample data for page load test (Tp), in seconds
# (For example, if TP_RESOLUTION=1, sample counters every 1 second
TP_RESOLUTION = 1
#killProcess = killableprocess.Popen
####
# Since python doesn't have 'switch' statements, this class
# aims to duplicate C's original switch functionality and
# structure with reasonable accuracy
####
class switch(object):
def __init__(self, value):
self.value = value
self.fall = False
def __iter__(self):
#Return the match method once, then stop
yield self.match
raise StopIteration
def match(self, *args):
#Indicate whether or not to enter a case suite
if self.fall or not args:
return True
elif self.value in args:
self.fall = True
return True
else:
return False
####
# This is a helper module that handles any exception
####
def ExpHandler(*posargs):
def nestedhandler(func,exptuple, *pargs, **kwargs):
#Function that creates a nested exception handler from the passed exception tuple
exp, handler = exptuple[0]
try:
if len(exptuple)==1:
func(*pargs, **kwargs)
else:
nestedhandler(func,exptuple[1:], *pargs, **kwargs)
except exp, e:
if handler:
handler(e)
else:
print e.__class__.__name__,':',e
sys.stdout.flush()
def wrapper(f):
t = tuple(item for item in posargs[0]
if issubclass(item,Exception)) or (Exception,)
def newfunc(*pargs, **kwargs):
try:
f(*pargs, **kwargs)
except t, e:
# Only inform the user that the exception occured
print e.__class__.__name__,':',e
sys.stdout.flush()
return newfunc
return wrapper
####
# Runs the Ts and Tp tests on the given config file and generates a report.
# The file is validated to ensure that the config file is valid. However, it
# doesn't check if all the profile items are valid. Only partial validation was
# done. The firefox profile is checked to ensure that it is a valid location
# of the browser that is used for testing.
#
# To change the preferences that are set on the profiles that are
# tested (config.yaml), you may edit the arrays in the main function below.
# Args:
# @type string
# @param filename: the name of the config file where the profile of the
# test is stored
#
# @type boolean
# @return returns whether the test was successful or not
#
# @type dict
# @param yaml[item]: the list of profile item values from the yaml file
#
# @type list of yaml[item] dict values
# @param test_configs
#
# @type list
# @param test_names; profile names; in our case, title, filename and other
# test profile items: firefox, extensions
#
####
#@ExpHandler((KeyError, Exception)) #wouldn't return value from module if this is not commented out
def test_file(filename):
test_configs = []
test_names = []
title = ''
filename_prefix = ''
testDone = False
"""Reads in the profile info from the YAML config file
Validates the profile items and terminates if the YAML
config file doesn't valildate. If there isn't a given
YAML filename, the program will terminate.
"""
try:
if not filename:
error = "Error... " + filename + " doesn't exist, please create it in the same directory!"
sys.exit(error) #why doesn't this exit the program?
else:
print "checking... reading %s" %filename
sys.stdout.flush()
if not os.path.exists(filename):
error = "Error... " + filename + " doesn't exist, please create it in the same directory!"
sys.exit(error) #why doesn't this exit the program?
else:
config_file = open(filename, 'r')
yaml = syck.load(config_file)
config_file.close()
print "checking... Values in %s" %filename
sys.stdout.flush()
for item in yaml:
c = item
for case in switch(c):
print "checking... Value of item in %s: %s" %(filename, item)
if case('filename'):
filename_prefix = yaml[item]
print "checking... filename: %s" %yaml[item]
break
if case('title'): # note the * for unpacking as arguments
title = yaml[item]
print "checking... title: %s" %yaml[item]
break
if case(): # normal argument passing style also applies
print "checking... Values of preferences, extensions, firefox"
new_config = [yaml[item]['preferences'],
yaml[item]['extensions'],
yaml[item]['firefox']]
print "checking... preferences: %s" %yaml[item]['preferences']
print "checking... extensions: %s" %yaml[item]['extensions']
print "checking... firefox: %s" %yaml[item]['firefox']
if not os.path.exists(yaml[item]['firefox']):
error = "\nError... Please check that the firefox path in "+filename+" points to the desired instance: " + yaml[item]['firefox']
sys.exit(error) #why doesn't this exit the program?
test_configs.append(new_config)
test_names.append(item)
break
print "processing... Performance Testing is starting"
except IOError:
return 0
IO_error = "\nError... Please check that your file is valid:" + filename
sys.exit(IO_error) #why doesn't this exit the program?
config_file.close()
from time import sleep
# Instantiating a progress bar object to show the progress each time a profile is used for testing
# With that said, the progress bar will show again if the user runs more than 1 config file
progbar = pb.progressbarClass(4) #4 stages
# Run startup time test
progbar.progress(1) #progress bar stage 1 completed
ts_times = ts.RunStartupTests(paths.BASE_PROFILE_DIR,
test_configs,
TS_NUM_RUNS)
progbar.progress(2) #progress bar stage 2 completed
# Run page load test. For possible values of counters argument, see
# http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
(tp_times, tp_counters) = tp.RunPltTests(paths.BASE_PROFILE_DIR,
test_configs,
TP_NUM_CYCLES,
['Private Bytes', 'Working Set', '% Processor Time'],
TP_RESOLUTION)
#Generate a report of the merits of the Performance Testing
#YAML profile items that are passed into GenerateReport
#are: title, filename and other test items: firefox, extensions
#and preferences
progbar.progress(3) #progress bar stage 3 completed
report.GenerateReport(title,
filename_prefix,
test_names,
ts_times,
tp_times,
tp_counters,
TP_RESOLUTION)
progbar.progress(4) #progress bar stage 4 completed
return 1
####
#
# Checks the paths in paths.py to see if they are pointing to the right location.
# If it doesn't, the user will be shown informative messages to fix the problem.
# In the case of a missing directory, the user will be prompted if he/she would like
# the directory to be created. It also checks if the base profile directory have
# any contents or else the program will throw a nasty ZeroDivision Error. File paths
# that uses the file url has to be parsed before being checked.
#
# Args:
# @type list
# @param path_dirs: a list of the dir path constant values in paths.py
#
# @type list
# @param test_files: a list of the file path constant values in paths.py (file urls)
#
####
#@ExpHandler((IOError, Exception)) #wouldn't return value from module if this is not commented out
def checkPaths():
# Checks if the following paths exist (initialized in paths.py)
# The paths in the paths.py file may be modifiled to direct to
# the desired locations.
validates = False
path_dirs = [paths.BASE_PROFILE_DIR, paths.REPORTS_DIR]
test_files =[paths.INIT_URL, paths.TP_URL, paths.TS_URL]
# Checks if the paths in paths.py exist. If it doesn't exist, the user will
# be warned and it will ask the user if he/she wants the missing directory
# to be created
for dir in path_dirs:
#head, tail = os.path.split(dir)
head = dir
if not os.path.exists(head):
print "Warning... Dir doesn't exist: %s" % head
sys.stdout.flush()
resp = raw_input("Do you want the directory to be created (y/n)? ")
while resp not in ("y", "Y", "n", "N"):
resp = raw_input("Do you want the directory to be created (y/n)? ")
sys.stdout.flush()
if resp in ("y", "Y"):
print "creating dir..."
dirname = head
os.mkdir(dirname)
print "Dir %s is created" % dirname
break
elif resp in ("n", "N"):
#killProcess.kill()
error = "\nError... Please check that" + head + " is created or exist or change path in paths.py to correct path"
sys.exit(error) #why doesn't this exit the program?
break
elif os.path.exists(head):
print "checking... Exist: Dir %s" % head
sys.stdout.flush()
# Checks if contents of base_profile_dir exist. Initially, if contents of the base_profile_dir doesn't exist,
# it will throw a ZeroDivision error. Since this module's exception is captured by the ExpHandler module,
# it throws a more appopriate error message. However, it should terminate the program, but it doesn't for
# some reason. sys.exit doesn't seem to work!!!
validatePaths = True
if os.path.isdir(paths.BASE_PROFILE_DIR):
for root, dirs, files in os.walk(paths.BASE_PROFILE_DIR):
print root, "consumes",
print sum(os.path.getsize(os.path.join(root, name)) for name in files),
print "bytes in", len(files), "non-directory files"
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
if sum(os.path.getsize(os.path.join(root, name)) for name in files) == 0:
#killProcess.kill()
validatePaths = False
error = "\nError... Please check that you have files in " + paths.BASE_PROFILE_DIR
sys.exit(error) #why doesn't this exit the program?
elif not os.path.isdir(paths.BASE_PROFILE_DIR):
#killProcess.kill()
validatePaths = False
error = "\nError... Please check that " + paths.BASE_PROFILE_DIR + " exist"
sys.exit(error) # it doesn't exit the program, why?
sys.stdout.flush()
#validates the file urls
validateURLS = True
for f in test_files:
#Converts file urls to file paths to check if the paths initialized
#in paths.py are valid. (INIT_URL, TS_URL, TP_URL)
file_url = f
file_path = file_url[8:]
file_path = file_path.replace('/', '\\')
file_path = re.split("[?]", file_path)
if not os.path.exists(file_path[0]):
error = "\nError... Please check if the path in paths.py for " + file_path[0] + " is right"
validateURLS = False
else:
print "checking... Exist: %s" %file_path[0]
sys.stdout.flush()
if not validateURLS:
sys.exit(error) # it doesn't exit the program, why?
return validateURLS and validatePaths
####
#
# Main function:
# It will validate the required files, directories and configuration before performing the
# Firefox Performance Testing. If the required configuration files/dirs don't validate, users
# cannot start Firefox Performance Testing until the error is fixed.
# User may list as many space-seperated config file arguments as possible when they run this file
# Tells the user when the performance testing is starting, completed or failed/halted.
#
####
if __name__=='__main__':
testComplete = False
filesValidate = False
# Check if paths in paths.py exist
filesValidate = checkPaths()
# Read in each config file and run the tests on it.
if filesValidate:
for i in range(1, len(sys.argv)):
testComplete = test_file(sys.argv[i])
elif not filesValidate:
print "Files didn't validate!"
if testComplete:
print "Performance testing is done... check your results in %s" %paths.REPORTS_DIR
elif not testComplete:
print "Performance testing failed"
