Winter 2009 SYA810 Block Device Benchmark Scripts

Please post your block device benchmark script here. I'm looking for a wide range of benchmark methodologies, so your script should test in a different way than the other scripts. I will collect these scripts on Monday, Jan 26 and create a master script which runs all of these tests and reports the overall results.

John Doe (example)

Script or link to script goes here

Gregory Masseau

"""

DiskTest.py: This script tests a disks write/read performance for both block devices and individual files, presenting it's output as an HTML file contaning a table.

Example invocation: sudo python DiskTest.py D="/dev/sda1" F="./x" -readd -writef -readf -d18 O=/mnt/md0/webshare/stuff/out.html

To (destructively) write test a block device add the '-writed' option. """

from sys import stdout verbosity = 2 debug = True if __name__=="__main__": repeat = 5

from os import remove, system from sys import argv from time import sleep from timeit import Timer

def setup(argv): """Takes an argument list argv and returns paraters needed for tests.""" depth = list(set([a[2:] for a in argv if a[0:2] == "-d"])) disk = list(set([a[2:] for a in argv if a[0:2] == "D="])) scratch = list(set([a[2:] for a in argv if a[0:2] == "F="])) outf = list(set([a[2:] for a in argv if a[0:2] == "O="])) tests = list(set([a[1:] for a in argv if a[1:] in ["readf","readd","writef","writed"]])) errors = list(set([a for a in argv if a not in (["-d"+x for x in depth]+["D="+x for x in disk]+["F="+x for x in scratch]+["-"+x for x in tests]+["O="+x for x in outf]+[argv[0]])])) if errors or (not(len(disk)==1)) or (not(len(scratch)==1)) or (not(len(depth))==1) or (not(len(outf)) == 1): print "USAGE -- "+argv[0]+" D=<disk> F=<scratch file> O=<output> -d<depth> [-readd] [-readf] [-writed] [-writef]" exit(1) if ("writed" in tests): system('clear') print ("WARNING!!! "*7)+"\n\nYou have selected the 'write to disk' test.\nThis test will DESTROY any file system on the device selected (if it is a partition), possibly multiple filesystems if it is an entire disk. \n\nPLEASE DOUBLE CHECK TO ENSURE NO IMPORTANT DATA IS ON THIS DISK PRIOR TO PROCEEDING!\n\nTo proceed with the test, please enter the name of the device whose contents you are about to destroy.\n\n(The test will be aborted if input does not match the disk specified previously.\n\nDevice:", if (not (raw_input() == disk[0])): print "\nAborting.\n\n" exit(1) else: tests.remove("writed") tests.append("writed") if ("writef" in tests): tests.remove("writef") tests = ["writef"]+tests return [disk,scratch,outf,tests,depth]

def time(p): """Runs a test p, returning the time in seconds it took.""" print " - time(%r)\n"%(p,) if debug else "", f,d,s,sf = p importstr = "from "+"".join(argv[0].split(".")[0:-1])+" import "+", ".join(tests) teststr = stringifyparams(p) timer = Timer(teststr,importstr) print " - timer = Timer('%s','%s')\n"%(teststr,importstr) if debug else "", stdout.write("\nTest: "+teststr if (verbosity==1) else "") time = timer.timeit(repeat) print " - time = %s\n"%time if debug else "", stdout.write("Time: "+("%.3lf"%time)+" seconds.\n" if (verbosity==1) else "") sleep(0.025) return (f,d,s,sf,time)

def stringifyparams(p): """Stringifies a set of test parameters p.""" print " - stringifyparams(%r)\n"%(p,) if debug else "", func,disk,size,scratchfile = p return "".join([func,"('",disk,"',",("%s"%size),",'",scratchf,"')"])

def dropextraneous(rs): """Takes a tuple rs and drops uneeded fields.""" print " - dropextraneous(%r)\n"%(rs,) if debug else "", f,d,s,sf,t = rs return [f,s,t]

def group(inp): """Takes a list of lists inp, returning a list of lists grouped by the first element, i.e.: [['a',1,2],['a',2,3],['b',4,5],['b',6,7],['c',3,3]] => [['a', [[1, 2], [2, 3]]], ['b', [[4, 5], [6, 7]]], ['c', 3, 3]] """ print " - group(%r)\n"%inp if debug else "", return __group(sorted(inp),[],[],"") def __group(inp,buf,out,seek): print " - group(%r,%r,%r,'%r')\n"%(inp,out,buf,seek) if debug else "", if (not (inp==[])): inph = inp[0] inpt = inp[1:] inphh = inph[0] inpht = inph[1:] seek = inphh if (seek=="") else seek return out+seek,buf if (inp==[]) else __group(inpt,buf+[inpht],out,seek) if (seek==inphh) else __group(inpt,[inpht],out+seek,buf,inphh)

def preformat(d): """Adjusts the values returned by the test loop (d) into something suitable for table."""

left = "

Bytes

Test

" right = "

Test

Bytes

"

colns = (lambda d: [""]+[("%s"%x[0]) for x in d[0][1:][0]]+[""])(d) rowns = (lambda d: [("%s"%x[0]) for x in d])(d) fdata = (lambda d: [["%.3lf s"%x[1] for x in x[1]] for x in d])(d) lol = map(list,zip(*[colns]+[[rowns[x]]+fdata[x]+[rowns[x]] for x in range(0,len(rowns))]+[colns])) height = (lambda x:len(x)-1)(lol) width = (lambda x:len(x[0])-1)(lol) lol[0] [0] = left lol[height][0] = left lol[0] [width] = right lol[height][width] = right return lol

def table(title,lol): """Takes a list of lists lol and a title title, returning as a string an HTML page titled title containing the contents of lol in a table.""" print "table(%r)\n"%lol if debug else "",

return "".join(["".join(x) for x in [["<html>\n

%s

"%title]]+[["\n "]+["\n "%w for w in l]+["\n "] for l in lol]+[["\n %s

\n</html>"]]])

settings = setup(argv) disks = settings[0] scratchfs = settings[1] outputf = settings[2][0] tests = settings[3] depth = int(settings[4][0])

testparams = [(test,disk,size,scratchf) for test in tests for size in [siz*1024 for siz in [1<<(x-1) for x in range(1,depth)]] for scratchf in scratchfs for disk in disks for depth in [depth]]

OUTFILE = open(outputf,"w") OUTFILE.write(table("Disk Benchmark Results",preformat(group(map((lambda p:dropextraneous(time(p))),testparams))))) OUTFILE.flush() OUTFILE.close()

print "\nDone. Output is in: "+outputf+"." else: from os import SEEK_END from random import randint

def donefile(FILE,tag): """Close a file.""" FILE.flush() print " - donefile(FILE) <-[ %s ]\n"%tag if debug else "", print ' - FILE.close()\n' if debug else "", FILE.close() stdout.write("done.\n" if (verbosity==2) else ""), return True

def writef(disk,size,scratchf): """Do a file write test.""" print " - writef('%s',%d,'%s')\n"%(disk,size,scratchf) if debug else "", randomseek(disk) stdout.write("(D) Writing "+("%s"%size)+" bytes to "+scratchf+"...\n" if (verbosity==2) else ""), FILE = open(scratchf,"w") print ' - FILE.write("xxx"...)\n' if debug else "", FILE.write((lambda x: "".join(["x" for x in range(1,x)]))(size)) donefile(FILE,"writef") return True

def writed(disk,size,scratchf): """Do a disk write test.""" print " - writed('%s',%d,'%s')\n"%(disk,size,scratchf) if debug else "", return writef(disk,size,disk)

def readf(disk,size,scratchf): """Do a file read test.""" print " - readf('%s',%d,'%s')\n"%(disk,size,scratchf) if debug else "", randomseek(disk) print ' - FILE = open(%s),"r"\n'%disk if debug else "", FILE = open(scratchf,"r") FILE.seek(0,0) stdout.write("(D) Reading "+("%s"%size)+" bytes from "+scratchf+"...\n" if (verbosity==2) else "") print ' - FILE.read()\n' if debug else "", FILE.read(size) if (size < 4194304) else [FILE.read(4194304) for x in range(0,size/4194304)] donefile(FILE,"readf") return True

def readd(disk,size,scratchf): """Do a file read test.""" print " - readd('%s',%d,'%s')\n"%(disk,size,scratchf) if debug else "", return readf(disk,size,disk)

def randomseek(disk): """Seek the disk head to a random position.""" print " - randomseek('%s')\n"%disk if debug else "", jumploc = randint(0,disksize(disk)) stdout.write("Randomizing head position by seeking byte "+("%s"%jumploc)+" of "+disk+"...\n" if (verbosity==2) else "") print ' - FILE = open(%s),"r"\n'%disk if debug else "", FILE = open(disk,"r") FILE.seek(jumploc) donefile(FILE,"randomseek") return True

def disksize(disk): """Get the size of a disk.""" print " - disksize('%s')\n"%disk if debug else "", print ' - FILE = open(%s),"r"\n'%disk if debug else "", FILE = open(disk,"r") FILE.seek(0,SEEK_END) l = FILE.tell() print " - FILE.close()\n" if debug else "", FILE.close() return l

def writef(disk,size,scratchf): """Do a file write test.""" print " - writef('%s',%d,'%s')\n"%(disk,size,scratchf) if debug else "", randomseek(disk) stdout.write("(D) Writing "+("%s"%size)+" bytes to "+scratchf+"...\n" if (verbosity==2) else ""), FILE = open(scratchf,"w") print ' - FILE.write(<%d>)\n'%size if debug else "", [FILE.write('\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa') for x in range(0,size/64)] donefile(FILE,"writef") return True