#!/usr/bin/python3 # This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details import argparse import os import subprocess import math import sys import re import json # Taken from rotest from color import colored, Color from tabulate import TablePrinter, Alignment try: import matplotlib import matplotlib.pyplot as plt except ModuleNotFoundError: matplotlib = None try: import scipy from scipy import stats except ModuleNotFoundError: print("Warning: scipy package is not installed, confidence values will not be available") stats = None scriptdir = os.path.dirname(os.path.realpath(__file__)) defaultVm = 'luau.exe' if os.name == "nt" else './luau' argumentParser = argparse.ArgumentParser(description='Benchmark Lua script execution with an option to compare different VMs') argumentParser.add_argument('--vm', dest='vm',default=defaultVm,help='Lua executable to test (' + defaultVm + ' by default)') argumentParser.add_argument('--folder', dest='folder',default=os.path.join(scriptdir, 'tests'),help='Folder with tests (tests by default)') argumentParser.add_argument('--compare', dest='vmNext',type=str,nargs='*',help='List of Lua executables to compare against') argumentParser.add_argument('--results', dest='results',type=str,nargs='*',help='List of json result files to compare and graph') argumentParser.add_argument('--run-test', action='store', default=None, help='Regex test filter') argumentParser.add_argument('--extra-loops', action='store',type=int,default=0, help='Amount of times to loop over one test (one test already performs multiple runs)') argumentParser.add_argument('--filename', action='store',type=str,default='bench', help='File name for graph and results file') argumentParser.add_argument('--callgrind', dest='callgrind',action='store_const',const=1,default=0,help='Use callgrind to run benchmarks') argumentParser.add_argument('--show-commands', dest='show_commands',action='store_const',const=1,default=0,help='Show the command line used to launch the VM and tests') if matplotlib != None: argumentParser.add_argument('--absolute', dest='absolute',action='store_const',const=1,default=0,help='Display absolute values instead of relative (enabled by default when benchmarking a single VM)') argumentParser.add_argument('--speedup', dest='speedup',action='store_const',const=1,default=0,help='Draw a speedup graph') argumentParser.add_argument('--sort', dest='sort',action='store_const',const=1,default=0,help='Sort values from worst to best improvements, ignoring conf. int. (disabled by default)') argumentParser.add_argument('--window', dest='window',action='store_const',const=1,default=0,help='Display window with resulting plot (disabled by default)') argumentParser.add_argument('--graph-vertical', action='store_true',dest='graph_vertical', help="Draw graph with vertical bars instead of horizontal") argumentParser.add_argument('--report-metrics', dest='report_metrics', help="Send metrics about this session to InfluxDB URL upon completion.") argumentParser.add_argument('--print-influx-debugging', action='store_true', dest='print_influx_debugging', help="Print output to aid in debugging of influx metrics reporting.") argumentParser.add_argument('--no-print-influx-debugging', action='store_false', dest='print_influx_debugging', help="Don't print output to aid in debugging of influx metrics reporting.") argumentParser.add_argument('--no-print-final-summary', action='store_false', dest='print_final_summary', help="Don't print a table summarizing the results after all tests are run") # Assume 2.5 IPC on a 4 GHz CPU; this is obviously incorrect but it allows us to display simulated instruction counts using regular time units CALLGRIND_INSN_PER_SEC = 2.5 * 4e9 def arrayRange(count): result = [] for i in range(count): result.append(i) return result def arrayRangeOffset(count, offset): result = [] for i in range(count): result.append(i + offset) return result def getCallgrindOutput(stdout, lines): result = [] name = None for l in lines: if l.startswith("desc: Trigger: Client Request: "): name = l[31:].strip() elif l.startswith("summary: ") and name != None: insn = int(l[9:]) # Note: we only run each bench once under callgrind so we only report a single time per run; callgrind instruction count variance is ~0.01% so it might as well be zero result += "|><|" + name + "|><|" + str(insn / CALLGRIND_INSN_PER_SEC * 1000.0) + "||_||" name = None # If no results were found above, this may indicate the native executable running # the benchmark doesn't have support for callgrind builtin. In that case just # report the "totals" from the output file. if len(result) == 0: elements = stdout.decode('utf8').split("|><|") if len(elements) >= 2: name = elements[1] for l in lines: if l.startswith("totals: "): insn = int(l[8:]) # Note: we only run each bench once under callgrind so we only report a single time per run; callgrind instruction count variance is ~0.01% so it might as well be zero result += "|><|" + name + "|><|" + str(insn / CALLGRIND_INSN_PER_SEC * 1000.0) + "||_||" return "".join(result) def conditionallyShowCommand(cmd): if arguments.show_commands: print(f'{colored(Color.BLUE, "EXECUTING")}: {cmd}') def checkValgrindExecutable(): """Return true if valgrind can be successfully spawned""" try: subprocess.check_call("valgrind --version", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except: print(f"{colored(Color.YELLOW, 'WARNING')}: Unable to spawn 'valgrind'. Please ensure valgrind is installed when using '--callgrind'.") return False return True def getVmOutput(cmd): if os.name == "nt": try: fullCmd = "start /realtime /affinity 1 /b /wait cmd /C \"" + cmd + "\"" conditionallyShowCommand(fullCmd) return subprocess.check_output(fullCmd, shell=True, cwd=scriptdir).decode() except KeyboardInterrupt: exit(1) except: return "" elif arguments.callgrind: if not checkValgrindExecutable(): return "" output_path = os.path.join(scriptdir, "callgrind.out") try: os.unlink(output_path) # Remove stale output except: pass fullCmd = "valgrind --tool=callgrind --callgrind-out-file=callgrind.out --combine-dumps=yes --dump-line=no " + cmd conditionallyShowCommand(fullCmd) try: output = subprocess.check_output(fullCmd, shell=True, stderr=subprocess.DEVNULL, cwd=scriptdir) except subprocess.CalledProcessError as e: print(f"{colored(Color.YELLOW, 'WARNING')}: Valgrind returned error code {e.returncode}") output = e.output with open(output_path, "r") as file: lines = file.readlines() os.unlink(output_path) return getCallgrindOutput(output, lines) else: conditionallyShowCommand(cmd) with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=scriptdir) as p: # Try to lock to a single processor if sys.platform != "darwin": os.sched_setaffinity(p.pid, { 0 }) # Try to set high priority (requires sudo) try: os.nice(-10) except: pass return p.communicate()[0] def getShortVmName(name): # Hope that the path to executable doesn't contain spaces argumentPos = name.find(" ") if argumentPos != -1: executableName = name[0:argumentPos] arguments = name[argumentPos+1:] pathPos = executableName.rfind("\\") if pathPos == -1: pathPos = executableName.rfind("/") if pathPos != -1: executableName = executableName[pathPos+1:] return executableName + " " + arguments pathPos = name.rfind("\\") if pathPos == -1: pathPos = name.rfind("/") if pathPos != -1: return name[pathPos+1:] return name class TestResult: filename = "" vm = "" shortVm = "" name = "" values = [] count = 0 min = None avg = 0 max = None sampleStdDev = 0 unbiasedEst = 0 sampleConfidenceInterval = 0 def extractResult(filename, vm, output): elements = output.split("|><|") # Remove test output elements.remove(elements[0]) result = TestResult() result.filename = filename result.vm = vm result.shortVm = getShortVmName(vm) result.name = elements[0] elements.remove(elements[0]) timeTable = [] for el in elements: timeTable.append(float(el)) result.values = timeTable result.count = len(timeTable) return result def mergeResult(lhs, rhs): for value in rhs.values: lhs.values.append(value) lhs.count = len(lhs.values) def mergeResults(lhs, rhs): for a, b in zip(lhs, rhs): mergeResult(a, b) def finalizeResult(result): total = 0.0 # Compute basic parameters for v in result.values: if result.min == None or v < result.min: result.min = v if result.max == None or v > result.max: result.max = v total = total + v if result.count > 0: result.avg = total / result.count else: result.avg = 0 # Compute standard deviation sumOfSquares = 0 for v in result.values: sumOfSquares = sumOfSquares + (v - result.avg) ** 2 if result.count > 1: result.sampleStdDev = math.sqrt(sumOfSquares / (result.count - 1)) result.unbiasedEst = result.sampleStdDev * result.sampleStdDev if stats: # Two-tailed distribution with 95% conf. tValue = stats.t.ppf(1 - 0.05 / 2, result.count - 1) # Compute confidence interval result.sampleConfidenceInterval = tValue * result.sampleStdDev / math.sqrt(result.count) else: result.sampleConfidenceInterval = result.sampleStdDev else: result.sampleStdDev = 0 result.unbiasedEst = 0 result.sampleConfidenceInterval = 0 return result # Full result set allResults = [] # Data for the graph plotLegend = [] plotLabels = [] plotValueLists = [] plotConfIntLists = [] # Totals vmTotalMin = [] vmTotalAverage = [] vmTotalImprovement = [] vmTotalResults = [] # Data for Telegraf report mainTotalMin = 0 mainTotalAverage = 0 mainTotalMax = 0 def getExtraArguments(filepath): try: with open(filepath) as f: for i in f.readlines(): pos = i.find("--bench-args:") if pos != -1: return i[pos + 13:].strip() except: pass return "" def substituteArguments(cmd, extra): if argumentSubstituionCallback != None: cmd = argumentSubstituionCallback(cmd) if cmd.find("@EXTRA") != -1: cmd = cmd.replace("@EXTRA", extra) else: cmd = cmd + " " + extra return cmd def extractResults(filename, vm, output, allowFailure): results = [] splitOutput = output.split("||_||") if len(splitOutput) <= 1: if allowFailure: result = TestResult() result.filename = filename result.vm = vm result.shortVm = getShortVmName(vm) results.append(result) return results splitOutput.remove(splitOutput[len(splitOutput) - 1]) for el in splitOutput: results.append(extractResult(filename, vm, el)) return results def analyzeResult(subdir, main, comparisons): # Aggregate statistics global mainTotalMin, mainTotalAverage, mainTotalMax mainTotalMin = mainTotalMin + main.min mainTotalAverage = mainTotalAverage + main.avg mainTotalMax = mainTotalMax + main.max if arguments.vmNext != None: resultPrinter.add_row({ 'Test': main.name, 'Min': '{:8.3f}ms'.format(main.min), 'Average': '{:8.3f}ms'.format(main.avg), 'StdDev%': '{:8.3f}%'.format(main.sampleConfidenceInterval / main.avg * 100), 'Driver': main.shortVm, 'Speedup': "", 'Significance': "", 'P(T<=t)': "" }) else: resultPrinter.add_row({ 'Test': main.name, 'Min': '{:8.3f}ms'.format(main.min), 'Average': '{:8.3f}ms'.format(main.avg), 'StdDev%': '{:8.3f}%'.format(main.sampleConfidenceInterval / main.avg * 100), 'Driver': main.shortVm }) if influxReporter != None: influxReporter.report_result(subdir, main.name, main.filename, "SUCCESS", main.min, main.avg, main.max, main.sampleConfidenceInterval, main.shortVm, main.vm) print(colored(Color.GREEN, 'SUCCESS') + ': {:<40}'.format(main.name) + ": " + '{:8.3f}'.format(main.avg) + "ms +/- " + '{:6.3f}'.format(main.sampleConfidenceInterval / main.avg * 100) + "% on " + main.shortVm) plotLabels.append(main.name) index = 0 if len(plotValueLists) < index + 1: plotValueLists.append([]) plotConfIntLists.append([]) vmTotalMin.append(0.0) vmTotalAverage.append(0.0) vmTotalImprovement.append(0.0) vmTotalResults.append(0) if arguments.absolute or arguments.speedup: scale = 1 else: scale = 100 / main.avg plotValueLists[index].append(main.avg * scale) plotConfIntLists[index].append(main.sampleConfidenceInterval * scale) vmTotalMin[index] += main.min vmTotalAverage[index] += main.avg for compare in comparisons: index = index + 1 if len(plotValueLists) < index + 1 and not arguments.speedup: plotValueLists.append([]) plotConfIntLists.append([]) vmTotalMin.append(0.0) vmTotalAverage.append(0.0) vmTotalImprovement.append(0.0) vmTotalResults.append(0) if compare.min == None: print(colored(Color.RED, 'FAILED') + ": '" + main.name + "' on '" + compare.vm + "'") resultPrinter.add_row({ 'Test': main.name, 'Min': "", 'Average': "FAILED", 'StdDev%': "", 'Driver': compare.shortVm, 'Speedup': "", 'Significance': "", 'P(T<=t)': "" }) if influxReporter != None: influxReporter.report_result(subdir, main.filename, main.filename, "FAILED", 0.0, 0.0, 0.0, 0.0, compare.shortVm, compare.vm) if arguments.speedup: plotValueLists[0].pop() plotValueLists[0].append(0) plotConfIntLists[0].pop() plotConfIntLists[0].append(0) else: plotValueLists[index].append(0) plotConfIntLists[index].append(0) continue if main.count > 1 and stats: pooledStdDev = math.sqrt((main.unbiasedEst + compare.unbiasedEst) / 2) tStat = abs(main.avg - compare.avg) / (pooledStdDev * math.sqrt(2 / main.count)) degreesOfFreedom = 2 * main.count - 2 # Two-tailed distribution with 95% conf. tCritical = stats.t.ppf(1 - 0.05 / 2, degreesOfFreedom) noSignificantDifference = tStat < tCritical pValue = 2 * (1 - stats.t.cdf(tStat, df = degreesOfFreedom)) else: noSignificantDifference = None pValue = -1 if noSignificantDifference is None: verdict = "" elif noSignificantDifference: verdict = "likely same" elif main.avg < compare.avg: verdict = "likely worse" else: verdict = "likely better" speedup = (plotValueLists[0][-1] / (compare.avg * scale) - 1) speedupColor = Color.YELLOW if speedup < 0 and noSignificantDifference else Color.RED if speedup < 0 else Color.GREEN if speedup > 0 else Color.YELLOW resultPrinter.add_row({ 'Test': main.name, 'Min': '{:8.3f}ms'.format(compare.min), 'Average': '{:8.3f}ms'.format(compare.avg), 'StdDev%': '{:8.3f}%'.format(compare.sampleConfidenceInterval / compare.avg * 100), 'Driver': compare.shortVm, 'Speedup': colored(speedupColor, '{:8.3f}%'.format(speedup * 100)), 'Significance': verdict, 'P(T<=t)': '---' if pValue < 0 else '{:.0f}%'.format(pValue * 100) }) print(colored(Color.GREEN, 'SUCCESS') + ': {:<40}'.format(main.name) + ": " + '{:8.3f}'.format(compare.avg) + "ms +/- " + '{:6.3f}'.format(compare.sampleConfidenceInterval / compare.avg * 100) + "% on " + compare.shortVm + ' ({:+7.3f}%, '.format(speedup * 100) + verdict + ")") if influxReporter != None: influxReporter.report_result(subdir, main.name, main.filename, "SUCCESS", compare.min, compare.avg, compare.max, compare.sampleConfidenceInterval, compare.shortVm, compare.vm) if arguments.speedup: oldValue = plotValueLists[0].pop() newValue = compare.avg plotValueLists[0].append((oldValue / newValue - 1) * 100) plotConfIntLists[0].pop() plotConfIntLists[0].append(0) else: plotValueLists[index].append(compare.avg * scale) plotConfIntLists[index].append(compare.sampleConfidenceInterval * scale) vmTotalMin[index] += compare.min vmTotalAverage[index] += compare.avg vmTotalImprovement[index] += math.log(main.avg / compare.avg) vmTotalResults[index] += 1 def runTest(subdir, filename, filepath): filepath = os.path.abspath(filepath) mainVm = os.path.abspath(arguments.vm) # Process output will contain the test name and execution times mainOutput = getVmOutput(substituteArguments(mainVm, getExtraArguments(filepath)) + " " + filepath) mainResultSet = extractResults(filename, mainVm, mainOutput, False) if len(mainResultSet) == 0: print(colored(Color.RED, 'FAILED') + ": '" + filepath + "' on '" + mainVm + "'") if arguments.vmNext != None: resultPrinter.add_row({ 'Test': filepath, 'Min': "", 'Average': "FAILED", 'StdDev%': "", 'Driver': getShortVmName(mainVm), 'Speedup': "", 'Significance': "", 'P(T<=t)': "" }) else: resultPrinter.add_row({ 'Test': filepath, 'Min': "", 'Average': "FAILED", 'StdDev%': "", 'Driver': getShortVmName(mainVm) }) if influxReporter != None: influxReporter.report_result(subdir, filename, filename, "FAILED", 0.0, 0.0, 0.0, 0.0, getShortVmName(mainVm), mainVm) return compareResultSets = [] if arguments.vmNext != None: for compareVm in arguments.vmNext: compareVm = os.path.abspath(compareVm) compareOutput = getVmOutput(substituteArguments(compareVm, getExtraArguments(filepath)) + " " + filepath) compareResultSet = extractResults(filename, compareVm, compareOutput, True) compareResultSets.append(compareResultSet) if arguments.extra_loops > 0: # get more results for i in range(arguments.extra_loops): extraMainOutput = getVmOutput(substituteArguments(mainVm, getExtraArguments(filepath)) + " " + filepath) extraMainResultSet = extractResults(filename, mainVm, extraMainOutput, False) mergeResults(mainResultSet, extraMainResultSet) if arguments.vmNext != None: i = 0 for compareVm in arguments.vmNext: compareVm = os.path.abspath(compareVm) extraCompareOutput = getVmOutput(substituteArguments(compareVm, getExtraArguments(filepath)) + " " + filepath) extraCompareResultSet = extractResults(filename, compareVm, extraCompareOutput, True) mergeResults(compareResultSets[i], extraCompareResultSet) i += 1 # finalize results for result in mainResultSet: finalizeResult(result) for compareResultSet in compareResultSets: for result in compareResultSet: finalizeResult(result) # analyze results for i in range(len(mainResultSet)): mainResult = mainResultSet[i] compareResults = [] for el in compareResultSets: if i < len(el): compareResults.append(el[i]) else: noResult = TestResult() noResult.filename = el[0].filename noResult.vm = el[0].vm noResult.shortVm = el[0].shortVm compareResults.append(noResult) analyzeResult(subdir, mainResult, compareResults) mergedResults = [] mergedResults.append(mainResult) for el in compareResults: mergedResults.append(el) allResults.append(mergedResults) def rearrangeSortKeyForComparison(e): if plotValueLists[1][e] == 0: return 1 return plotValueLists[0][e] / plotValueLists[1][e] def rearrangeSortKeyForSpeedup(e): return plotValueLists[0][e] def rearrangeSortKeyDescending(e): return -plotValueLists[0][e] # Re-arrange results from worst to best def rearrange(key): global plotLabels index = arrayRange(len(plotLabels)) index = sorted(index, key=key) # Recreate value lists in sorted order plotLabelsPrev = plotLabels plotLabels = [] for i in index: plotLabels.append(plotLabelsPrev[i]) for group in range(len(plotValueLists)): plotValueListPrev = plotValueLists[group] plotValueLists[group] = [] plotConfIntListPrev = plotConfIntLists[group] plotConfIntLists[group] = [] for i in index: plotValueLists[group].append(plotValueListPrev[i]) plotConfIntLists[group].append(plotConfIntListPrev[i]) # Graph def graph(): if len(plotValueLists) == 0: print("No results") return ind = arrayRange(len(plotLabels)) width = 0.8 / len(plotValueLists) if arguments.graph_vertical: # Extend graph width when we have a lot of tests to draw barcount = len(plotValueLists[0]) plt.figure(figsize=(max(8, barcount * 0.3), 8)) else: # Extend graph height when we have a lot of tests to draw barcount = len(plotValueLists[0]) plt.figure(figsize=(8, max(8, barcount * 0.3))) plotBars = [] matplotlib.rc('xtick', labelsize=10) matplotlib.rc('ytick', labelsize=10) if arguments.graph_vertical: # Draw Y grid behind the bars plt.rc('axes', axisbelow=True) plt.grid(True, 'major', 'y') for i in range(len(plotValueLists)): bar = plt.bar(arrayRangeOffset(len(plotLabels), i * width), plotValueLists[i], width, yerr=plotConfIntLists[i]) plotBars.append(bar[0]) if arguments.absolute: plt.ylabel('Time (ms)') elif arguments.speedup: plt.ylabel('Speedup (%)') else: plt.ylabel('Relative time (%)') plt.title('Benchmark') plt.xticks(ind, plotLabels, rotation='vertical') else: # Draw X grid behind the bars plt.rc('axes', axisbelow=True) plt.grid(True, 'major', 'x') for i in range(len(plotValueLists)): bar = plt.barh(arrayRangeOffset(len(plotLabels), i * width), plotValueLists[i], width, xerr=plotConfIntLists[i]) plotBars.append(bar[0]) if arguments.absolute: plt.xlabel('Time (ms)') elif arguments.speedup: plt.xlabel('Speedup (%)') else: plt.xlabel('Relative time (%)') plt.title('Benchmark') plt.yticks(ind, plotLabels) plt.gca().invert_yaxis() plt.legend(plotBars, plotLegend) plt.tight_layout() plt.savefig(arguments.filename + ".png", dpi=200) if arguments.window: plt.show() def addTotalsToTable(): if len(vmTotalMin) == 0: return if arguments.vmNext != None: index = 0 resultPrinter.add_row({ 'Test': 'Total', 'Min': '{:8.3f}ms'.format(vmTotalMin[index]), 'Average': '{:8.3f}ms'.format(vmTotalAverage[index]), 'StdDev%': "---", 'Driver': getShortVmName(os.path.abspath(arguments.vm)), 'Speedup': "", 'Significance': "", 'P(T<=t)': "" }) for compareVm in arguments.vmNext: index = index + 1 speedup = vmTotalAverage[0] / vmTotalAverage[index] * 100 - 100 resultPrinter.add_row({ 'Test': 'Total', 'Min': '{:8.3f}ms'.format(vmTotalMin[index]), 'Average': '{:8.3f}ms'.format(vmTotalAverage[index]), 'StdDev%': "---", 'Driver': getShortVmName(os.path.abspath(compareVm)), 'Speedup': colored(Color.RED if speedup < 0 else Color.GREEN if speedup > 0 else Color.YELLOW, '{:8.3f}%'.format(speedup)), 'Significance': "", 'P(T<=t)': "" }) else: resultPrinter.add_row({ 'Test': 'Total', 'Min': '{:8.3f}ms'.format(vmTotalMin[0]), 'Average': '{:8.3f}ms'.format(vmTotalAverage[0]), 'StdDev%': "---", 'Driver': getShortVmName(os.path.abspath(arguments.vm)) }) def writeResultsToFile(): class TestResultEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, TestResult): return [obj.filename, obj.vm, obj.shortVm, obj.name, obj.values, obj.count] return json.JSONEncoder.default(self, obj) try: with open(arguments.filename + ".json", "w") as allResultsFile: allResultsFile.write(json.dumps(allResults, cls=TestResultEncoder)) except: print("Failed to write results to a file") def run(args, argsubcb): global arguments, resultPrinter, influxReporter, argumentSubstituionCallback, allResults arguments = args argumentSubstituionCallback = argsubcb if os.name == "nt" and arguments.callgrind: print(f"{colored(Color.RED, 'ERROR')}: --callgrind is not supported on Windows. Please consider using this option on another OS, or Linux using WSL.") sys.exit(1) if arguments.report_metrics or arguments.print_influx_debugging: import influxbench influxReporter = influxbench.InfluxReporter(arguments) else: influxReporter = None if matplotlib == None: arguments.absolute = 0 arguments.speedup = 0 arguments.sort = 0 arguments.window = 0 # Load results from files if arguments.results != None: vmList = [] for result in arguments.results: with open(result) as resultsFile: resultArray = json.load(resultsFile) for test in resultArray: for i in range(len(test)): arr = test[i] tr = TestResult() tr.filename = arr[0] tr.vm = arr[1] tr.shortVm = arr[2] tr.name = arr[3] tr.values = arr[4] tr.count = arr[5] test[i] = tr for test in resultArray[0]: if vmList.count(test.vm) > 0: pointPos = result.rfind(".") if pointPos != -1: vmList.append(test.vm + " [" + result[0:pointPos] + "]") else: vmList.append(test.vm + " [" + result + "]") else: vmList.append(test.vm) if len(allResults) == 0: allResults = resultArray else: for prevEl in allResults: found = False for nextEl in resultArray: if nextEl[0].filename == prevEl[0].filename and nextEl[0].name == prevEl[0].name: for run in nextEl: prevEl.append(run) found = True if not found: el = resultArray[0] for run in el: result = TestResult() result.filename = run.filename result.vm = run.vm result.shortVm = run.shortVm result.name = run.name prevEl.append(result) arguments.vmNext = [] for i in range(len(vmList)): if i == 0: arguments.vm = vmList[i] else: arguments.vmNext.append(vmList[i]) plotLegend.append(getShortVmName(arguments.vm)) if arguments.vmNext != None: for compareVm in arguments.vmNext: plotLegend.append(getShortVmName(compareVm)) else: arguments.absolute = 1 # When looking at one VM, I feel that relative graph doesn't make a lot of sense # Results table formatting if arguments.vmNext != None: resultPrinter = TablePrinter([ {'label': 'Test', 'align': Alignment.LEFT}, {'label': 'Min', 'align': Alignment.RIGHT}, {'label': 'Average', 'align': Alignment.RIGHT}, {'label': 'StdDev%', 'align': Alignment.RIGHT}, {'label': 'Driver', 'align': Alignment.LEFT}, {'label': 'Speedup', 'align': Alignment.RIGHT}, {'label': 'Significance', 'align': Alignment.LEFT}, {'label': 'P(T<=t)', 'align': Alignment.RIGHT} ]) else: resultPrinter = TablePrinter([ {'label': 'Test', 'align': Alignment.LEFT}, {'label': 'Min', 'align': Alignment.RIGHT}, {'label': 'Average', 'align': Alignment.RIGHT}, {'label': 'StdDev%', 'align': Alignment.RIGHT}, {'label': 'Driver', 'align': Alignment.LEFT} ]) if arguments.results != None: for resultSet in allResults: # finalize results for result in resultSet: finalizeResult(result) # analyze results mainResult = resultSet[0] compareResults = [] for i in range(len(resultSet)): if i != 0: compareResults.append(resultSet[i]) analyzeResult('', mainResult, compareResults) else: all_files = [subdir + os.sep + filename for subdir, dirs, files in os.walk(arguments.folder) for filename in files] for filepath in sorted(all_files): subdir, filename = os.path.split(filepath) if filename.endswith(".lua"): if arguments.run_test == None or re.match(arguments.run_test, filename[:-4]): runTest(subdir, filename, filepath) if arguments.sort and len(plotValueLists) > 1: rearrange(rearrangeSortKeyForComparison) elif arguments.sort and len(plotValueLists) == 1: rearrange(rearrangeSortKeyDescending) elif arguments.speedup: rearrange(rearrangeSortKeyForSpeedup) plotLegend[0] = arguments.vm + " vs " + arguments.vmNext[0] if arguments.print_final_summary: addTotalsToTable() print() print(colored(Color.YELLOW, '==================================================RESULTS==================================================')) resultPrinter.print(summary=False) print(colored(Color.YELLOW, '---')) if len(vmTotalMin) != 0 and arguments.vmNext != None: index = 0 for compareVm in arguments.vmNext: index = index + 1 name = getShortVmName(os.path.abspath(compareVm)) deltaGeoMean = math.exp(vmTotalImprovement[index] / vmTotalResults[index]) * 100 - 100 if deltaGeoMean > 0: print("'{}' change is {:.3f}% positive on average".format(name, deltaGeoMean)) else: print("'{}' change is {:.3f}% negative on average".format(name, deltaGeoMean)) if matplotlib != None: graph() writeResultsToFile() if influxReporter != None: influxReporter.report_result(arguments.folder, "Total", "all", "SUCCESS", mainTotalMin, mainTotalAverage, mainTotalMax, 0.0, getShortVmName(arguments.vm), os.path.abspath(arguments.vm)) influxReporter.flush(0) if __name__ == "__main__": arguments = argumentParser.parse_args() run(arguments, None)