#!/usr/bin/python import argparse import difflib import enum import os import os.path import subprocess import sys SUITES = ["interpreter", "prettyprinter"] IN_FILE_NAME = "in.lua" OUT_FILE_NAME = "out.txt" SUITE_EXE_NAMES = { "interpreter": "Interpreter", "prettyprinter": "PrettyPrinter", } SUITE_ENTRY_POINTS = { "interpreter": "Interpreter.agda", "prettyprinter": "PrettyPrinter.agda", } SUITE_ROOTS = { "interpreter": "Tests/Interpreter", "prettyprinter": "Tests/PrettyPrinter", } class TestResultStatus(enum.Enum): CLI_ERROR = 0 EXE_ERROR = 1 DIFF_ERROR = 2 SUCCESS = 3 WROTE_NEW = 4 class DiffFailure: def __init__(self, expected, actual): self.expected = expected self.actual = actual def diff_text(self): diff_generator = difflib.context_diff(self.expected.splitlines(), self.actual.splitlines(), fromfile="expected", tofile="actual", n=3) return "".join(diff_generator) def diff_html(self): differ = difflib.HtmlDiff(tabsize=4) return differ.make_file(self.expected.splitlines(), self.actual.splitlines(), fromdesc="Expected", todesc="Actual", context=True, numlines=5) class TestCaseResult: def __init__(self, suite, case, status, details): self.suite = suite self.case = case self.status = status self.details = details def did_pass(self): return self.status == TestResultStatus.SUCCESS or self.status == TestResultStatus.WROTE_NEW def to_string(self): prefix = f"[{self.suite}/{self.case}]: " if self.status == TestResultStatus.CLI_ERROR: return f"{prefix}CLI ERROR: {self.details}" elif self.status == TestResultStatus.EXE_ERROR: return f"{prefix}EXE ERROR: {self.details}" elif self.status == TestResultStatus.DIFF_ERROR: text_diff = self.details.diff_text() return f"{prefix}FAILED:\n{text_diff}" elif self.status == TestResultStatus.SUCCESS: return f"{prefix}SUCCEEDED" elif self.status == TestResultStatus.WROTE_NEW: return f"{prefix}WROTE NEW RESULT" def write_artifact(self, artifact_root): if self.status != TestResultStatus.DIFF_ERROR: return filename = f"{self.suite}-{self.case}.out.html" path = os.path.join(artifact_root, filename) html = self.details.diff_html() with open(path, "w") as file: file.write(html) parser = argparse.ArgumentParser(description="Runs prototype test cases") parser.add_argument("--luau-cli", "-l", dest="cli_location", required=True, help="The location of luau-cli") parser.add_argument("--root", "-r", dest="prototype_root", required=False, default=os.getcwd(), help="The root of the prototype") parser.add_argument("--build", "-b", dest="build", action="store_true", default=True, help="Whether to automatically build required test binaries") parser.add_argument("--suite", "-s", dest="suites", action="append", default=[], choices=SUITES, help="Which test suites to run") parser.add_argument("--case", "-c", dest="cases", action="append", default=[], help="Which test cases to run") parser.add_argument("--accept-new-output", "-a", dest="snapshot", action="store_true", default=False, help="Whether to write the new output to files, instead of diffing against it") parser.add_argument("--write-diff-failures", dest="write_diffs", action="store_true", default=False, help="Whether to write test failure diffs to files") parser.add_argument("--diff-failure-location", dest="diff_location", default=None, help="Where to write diff failure files to") def build_suite(root, suite): entry_point = SUITE_ENTRY_POINTS.get(suite) if entry_point is None: return (False, "Invalid suite") result = subprocess.run(["~/.cabal/bin/agda", "--compile", entry_point], shell=True, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if result.returncode == 0: return (True, None) else: return (False, result.stdout) def run_test(in_path, out_path, cli_path, exe_path, snapshot): cli_result = subprocess.run([cli_path, in_path], capture_output=True) if cli_result.returncode != 0: return (TestResultStatus.CLI_ERROR, f"CLI error: {cli_result.stderr}") exe_result = subprocess.run(exe_path, input=cli_result.stdout, capture_output=True) if exe_result.returncode != 0: return (TestResultStatus.EXE_ERROR, f"Executable error; stdout:{exe_result.stdout}\n\nstderr: {exe_result.stderr}") actual_result = exe_result.stdout.decode("utf-8") if snapshot: with open(out_path, "w") as out_file: out_file.write(actual_result) return (TestResultStatus.WROTE_NEW, None) else: with open(out_path, "r") as out_file: expected_result = out_file.read() if expected_result != actual_result: return (TestResultStatus.DIFF_ERROR, DiffFailure(expected_result, actual_result)) return (TestResultStatus.SUCCESS, None) def should_run_case(case_name, filters): if len(filters) == 0: return True return any([f in case_name for f in filters]) def run_test_suite(args, suite, suite_root, suite_exe): results = [] for entry in os.listdir(suite_root): if not should_run_case(entry, args.cases): continue case_path = os.path.join(suite_root, entry) if os.path.isdir(case_path): in_path = os.path.join(case_path, IN_FILE_NAME) out_path = os.path.join(case_path, OUT_FILE_NAME) if not os.path.exists(in_path) or not os.path.exists(out_path): continue status, details = run_test(in_path, out_path, args.cli_location, suite_exe, args.snapshot) result = TestCaseResult(suite, entry, status, details) results.append(result) return results def main(): args = parser.parse_args() suites = args.suites if len(args.suites) > 0 else SUITES root = os.path.abspath(args.prototype_root) if args.build: for suite in suites: success, reason = build_suite(root, suite) if not success: print(f"Error building executable for test suite {suite}:\n{reason}") sys.exit(1) else: print(f"Built executable for test suite {suite} successfully.") failed = False for suite in suites: suite_root = os.path.join(root, SUITE_ROOTS.get(suite)) suite_exe = os.path.join(root, SUITE_EXE_NAMES.get(suite)) print(f"Running test suite {suite}...") results = run_test_suite(args, suite, suite_root, suite_exe) passed = 0 total = len(results) for result in results: if result.did_pass(): passed += 1 else: failed = True print(f"Suite {suite} [{passed} / {total} passed]:") for result in results: print(result.to_string()) if args.write_diffs: result.write_artifact(args.diff_location) if failed: sys.exit(1) if __name__ == "__main__": main()