name: benchmark on: push: branches: - master paths-ignore: - "docs/**" - "papers/**" - "rfcs/**" - "*.md" - "prototyping/**" jobs: callgrind: strategy: matrix: os: [ubuntu-22.04] benchResultsRepo: - { name: "luau-lang/benchmark-data", branch: "main" } runs-on: ${{ matrix.os }} steps: - name: Checkout Luau repository uses: actions/checkout@v3 - name: Install valgrind run: | sudo apt-get install valgrind - name: Build Luau (gcc) run: | CXX=g++ make config=profile luau cp luau luau-gcc - name: Build Luau (clang) run: | make config=profile clean CXX=clang++ make config=profile luau luau-analyze - name: Run benchmark (bench-gcc) run: | python bench/bench.py --callgrind --vm "./luau-gcc -O2" | tee -a bench-gcc-output.txt - name: Run benchmark (bench) run: | python bench/bench.py --callgrind --vm "./luau -O2" | tee -a bench-output.txt - name: Run benchmark (analyze) run: | filter() { awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-analyze"}' } valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-nonstrict | tee -a analyze-output.txt valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-strict | tee -a analyze-output.txt valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/regex.lua 2>&1 | filter regex-nonstrict | tee -a analyze-output.txt valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/regex.lua 2>&1 | filter regex-strict | tee -a analyze-output.txt - name: Run benchmark (compile) run: | filter() { awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau --compile"}' } valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O0 | tee -a compile-output.txt valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O1 | tee -a compile-output.txt valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2 | tee -a compile-output.txt valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/regex.lua 2>&1 | filter regex-O0 | tee -a compile-output.txt valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/regex.lua 2>&1 | filter regex-O1 | tee -a compile-output.txt valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/regex.lua 2>&1 | filter regex-O2 | tee -a compile-output.txt - name: Checkout benchmark results uses: actions/checkout@v3 with: repository: ${{ matrix.benchResultsRepo.name }} ref: ${{ matrix.benchResultsRepo.branch }} token: ${{ secrets.BENCH_GITHUB_TOKEN }} path: "./gh-pages" - name: Store results (bench) uses: Roblox/rhysd-github-action-benchmark@v-luau with: name: callgrind clang tool: "benchmarkluau" output-file-path: ./bench-output.txt external-data-json-path: ./gh-pages/bench.json - name: Store results (bench-gcc) uses: Roblox/rhysd-github-action-benchmark@v-luau with: name: callgrind gcc tool: "benchmarkluau" output-file-path: ./bench-gcc-output.txt external-data-json-path: ./gh-pages/bench-gcc.json - name: Store results (analyze) uses: Roblox/rhysd-github-action-benchmark@v-luau with: name: luau-analyze tool: "benchmarkluau" output-file-path: ./analyze-output.txt external-data-json-path: ./gh-pages/analyze.json - name: Store results (compile) uses: Roblox/rhysd-github-action-benchmark@v-luau with: name: luau --compile tool: "benchmarkluau" output-file-path: ./compile-output.txt external-data-json-path: ./gh-pages/compile.json - name: Push benchmark results if: github.event_name == 'push' run: | echo "Pushing benchmark results..." cd gh-pages git config user.name github-actions git config user.email github@users.noreply.github.com git add *.json git commit -m "Add benchmarks results for ${{ github.sha }}" git push cd ..