From 366df9639384e8d7ab92b29744303adec6770d77 Mon Sep 17 00:00:00 2001 From: Allan N Jeremy Date: Mon, 12 Sep 2022 18:45:55 +0300 Subject: [PATCH] Feat: Added retry functionality to benchmarks (#667) Resolves #668 ## The problem Benchmarks jobs run concurrently for the different operating systems. This means that when it comes time to push the benchmark results to [the assigned benchmark results repo](https://github.com/luau-lang/benchmark-data), there can be two different jobs trying to push changes at the same time. In such a case, one of the pushes will fail and we end up missing some benchmark results data from the workflow run. ## The solution Whenever a push fails, we need to retry the steps leading up to the push (checking out the benchmark results repo, storing benchmark results, pushing the results to [a specific repo](https://github.com/luau-lang/benchmark-data)). ### Note There are 3 push attempts before submitting to failure. ## TL;DR This PR retries pushing benchmark results when they fail to get pushed (often due to pushing from multiple jobs concurrently) Co-authored-by: Jamie Kuppens Co-authored-by: Ignacio Falk --- .github/workflows/benchmark-dev.yml | 275 +++++++++++++++------- .github/workflows/push-results/action.yml | 63 +++++ 2 files changed, 258 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/push-results/action.yml diff --git a/.github/workflows/benchmark-dev.yml b/.github/workflows/benchmark-dev.yml index 2c6eae4..210a38e 100644 --- a/.github/workflows/benchmark-dev.yml +++ b/.github/workflows/benchmark-dev.yml @@ -4,6 +4,7 @@ on: push: branches: - master + paths-ignore: - "docs/**" - "papers/**" @@ -61,34 +62,49 @@ jobs: run: | python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 + - name: Push benchmark results + id: pushBenchmarkAttempt1 + continue-on-error: true + uses: ./.github/workflows/push-results with: repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} + branch: ${{ matrix.benchResultsRepo.branch }} token: ${{ secrets.BENCH_GITHUB_TOKEN }} path: "./gh-pages" + bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})" + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau + - name: Push benchmark results (Attempt 2) + id: pushBenchmarkAttempt2 + continue-on-error: true + if: steps.pushBenchmarkAttempt1.outcome == 'failure' + uses: ./.github/workflows/push-results with: - name: ${{ matrix.bench.title }} (Windows ${{matrix.arch}}) - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})" + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data-${{ matrix.os }}.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. + - name: Push benchmark results (Attempt 3) + id: pushBenchmarkAttempt3 + continue-on-error: true + if: steps.pushBenchmarkAttempt2.outcome == 'failure' + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})" + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" unix: name: ${{matrix.os}} @@ -142,44 +158,94 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 + - name: Push benchmark results + id: pushBenchmarkAttempt1 + continue-on-error: true + uses: ./.github/workflows/push-results with: repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} + branch: ${{ matrix.benchResultsRepo.branch }} token: ${{ secrets.BENCH_GITHUB_TOKEN }} path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau + - name: Push benchmark results (Attempt 2) + id: pushBenchmarkAttempt2 + continue-on-error: true + if: steps.pushBenchmarkAttempt1.outcome == 'failure' + uses: ./.github/workflows/push-results with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Store ${{ matrix.bench.title }} result (CacheGrind) + - name: Push benchmark results (Attempt 3) + id: pushBenchmarkAttempt3 + continue-on-error: true + if: steps.pushBenchmarkAttempt2.outcome == 'failure' + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" + + - name: Push Cachegrind benchmark results if: matrix.os == 'ubuntu-latest' - uses: Roblox/rhysd-github-action-benchmark@v-luau + id: pushBenchmarkCachegrindAttempt1 + continue-on-error: true + uses: ./.github/workflows/push-results with: - name: ${{ matrix.bench.title }} (CacheGrind) - tool: "roblox" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} (CacheGrind) + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data-${{ matrix.os }}.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. + - name: Push Cachegrind benchmark results (Attempt 2) + if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt1.outcome == 'failure' + id: pushBenchmarkCachegrindAttempt2 + continue-on-error: true + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} (CacheGrind) + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" + + - name: Push Cachegrind benchmark results (Attempt 3) + if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt2.outcome == 'failure' + id: pushBenchmarkCachegrindAttempt3 + continue-on-error: true + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} (CacheGrind) + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" static-analysis: name: luau-analyze @@ -197,6 +263,7 @@ jobs: } benchResultsRepo: - { name: "luau-lang/benchmark-data", branch: "main" } + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -228,43 +295,91 @@ jobs: - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 + - name: Push static analysis results + id: pushStaticAnalysisAttempt1 + continue-on-error: true + uses: ./.github/workflows/push-results with: repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} + branch: ${{ matrix.benchResultsRepo.branch }} token: ${{ secrets.BENCH_GITHUB_TOKEN }} path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau + - name: Push static analysis results (Attempt 2) + if: steps.pushStaticAnalysisAttempt1.outcome == 'failure' + id: pushStaticAnalysisAttempt2 + continue-on-error: true + uses: ./.github/workflows/push-results with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - uses: Roblox/rhysd-github-action-benchmark@v-luau + - name: Push static analysis results (Attempt 3) + if: steps.pushStaticAnalysisAttempt2.outcome == 'failure' + id: pushStaticAnalysisAttempt3 + continue-on-error: true + uses: ./.github/workflows/push-results with: - name: ${{ matrix.bench.title }} - tool: "roblox" - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data-${{ matrix.os }}.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. + - name: Push static analysis Cachegrind results + if: matrix.os == 'ubuntu-latest' + id: pushStaticAnalysisCachegrindAttempt1 + continue-on-error: true + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" + + - name: Push static analysis Cachegrind results (Attempt 2) + if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt1.outcome == 'failure' + id: pushStaticAnalysisCachegrindAttempt2 + continue-on-error: true + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" + + - name: Push static analysis Cachegrind results (Attempt 2) + if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt2.outcome == 'failure' + id: pushStaticAnalysisCachegrindAttempt3 + continue-on-error: true + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: ${{ matrix.bench.title }} + bench_tool: "roblox" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" diff --git a/.github/workflows/push-results/action.yml b/.github/workflows/push-results/action.yml new file mode 100644 index 0000000..b5ffebe --- /dev/null +++ b/.github/workflows/push-results/action.yml @@ -0,0 +1,63 @@ +name: Checkout & push results +description: Checkout a given repo and push results to GitHub +inputs: + repository: + required: true + type: string + description: The benchmark results repository to check out + branch: + required: true + type: string + description: The benchmark results repository's branch to check out + token: + required: true + type: string + description: The GitHub token to use for pushing results + path: + required: true + type: string + description: The path to check out the results repository to + bench_name: + required: true + type: string + bench_tool: + required: true + type: string + bench_output_file_path: + required: true + type: string + bench_external_data_json_path: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.branch }} + token: ${{ inputs.token }} + path: ${{ inputs.path }} + + - name: Store results + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ inputs.bench_name }} + tool: ${{ inputs.bench_tool }} + gh-pages-branch: ${{ inputs.branch }} + output-file-path: ${{ inputs.bench_output_file_path }} + external-data-json-path: ${{ inputs.bench_external_data_json_path }} + + - name: Push benchmark results + shell: bash + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add *.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd ..