Feat: Added retry functionality to benchmarks (#667)
Resolves #668 ## The problem Benchmarks jobs run concurrently for the different operating systems. This means that when it comes time to push the benchmark results to [the assigned benchmark results repo](https://github.com/luau-lang/benchmark-data), there can be two different jobs trying to push changes at the same time. In such a case, one of the pushes will fail and we end up missing some benchmark results data from the workflow run. ## The solution Whenever a push fails, we need to retry the steps leading up to the push (checking out the benchmark results repo, storing benchmark results, pushing the results to [a specific repo](https://github.com/luau-lang/benchmark-data)). ### Note There are 3 push attempts before submitting to failure. ## TL;DR This PR retries pushing benchmark results when they fail to get pushed (often due to pushing from multiple jobs concurrently) Co-authored-by: Jamie Kuppens <reshurum@gmail.com> Co-authored-by: Ignacio Falk <flakolefluk@gmail.com>
This commit is contained in:
Allan N Jeremy
GitHub
parent
ce2c3b3a4e
commit
366df96393
|
|
@ -4,6 +4,7 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "papers/**"
|
||||
|
|
@ -61,34 +62,49 @@ jobs:
|
|||
run: |
|
||||
python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt
|
||||
|
||||
- name: Checkout Benchmark Results repository
|
||||
uses: actions/checkout@v3
|
||||
- name: Push benchmark results
|
||||
id: pushBenchmarkAttempt1
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
ref: ${{ matrix.benchResultsRepo.branch }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Store ${{ matrix.bench.title }} result
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
- name: Push benchmark results (Attempt 2)
|
||||
id: pushBenchmarkAttempt2
|
||||
continue-on-error: true
|
||||
if: steps.pushBenchmarkAttempt1.outcome == 'failure'
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
name: ${{ matrix.bench.title }} (Windows ${{matrix.arch}})
|
||||
tool: "benchmarkluau"
|
||||
output-file-path: ./${{ matrix.bench.script }}-output.txt
|
||||
external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push benchmark results
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
echo "Pushing benchmark results..."
|
||||
cd gh-pages
|
||||
git config user.name github-actions
|
||||
git config user.email github@users.noreply.github.com
|
||||
git add ./dev/bench/data-${{ matrix.os }}.json
|
||||
git commit -m "Add benchmarks results for ${{ github.sha }}"
|
||||
git push
|
||||
cd ..
|
||||
- name: Push benchmark results (Attempt 3)
|
||||
id: pushBenchmarkAttempt3
|
||||
continue-on-error: true
|
||||
if: steps.pushBenchmarkAttempt2.outcome == 'failure'
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
unix:
|
||||
name: ${{matrix.os}}
|
||||
|
|
@ -142,44 +158,94 @@ jobs:
|
|||
if: matrix.os == 'ubuntu-latest'
|
||||
run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt
|
||||
|
||||
- name: Checkout Benchmark Results repository
|
||||
uses: actions/checkout@v3
|
||||
- name: Push benchmark results
|
||||
id: pushBenchmarkAttempt1
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
ref: ${{ matrix.benchResultsRepo.branch }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Store ${{ matrix.bench.title }} result
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
- name: Push benchmark results (Attempt 2)
|
||||
id: pushBenchmarkAttempt2
|
||||
continue-on-error: true
|
||||
if: steps.pushBenchmarkAttempt1.outcome == 'failure'
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
name: ${{ matrix.bench.title }}
|
||||
tool: "benchmarkluau"
|
||||
output-file-path: ./${{ matrix.bench.script }}-output.txt
|
||||
external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
|
||||
github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Store ${{ matrix.bench.title }} result (CacheGrind)
|
||||
- name: Push benchmark results (Attempt 3)
|
||||
id: pushBenchmarkAttempt3
|
||||
continue-on-error: true
|
||||
if: steps.pushBenchmarkAttempt2.outcome == 'failure'
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "benchmarkluau"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push Cachegrind benchmark results
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
id: pushBenchmarkCachegrindAttempt1
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
name: ${{ matrix.bench.title }} (CacheGrind)
|
||||
tool: "roblox"
|
||||
output-file-path: ./${{ matrix.bench.script }}-output.txt
|
||||
external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
|
||||
github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }} (CacheGrind)
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push benchmark results
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
echo "Pushing benchmark results..."
|
||||
cd gh-pages
|
||||
git config user.name github-actions
|
||||
git config user.email github@users.noreply.github.com
|
||||
git add ./dev/bench/data-${{ matrix.os }}.json
|
||||
git commit -m "Add benchmarks results for ${{ github.sha }}"
|
||||
git push
|
||||
cd ..
|
||||
- name: Push Cachegrind benchmark results (Attempt 2)
|
||||
if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt1.outcome == 'failure'
|
||||
id: pushBenchmarkCachegrindAttempt2
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }} (CacheGrind)
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push Cachegrind benchmark results (Attempt 3)
|
||||
if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt2.outcome == 'failure'
|
||||
id: pushBenchmarkCachegrindAttempt3
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }} (CacheGrind)
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
static-analysis:
|
||||
name: luau-analyze
|
||||
|
|
@ -197,6 +263,7 @@ jobs:
|
|||
}
|
||||
benchResultsRepo:
|
||||
- { name: "luau-lang/benchmark-data", branch: "main" }
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
@ -228,43 +295,91 @@ jobs:
|
|||
- name: Run ${{ matrix.bench.title }} (Warm Cachegrind)
|
||||
run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
|
||||
|
||||
- name: Checkout Benchmark Results repository
|
||||
uses: actions/checkout@v3
|
||||
- name: Push static analysis results
|
||||
id: pushStaticAnalysisAttempt1
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
ref: ${{ matrix.benchResultsRepo.branch }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Store ${{ matrix.bench.title }} result
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
- name: Push static analysis results (Attempt 2)
|
||||
if: steps.pushStaticAnalysisAttempt1.outcome == 'failure'
|
||||
id: pushStaticAnalysisAttempt2
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
name: ${{ matrix.bench.title }}
|
||||
tool: "benchmarkluau"
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
gh-pages-branch: "main"
|
||||
output-file-path: ./${{ matrix.bench.script }}-output.txt
|
||||
external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
|
||||
github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
|
||||
- name: Store ${{ matrix.bench.title }} result (CacheGrind)
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
- name: Push static analysis results (Attempt 3)
|
||||
if: steps.pushStaticAnalysisAttempt2.outcome == 'failure'
|
||||
id: pushStaticAnalysisAttempt3
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
name: ${{ matrix.bench.title }}
|
||||
tool: "roblox"
|
||||
gh-pages-branch: "main"
|
||||
output-file-path: ./${{ matrix.bench.script }}-output.txt
|
||||
external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
|
||||
github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push benchmark results
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
echo "Pushing benchmark results..."
|
||||
cd gh-pages
|
||||
git config user.name github-actions
|
||||
git config user.email github@users.noreply.github.com
|
||||
git add ./dev/bench/data-${{ matrix.os }}.json
|
||||
git commit -m "Add benchmarks results for ${{ github.sha }}"
|
||||
git push
|
||||
cd ..
|
||||
- name: Push static analysis Cachegrind results
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
id: pushStaticAnalysisCachegrindAttempt1
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push static analysis Cachegrind results (Attempt 2)
|
||||
if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt1.outcome == 'failure'
|
||||
id: pushStaticAnalysisCachegrindAttempt2
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
||||
- name: Push static analysis Cachegrind results (Attempt 2)
|
||||
if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt2.outcome == 'failure'
|
||||
id: pushStaticAnalysisCachegrindAttempt3
|
||||
continue-on-error: true
|
||||
uses: ./.github/workflows/push-results
|
||||
with:
|
||||
repository: ${{ matrix.benchResultsRepo.name }}
|
||||
branch: ${{ matrix.benchResultsRepo.branch }}
|
||||
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
||||
path: "./gh-pages"
|
||||
bench_name: ${{ matrix.bench.title }}
|
||||
bench_tool: "roblox"
|
||||
bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
|
||||
bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
name: Checkout & push results
|
||||
description: Checkout a given repo and push results to GitHub
|
||||
inputs:
|
||||
repository:
|
||||
required: true
|
||||
type: string
|
||||
description: The benchmark results repository to check out
|
||||
branch:
|
||||
required: true
|
||||
type: string
|
||||
description: The benchmark results repository's branch to check out
|
||||
token:
|
||||
required: true
|
||||
type: string
|
||||
description: The GitHub token to use for pushing results
|
||||
path:
|
||||
required: true
|
||||
type: string
|
||||
description: The path to check out the results repository to
|
||||
bench_name:
|
||||
required: true
|
||||
type: string
|
||||
bench_tool:
|
||||
required: true
|
||||
type: string
|
||||
bench_output_file_path:
|
||||
required: true
|
||||
type: string
|
||||
bench_external_data_json_path:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: ${{ inputs.repository }}
|
||||
ref: ${{ inputs.branch }}
|
||||
token: ${{ inputs.token }}
|
||||
path: ${{ inputs.path }}
|
||||
|
||||
- name: Store results
|
||||
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
||||
with:
|
||||
name: ${{ inputs.bench_name }}
|
||||
tool: ${{ inputs.bench_tool }}
|
||||
gh-pages-branch: ${{ inputs.branch }}
|
||||
output-file-path: ${{ inputs.bench_output_file_path }}
|
||||
external-data-json-path: ${{ inputs.bench_external_data_json_path }}
|
||||
|
||||
- name: Push benchmark results
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Pushing benchmark results..."
|
||||
cd gh-pages
|
||||
git config user.name github-actions
|
||||
git config user.email github@users.noreply.github.com
|
||||
git add *.json
|
||||
git commit -m "Add benchmarks results for ${{ github.sha }}"
|
||||
git push
|
||||
cd ..
|
||||
Loading…
Reference in New Issue