From 8fa2e9c152ced00730687c1dc0405e45060de1d5 Mon Sep 17 00:00:00 2001 From: Wilson Lin Date: Fri, 19 Jun 2020 09:49:48 +1000 Subject: [PATCH] Switch over to B2 for CICD --- .github/workflows/bench.yaml | 15 ++++++++------- .github/workflows/cli.yml | 21 ++++++++++++++------- .github/workflows/nodejs.yml | 34 +++++++++++++++++++--------------- .github/workflows/python.yml | 32 +++++++++++++++++--------------- .github/workflows/ruby.yml | 32 +++++++++++++++++++------------- 5 files changed, 77 insertions(+), 57 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 1bb45d1..37bfc74 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -36,10 +36,11 @@ jobs: - name: Run bench and graph results working-directory: ./bench run: ./bench.sh - - uses: chrislennon/action-aws-cli@v1.1 - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 - run: aws s3 cp ./bench/results/ s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bench/${{ steps.version.outputs.VERSION }}/ --recursive + - name: Install B2 CLI + run: | + pip install setuptools + pip install --upgrade b2 + - name: Upload to B2 + run: | + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 sync ./bench/results/ b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/hyperbuild/bench/${{ steps.version.outputs.VERSION }}/ diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index c22565f..4560037 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -41,11 +41,18 @@ jobs: profile: minimal default: true - name: Build CLI + working-directory: ./cli run: cargo build --release - - uses: chrislennon/action-aws-cli@v1.1 - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 - run: aws s3 cp ./target/release/hyperbuild${{ matrix.EXT }} s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/${{ steps.file.outputs.FILE }} --content-type ${{ matrix.MIME }} + - name: Install B2 CLI (macOS, Linux) + if: runner.os != 'Windows' + run: | + sudo pip install setuptools + sudo pip install --upgrade b2 + - name: Install B2 CLI (Windows) + if: runner.os == 'Windows' + run: | + pip install --upgrade b2 + - name: Upload to B2 + run: | + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 upload-file --contentType ${{ matrix.MIME }} ${{ secrets.CICD_CLI_B2_BUCKET_NAME }} ./cli/target/release/hyperbuild${{ matrix.EXT }} hyperbuild/bin/${{ steps.file.outputs.FILE }} diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index c39080c..b912b7a 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -12,7 +12,7 @@ jobs: matrix: # Use windows-2016 as windows-2019 has LNK1181 error when building Neon module. os: [ubuntu-latest, windows-2016, macos-latest] - node: [8, 9, 10, 11, 12, 13] + node: [8, 9, 10, 11, 12, 13, 14] include: - os: ubuntu-latest ARCH: linux-x86_64 @@ -41,13 +41,19 @@ jobs: run: | npm install node node_modules/neon-cli/bin/cli.js build --release - - uses: chrislennon/action-aws-cli@v1.1 - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 - run: aws s3 cp ./nodejs/native/index.node s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/nodejs/${{ steps.version.outputs.VERSION }}/${{ matrix.ARCH }}-node${{ matrix.node }}.node + - name: Install B2 CLI (macOS, Linux) + if: runner.os != 'Windows' + run: | + sudo pip install setuptools + sudo pip install --upgrade b2 + - name: Install B2 CLI (Windows) + if: runner.os == 'Windows' + run: | + pip install --upgrade b2 + - name: Upload to B2 + run: | + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 upload-file --contentType ${{ matrix.MIME }} ${{ secrets.CICD_CLI_B2_BUCKET_NAME }} ./nodejs/native/index.node hyperbuild/bin/nodejs/${{ steps.version.outputs.VERSION }}/${{ matrix.ARCH }}-node${{ matrix.node }}.node package: runs-on: ubuntu-latest needs: build @@ -60,14 +66,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 13.x - - uses: chrislennon/action-aws-cli@v1.1 + node-version: 14.x + - name: Install B2 CLI + run: pip install --upgrade b2 - name: Pack and publish package working-directory: ./nodejs - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 run: | cat << 'EOF' > .npmrc package-lock=false @@ -76,6 +79,7 @@ jobs: npm install rm -rf dist node node_modules/typescript/bin/tsc - aws s3 mv s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/nodejs/${{ steps.version.outputs.VERSION }}/ ./dist/. --recursive + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 sync b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/hyperbuild/bin/nodejs/${{ steps.version.outputs.VERSION }}/ ./dist/. cp ../README.md . npm publish diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2148e99..00b9c48 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -52,14 +52,20 @@ jobs: - name: Build native module working-directory: ./python run: cargo build --release - - uses: chrislennon/action-aws-cli@v1.1 - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + - name: Install B2 CLI (macOS, Linux) + if: runner.os != 'Windows' + run: | + sudo pip install setuptools + sudo pip install --upgrade b2 + - name: Install B2 CLI (Windows) + if: runner.os == 'Windows' + run: | + pip install --upgrade b2 + - name: Upload to B2 # Convert dots to underscores in Python version as Python uses dots to represent import hierarchy. - run: aws s3 cp ./python/target/release/${{ matrix.LIBFILE }} s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/python/${{ steps.version.outputs.VERSION }}/${{ steps.native_file.outputs.NAME }} + run: | + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 upload-file --contentType ${{ matrix.MIME }} ${{ secrets.CICD_CLI_B2_BUCKET_NAME }} ./python/target/release/${{ matrix.LIBFILE }} hyperbuild/bin/python/${{ steps.version.outputs.VERSION }}/${{ steps.native_file.outputs.NAME }} package: runs-on: ubuntu-latest needs: build @@ -75,19 +81,14 @@ jobs: python-version: '3.5' architecture: 'x64' - name: Install Python dependencies - # chrislennon/action-aws-cli@v1.1 interferes with installed Python libraries. - # Installing using pip seems to be more reliable when done individually. run: | - pip install --upgrade awscli pip install --upgrade wheel pip install --upgrade setuptools pip install --upgrade twine + - name: Install B2 CLI + run: pip install --upgrade b2 - name: Pack and publish package working-directory: ./python - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 run: | cat << 'EOF' > "$HOME/.pypirc" [pypi] @@ -95,6 +96,7 @@ jobs: password = ${{ secrets.PYPI_API_TOKEN }} EOF cp ../README.md . - aws s3 mv s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/python/${{ steps.version.outputs.VERSION }}/ ./hyperbuild/. --recursive + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 sync b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/hyperbuild/bin/python/${{ steps.version.outputs.VERSION }}/ ./hyperbuild/. python setup.py sdist bdist_wheel python -m twine upload dist/* diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 7810024..c32c731 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -67,13 +67,19 @@ jobs: shell: bash -l {0} run: cargo build --release working-directory: ./ruby - - uses: chrislennon/action-aws-cli@v1.1 - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 - run: aws s3 cp ./ruby/target/release/${{ matrix.FILE }} s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/ruby/${{ steps.version.outputs.VERSION }}/${{ steps.native_file.outputs.NAME }} + - name: Install B2 CLI (macOS, Linux) + if: runner.os != 'Windows' + run: | + sudo pip install setuptools + sudo pip install --upgrade b2 + - name: Install B2 CLI (Windows) + if: runner.os == 'Windows' + run: | + pip install --upgrade b2 + - name: Upload to B2 + run: | + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 upload-file --contentType ${{ matrix.MIME }} ${{ secrets.CICD_CLI_B2_BUCKET_NAME }} ./ruby/target/release/${{ matrix.FILE }} hyperbuild/bin/ruby/${{ steps.version.outputs.VERSION }}/${{ steps.native_file.outputs.NAME }} package: runs-on: ubuntu-latest needs: build @@ -87,16 +93,16 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: '2.5' - - uses: chrislennon/action-aws-cli@v1.1 + - name: Install B2 CLI + run: | + pip install setuptools + pip install --upgrade b2 - name: Pack and publish gem working-directory: ./ruby - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 run: | cp ../README.md . - aws s3 mv s3://${{ secrets.AWS_S3_BUCKET }}/hyperbuild/bin/ruby/${{ steps.version.outputs.VERSION }}/ ./lib/. --recursive + b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }} + b2 sync b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/hyperbuild/bin/ruby/${{ steps.version.outputs.VERSION }}/ ./lib/. gem build hyperbuild.gemspec mkdir -p "$HOME/.gem" cat << 'EOF' > "$HOME/.gem/credentials"