Benchmark of pallet-contracts and pallet-evm #42
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schedule based benchmark of pallet-contracts and pallet-evm | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| env: | |
| MOONBEAM_ARTIFACT: moonbeam | |
| MOONBEAM_DIR: moonbeam_release | |
| MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam | |
| MOONBEAM_VERSION: version | |
| BENCHMARK_DIR: results | |
| TEST_PARAMS: --instance-count 1 --call-count 1000 | |
| jobs: | |
| build_dev_moonbeam: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| container: | |
| image: paritytech/ci-unified:bullseye-1.73.0 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| launch/moonbeam.patch | |
| sparse-checkout-cone-mode: false | |
| - name: Download Moonbeam Release | |
| run: | | |
| API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Remove control characters causing error while parsing | |
| RESPONSE=$(echo $RESPONSE | tr -cd '[:print:]') | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.tarball_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir ${{ env.MOONBEAM_DIR }} | |
| curl -L ${DOWNLOAD_URL} | tar -xzv -C ${{ env.MOONBEAM_DIR }} | |
| echo $RELEASE_TAG > ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }} | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Build | |
| run: | | |
| # Rebuild Moonbeam with Dev RPC support | |
| cd moonbeam_release/*/ | |
| git apply ../../launch/moonbeam.patch | |
| cargo build --release | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| path: | | |
| ./${{ env.MOONBEAM_BIN }} | |
| ./${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }} | |
| retention-days: 1 | |
| smart_contract_benchmark: | |
| strategy: | |
| matrix: | |
| type: [ink-wasm, sol-wasm, evm] | |
| contract: [erc20] | |
| env: | |
| BENCHMARK_FILE: benchmark-${{ matrix.type }}-${{ matrix.contract }}.csv | |
| needs: build_dev_moonbeam | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| path: ./${{ env.MOONBEAM_DIR }} | |
| - name: Set Moonbeam Release | |
| id: moonbeam_release | |
| run: | | |
| mkdir bin | |
| cp ${{ env.MOONBEAM_BIN }} bin/moonbeam | |
| chmod +x bin/moonbeam | |
| RELEASE_TAG=$(cat ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}) | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Download Polkadot-Parachain Release | |
| id: polkadot_parachain_release | |
| run: | | |
| API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Get Ink contract language verision | |
| id: ink_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/ink' | |
| - name: Get Solang contract language verision | |
| id: solang_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/solidity/wasm' | |
| - name: Get Solc contract language verision | |
| id: solc_version | |
| run: | | |
| language="" | |
| # Iterate over each contract in the directory | |
| for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do | |
| build_info=$(jq -r '.buildInfo' "$contract") | |
| build_info_path="./contracts/solidity/evm/contracts/${build_info#../}" | |
| solc_version=$(jq -r '.solcVersion' "$build_info_path") | |
| current_language="solc $solc_version" | |
| # Check if the current language is different from the previous one | |
| if [ -n "$language" ] && [ "$current_language" != "$language" ]; then | |
| echo "Error: Different language detected in contract $contract: $language != $current_language" | |
| exit 1 | |
| fi | |
| language="$current_language" | |
| echo "Contract: $contract, Language: $current_language" | |
| done | |
| echo "language=$language" >> $GITHUB_OUTPUT | |
| - name: Display variables | |
| run: | | |
| echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }} | |
| echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }} | |
| - name: Execute tests | |
| id: run_smart_bench | |
| run: | | |
| cd launch | |
| ./download-bins.sh | |
| # overwrite parachains | |
| cp ../bin/* ./bin | |
| ./build.sh | |
| STATS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS}) | |
| BLOCKS=$(echo ${STATS} | grep -o 'Total Blocks: [0-9]*' | awk '{print $3}') | |
| EXTRINSICS=$(echo ${STATS} | grep -o 'Total Extrinsics: [0-9]*' | awk '{print $3}') | |
| TPS=$(echo ${STATS} | grep -o 'TPS: [0-9]*' | awk '{print $2}') | |
| echo "Blocks: ${BLOCKS}" | |
| echo "Extrinsics: ${EXTRINSICS}" | |
| echo "TPS: ${TPS}" | |
| echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT | |
| - name: Extract Ink benchmark stats | |
| if: matrix.type == 'ink-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - name: Extract Solang benchmark stats | |
| if: matrix.type == 'sol-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - name: Extract Solidity benchmark stats | |
| if: matrix.type == 'evm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} | |
| retention-days: 1 | |
| collect: | |
| runs-on: ubuntu-latest | |
| needs: [smart_contract_benchmark] | |
| env: | |
| BENCHMARK_FILE: benchmark-results.csv | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ${{ env.BENCHMARK_DIR }} | |
| - name: Merge CSV | |
| run: | | |
| cat ${{ env.BENCHMARK_DIR }}/*/*.csv >> ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} | |
| - name: Generate graph | |
| run: | | |
| cd stats | |
| ./get_graph.sh --panel-id=2 --csv-data=../${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} --output=../${{ env.BENCHMARK_DIR }}/tps.png | |
| - name: Commit benchmark stats | |
| run: | | |
| CURRENT_DATE=$(date +"%Y%m%d") | |
| git config user.email "[email protected]" | |
| git config user.name "paritytech-ci" | |
| git add ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} ${{ env.BENCHMARK_DIR }}/tps.png | |
| git commit -m "Add benchmark results on ${CURRENT_DATE}" | |
| git push |