benchmark.yml

name: Schedule based benchmark of pallet-contracts and pallet-evm

on:
  schedule:
    - cron: '0 0 * * *'
  workflow_dispatch:

env:
  MOONBEAM_ARTIFACT: moonbeam
  MOONBEAM_DIR: moonbeam_release
  MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam
  MOONBEAM_VERSION: version
  BENCHMARK_DIR: results

jobs:
  build_dev_moonbeam:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    container:
      image: paritytech/ci-unified:bullseye-1.73.0
    steps:
      - uses: actions/checkout@v4
        with:
         sparse-checkout: |
          launch/moonbeam.patch
         sparse-checkout-cone-mode: false

      - name: Download Moonbeam Release
        run: |
          API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/latest"
          RESPONSE=$(curl -s "${API_URL}")
          # Remove control characters causing error while parsing
          RESPONSE=$(echo $RESPONSE |  tr -cd '[:print:]')
          DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.tarball_url')
          RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name')
          mkdir ${{ env.MOONBEAM_DIR }}
          curl -L ${DOWNLOAD_URL} | tar -xzv -C ${{ env.MOONBEAM_DIR }}
          echo $RELEASE_TAG > ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}

      - name: Cache
        uses: Swatinem/rust-cache@v2

      - name: Build
        run: |
          # Rebuild Moonbeam with Dev RPC support
          cd moonbeam_release/*/
          git apply ../../launch/moonbeam.patch
          cargo build --release

      - uses: actions/upload-artifact@v3
        with:
          name: ${{ env.MOONBEAM_ARTIFACT }}
          path: |
            ./${{ env.MOONBEAM_BIN }}
            ./${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}
          retention-days: 1

  smart_contract_benchmark:
    strategy:
      matrix:
        type: [ink-wasm, sol-wasm, evm]
        contract: [erc20]
    env:
      BENCHMARK_FILE: benchmark-${{ matrix.type }}-${{ matrix.contract }}.csv
    needs: build_dev_moonbeam
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Cache
        uses: Swatinem/rust-cache@v2

      - uses: actions/download-artifact@v3
        with:
          name: ${{ env.MOONBEAM_ARTIFACT }}
          path: ./${{ env.MOONBEAM_DIR }}

      - name: Set Moonbeam Release
        id: moonbeam_release
        run: |
          mkdir bin
          cp ${{ env.MOONBEAM_BIN }} bin/moonbeam
          chmod +x bin/moonbeam
          RELEASE_TAG=$(cat ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }})
          echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT

      - name: Download Polkadot-Parachain Release
        id: polkadot_parachain_release
        run: |
          API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest"
          RESPONSE=$(curl -s "${API_URL}")

          # Get the download URL of the release binary from Repo
          DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url')
          RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name')

          curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain
          echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT

      - name: Get Ink contract language verision
        id: ink_version
        uses: ./.github/actions/get-contract-language
        with:
          contracts-directory: './contracts/ink'

      - name: Get Solang contract language verision
        id: solang_version
        uses: ./.github/actions/get-contract-language
        with:
          contracts-directory: './contracts/solidity/wasm'

      - name: Get Solc contract language verision
        id: solc_version
        run: |
          language=""
          # Iterate over each contract in the directory
          for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do
            build_info=$(jq -r '.buildInfo' "$contract")
            build_info_path="./contracts/solidity/evm/contracts/${build_info#../}"
            solc_version=$(jq -r '.solcVersion' "$build_info_path")
            current_language="solc $solc_version"

            # Check if the current language is different from the previous one
            if [ -n "$language" ] && [ "$current_language" != "$language" ]; then
              echo "Error: Different language detected in contract $contract: $language != $current_language"
              exit 1
            fi

            language="$current_language"

            echo "Contract: $contract, Language: $current_language"
          done
          echo "language=$language" >> $GITHUB_OUTPUT

      - name: Display variables
        run: |
          echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }}
          echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }}

      - name: Execute tests
        id: run_smart_bench
        env:
          TEST_PARAMS: --instance-count 1 --call-count 100
        run: |
          cd launch
          ./download-bins.sh
          # overwrite parachains
          cp ../bin/* ./bin
          ./build.sh

          TPS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS} | grep -o 'TPS: [0-9]*' | awk '{print $2}')
          echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT

      - name: Extract Ink benchmark stats
        if: matrix.type == 'ink-wasm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \
          ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - name: Extract Solang benchmark stats
        if: matrix.type == 'sol-wasm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \
          ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - name: Extract Solidity benchmark stats
        if: matrix.type == 'evm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, ${{ matrix.contract }}, \
          ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - uses: actions/upload-artifact@v3
        with:
          name: ${{ env.BENCHMARK_FILE }}
          path: ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }}
          retention-days: 1

  collect:
    runs-on: ubuntu-latest
    needs: [smart_contract_benchmark]
    env:
      BENCHMARK_FILE: benchmark-results.csv
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.BENCHMARK_DIR }}

      - name: Merge CSV
        run: |
          cat ${{ env.BENCHMARK_DIR }}/*/*.csv >> ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }}

      - name: Generate graph
        run: |
          cd stats
          ./get_graph.sh --panel-id=2 --csv-data=../${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} --output=../${{ env.BENCHMARK_DIR }}/tps.png

      - name: Commit benchmark stats
        run: |
          CURRENT_DATE=$(date +"%Y%m%d")
          git config user.email "paritytech-ci@parity.io"
          git config user.name "paritytech-ci"
          git add ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} ${{ env.BENCHMARK_DIR }}/tps.png
          git commit -m "Add benchmark results on ${CURRENT_DATE}"
          git push