benchmark.yml

name: Benchmark of pallet-contracts and pallet-evm

on:
  schedule:
    - cron: '0 0 * * *'
  workflow_dispatch:

env:
  MOONBEAM_ARTIFACT: moonbeam
  MOONBEAM_DIR: moonbeam_release
  MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam
  MOONBEAM_VERSION: version
  BENCHMARK_DIR: stats
  TEST_PARAMS: --instance-count 1 --call-count 2000
  BENCHMARK_URI: https://raw.githubusercontent.com/paritytech/smart-bench/gh-pages

jobs:
  # build_dev_moonbeam:
  #   runs-on: ubuntu-20.04
  #   defaults:
  #     run:
  #       shell: bash
  #   steps:
  #     - name: Clean worker
  #       run: |
  #         sudo rm -rf /usr/share/dotnet
  #         sudo rm -rf /opt/ghc
  #         sudo rm -rf "/usr/local/share/boost"
  #         sudo rm -rf "$AGENT_TOOLSDIRECTORY"

  #     - uses: actions/checkout@v4
  #       with:
  #        sparse-checkout: |
  #         launch/moonbeam.patch
  #        sparse-checkout-cone-mode: false

  #     - name: Download Moonbeam Release
  #       run: |
  #         API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/131808906"
  #         RESPONSE=$(curl -s "${API_URL}")
  #         # Remove control characters causing error while parsing
  #         RESPONSE=$(echo $RESPONSE |  tr -cd '[:print:]')
  #         DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.tarball_url')
  #         RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name')
  #         mkdir ${{ env.MOONBEAM_DIR }}
  #         echo "Download moonbeam release: ${DOWNLOAD_URL}"
  #         curl -L ${DOWNLOAD_URL} | tar -xzv -C ${{ env.MOONBEAM_DIR }}
  #         echo $RELEASE_TAG > ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}

  #     - name: Patch
  #       run: |
  #         # Add Dev RPC support
  #         cd moonbeam_release/*/
  #         patch -p1 < ../../launch/moonbeam.patch

  #     - name: Build
  #       uses: docker://paritytech/ci-unified:bullseye-1.73.0
  #       with:
  #         args: /bin/bash -c "cd moonbeam_release/*/ && cargo build --release"

  #     - uses: actions/upload-artifact@v3
  #       with:
  #         name: ${{ env.MOONBEAM_ARTIFACT }}
  #         path: |
  #           ./${{ env.MOONBEAM_BIN }}
  #           ./${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}
  #         retention-days: 1

  smart_contract_benchmark:
    strategy:
      matrix:
        type: [ink-wasm, sol-wasm, evm]
        contract: [erc20, flipper, triangle-number, storage-read, storage-write]
    env:
      BENCHMARK_FILE: benchmark_${{ matrix.type }}_${{ matrix.contract }}.csv
    # needs: build_dev_moonbeam
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Cache
        uses: Swatinem/rust-cache@v2

      - uses: actions/download-artifact@v3
        with:
          name: ${{ env.MOONBEAM_ARTIFACT }}
          path: ./${{ env.MOONBEAM_DIR }}

      - name: Set Moonbeam Release
        id: moonbeam_release
        run: |
          mkdir bin
          # cp ${{ env.MOONBEAM_BIN }} bin/moonbeam
          # chmod +x bin/moonbeam
          # RELEASE_TAG=$(cat ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }})
          echo "tag=$(echo moon)" >> $GITHUB_OUTPUT

      - name: Download Polkadot-Parachain Release
        id: polkadot_parachain_release
        run: |
          API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest"
          RESPONSE=$(curl -s "${API_URL}")

          # Get the download URL of the release binary from Repo
          DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url')
          RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name')

          curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain
          echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT

      - name: Get Ink contract language verision
        id: ink_version
        uses: ./.github/actions/get-contract-language
        with:
          contracts-directory: './contracts/ink'

      - name: Get Solang contract language verision
        id: solang_version
        uses: ./.github/actions/get-contract-language
        with:
          contracts-directory: './contracts/solidity/wasm'

      - name: Get Solc contract language verision
        id: solc_version
        run: |
          language=""
          # Iterate over each contract in the directory
          for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do
            build_info=$(jq -r '.buildInfo' "$contract")
            build_info_path="./contracts/solidity/evm/contracts/${build_info#../}"
            solc_version=$(jq -r '.solcVersion' "$build_info_path")
            current_language="solc $solc_version"

            # Check if the current language is different from the previous one
            if [ -n "$language" ] && [ "$current_language" != "$language" ]; then
              echo "Error: Different language detected in contract $contract: $language != $current_language"
              exit 1
            fi

            language="$current_language"

            echo "Contract: $contract, Language: $current_language"
          done
          echo "language=$language" >> $GITHUB_OUTPUT

      - name: Display variables
        run: |
          echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }}
          echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }}

      - name: Execute tests
        id: run_smart_bench
        run: |
          cd launch
          ./download-bins.sh
          # overwrite parachains
          cp -f ../bin/* ./bin
          ./build.sh

          STATS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS})
          BLOCKS=$(echo ${STATS} | grep -o 'Total Blocks: [0-9]*' | awk '{print $3}')
          EXTRINSICS=$(echo ${STATS} | grep -o 'Total Extrinsics: [0-9]*' | awk '{print $3}')
          TPS=$(echo ${STATS} | grep -o 'sTPS: [0-9]*' | awk '{print $2}')
          echo "Blocks: ${BLOCKS}"
          echo "Extrinsics: ${EXTRINSICS}"
          echo "TPS: ${TPS}"

          echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT

      - name: Extract Ink benchmark stats
        if: matrix.type == 'ink-wasm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \
          ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}, \
          ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - name: Extract Solang benchmark stats
        if: matrix.type == 'sol-wasm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \
          ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}, \
          ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - name: Extract Solidity benchmark stats
        if: matrix.type == 'evm'
        run: |
          CURRENT_DATE=$(date +"%s")
          # date, parachain tag, contract, contract language, TPS
          echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, \
          ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}, \
          ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE}

      - uses: actions/upload-artifact@v3
        with:
          name: ${{ env.BENCHMARK_FILE }}
          path: ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }}
          retention-days: 1

  collect:
    runs-on: ubuntu-latest
    needs: [smart_contract_benchmark]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.BENCHMARK_DIR }}

      - name: Merge CSV
        run: |
          for file in ${{ env.BENCHMARK_DIR }}/*/*.csv; do
            # Extract contract name
            contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/')
            benchmark_file=bench_${contract_name}.csv
            if [ ! -f ${{ env.BENCHMARK_DIR }}/${benchmark_file} ]; then
              curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file}  ${{ env.BENCHMARK_URI }}/${benchmark_file} || exit 1
            fi
            cat $file >> ${{ env.BENCHMARK_DIR }}/${benchmark_file}
          done

      - name: Generate graph
        run: |
          for file in ${{ env.BENCHMARK_DIR }}/bench_*.csv; do
            contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/')
            pushd stats
            ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/stps_${contract_name}.png
            popd
          done

      - name: Commit benchmark stats
        run: |
          CURRENT_DATE=$(date +"%Y%m%d")
          # Set git config
          git config user.email "paritytech-ci@parity.io"
          git config user.name "paritytech-ci"
          git fetch origin gh-pages
          # saving stats
          mkdir /tmp/stats
          mv ${{ env.BENCHMARK_DIR }}/bench_*.csv /tmp/stats
          mv ${{ env.BENCHMARK_DIR }}/stps_*.png /tmp/stats
          git checkout gh-pages
          mv /tmp/stats/* .
          # Upload files
          git add *.csv *.png --force
          git status
          git commit -m "Updated stats in ${CURRENT_DATE} and pushed to gh-pages"
          git push origin gh-pages --force
          rm -rf .git/ ./*