ci.yml

name: CI

on:
  push:
    branches:
      - 'main'
      - 'hotfix**'
  pull_request:
    types: [opened, reopened, synchronize]
    paths-ignore: # ignore docs only changes since they use a dedicated workflows: docs.yml
      - 'docs/**'
      - 'mithril-explorer/**'
      - '.github/workflows/docs.yml'
    branches-ignore:
      - 'hotfix**' # hotfix are handled by the push trigger

concurrency:
  group: ci-build-test-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-ubuntu-X64:
    runs-on: ubuntu-20.04
    outputs:
      eras: ${{ steps.eras-test-lab.outputs.eras }}
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: 20.04-${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-deb

      # We separate the build in 2 steps as we want to avoid side effects with Rust feature unification.
      - name: Cargo build - Tooling
        shell: bash
        run: |
          cargo build --features portable --release --workspace \
              --exclude mithril-aggregator --exclude mithril-client-cli \
              --exclude mithril-client --exclude mithril-signer \
              --exclude mithril-stm

      - name: Build Mithril workspace & publish artifacts
        uses: ./.github/workflows/actions/build-upload-mithril-artifact
        with:
          binaries-build-args: --bin mithril-aggregator --bin mithril-signer --bin mithril-client --bin mithril-relay --features bundle_openssl,full
          libraries-build-args: --package mithril-stm --package mithril-client --features full

      - name: Build Debian packages
        shell: bash
        run: |
          cargo deb --no-build --package mithril-aggregator
          cargo deb --no-build --package mithril-signer
          cargo deb --no-build --package mithril-client-cli
          cargo deb --no-build --package mithril-relay
      
      - name: Publish Debian packages
        uses: actions/upload-artifact@v3
        with:
          name: mithril-deb-packages-${{ runner.os }}-${{ runner.arch }}
          path: target/debian/*.deb
          if-no-files-found: error

      - name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }})
        uses: actions/upload-artifact@v3
        with:
          name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
          path: target/release/mithril-end-to-end
          if-no-files-found: error

      - name: Prepare test lab eras
        id: eras-test-lab
        run: |
          ERAS=$(./target/release/mithril-aggregator era list --json)
          echo "Test Lab Eras: $ERAS"
          echo "eras=$ERAS" >> $GITHUB_OUTPUT
  
  build:
    strategy:
      fail-fast: false
      matrix:
        os: [ macos-12, windows-latest ]
        
        include:
        # Only build client on windows & mac
        - os: macos-12
          binaries-build-args: --bin mithril-client --features bundle_openssl
          libraries-build-args: --package mithril-stm --package mithril-client --features full
        - os: windows-latest
          # Use `--bins --package <package>` instead of `--bin <package>`, otherwise the 'windows' compatibility
          # hack in mithril common cargo.toml doesn't apply (we have no idea why).
          binaries-build-args: --bins --package mithril-client-cli --features bundle_openssl
          libraries-build-args: --package mithril-stm --package mithril-client --no-default-features --features num-integer-backend,full
    runs-on: ${{ matrix.os }}
    
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
      
      - name: Build Mithril workspace & publish artifacts
        uses: ./.github/workflows/actions/build-upload-mithril-artifact
        with:
          binaries-build-args: ${{ matrix.binaries-build-args }}
          libraries-build-args: ${{ matrix.libraries-build-args }}
          common-build-args: ${{ matrix.common-build-args }}

  build-test-wasm:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}-wasm
          cargo-tools: wasm-pack

      - name: Build 'mithril-client-wasm' library
        shell: bash
        run: |
          cp LICENSE ./mithril-client-wasm/
          wasm-pack build mithril-client-wasm --release --target bundler --out-dir ./pkg --scope mithril-dev --out-name index

      - name: Prepare 'mithril-client-wasm' package
        shell: bash
        run: |
          cp LICENSE ./mithril-client-wasm/
          wasm-pack pack mithril-client-wasm/pkg

      - name: Test 'mithril-client-wasm' - Chrome
        shell: bash
        run: |
          wasm-pack test --headless --chrome mithril-client-wasm --release

      - name: Test 'mithril-client-wasm' - Firefox
        shell: bash
        run: |
          wasm-pack test --headless --firefox mithril-client-wasm --release

      - name: Test 'mithril-client-wasm' - NodeJS
        shell: bash
        run: |
          wasm-pack test --node mithril-client-wasm --release

      - name: Publish Mithril Distribution (WASM)
        uses: actions/upload-artifact@v3
        with:
          name: mithril-distribution-wasm
          path: |
            mithril-client-wasm/**/*-mithril-client-wasm-*.tgz
          if-no-files-found: error
  test:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-22.04, macos-12, windows-latest ]
        
        include:
          - os: ubuntu-22.04
            test-args: --features portable,full --workspace
          # Only test client on windows & mac (since its the only binaries supported for those os for now)
          - os: macos-12
            test-args: --package mithril-client --package mithril-client-cli --features full
          - os: windows-latest
            test-args: --package mithril-client --package mithril-client-cli --features full
    
    runs-on: ${{ matrix.os }}
    
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-nextest
      
      - name: Build tests
        run: cargo nextest run --no-run ${{ matrix.test-args }}

      - name: Run tests
        run: cargo nextest run --profile ci ${{ matrix.test-args }}

      - name: Run doc tests
        run: cargo test --doc ${{ matrix.test-args }}

      - name: Ensure examples build
        run: cargo build --examples ${{ matrix.test-args }}

      - name: Rename junit file to include runner info
        shell: bash
        if: success() || failure()
        run: |
          mv target/nextest/ci/tests-result.junit.xml test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml

      - name: Upload Tests Results
        uses: actions/upload-artifact@v3
        if: success() || failure()
        with:
          name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}
          path: |
            ./test-results-*.xml
  
  check:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-sort clippy-sarif sarif-fmt

      - name: Clippy Check
        if: success() || failure()
        run: |
          cargo clippy \
            --all-features --all-targets --no-deps --message-format=json \
            | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
          
          # Make this step fail if any warning has been found
          if [[ $(cat rust-clippy-results.sarif | jq '.runs[0].results') != "[]" ]]; then
            false
          fi

      - name: Upload clippy analysis results to GitHub
        if: success() || failure()
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: rust-clippy-results.sarif
          wait-for-processing: true

      - name: Cargo fmt
        if: success() || failure()
        shell: bash
        run: cargo fmt --check

      - name: Cargo sort
        if: success() || failure()
        shell: bash
        run: cargo sort -w -c
      
      - name: Dependency & Vulnerabilities Review
        if: github.event_name == 'pull_request'
        uses: actions/dependency-review-action@v3
        with:
          base-ref: ${{ github.event.pull_request.base.sha || 'main' }}
          head-ref: ${{ github.event.pull_request.head.sha || github.ref }}

  run-test-lab:
    runs-on: ubuntu-22.04
    needs: [ build-ubuntu-X64 ]
    strategy:
      fail-fast: false
      matrix:
        era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }}
        run_id: [1,2,3]
        extra_args: [""]

        include:
          - era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }}
            run_id: 3
            extra_args: "--use-p2p-network"
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Download binaries
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
          path: ./bin

      - name: Download rust test runner
        uses: actions/download-artifact@v3
        with:
          name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
          path: ./

      - run: |
          chmod +x ./bin/mithril-aggregator
          chmod +x ./bin/mithril-client
          chmod +x ./bin/mithril-signer
          chmod +x ./bin/mithril-relay
          chmod +x ./mithril-end-to-end
          mkdir artifacts

      - name: Test
        run: ./mithril-end-to-end -vvv --bin-directory ./bin --work-directory=./artifacts --devnet-scripts-directory=./mithril-test-lab/mithril-devnet --mithril-era=${{ matrix.era }} ${{ matrix.extra_args }}

      - name: Upload E2E Tests Artifacts
        if:  ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-era_${{ matrix.era }}-run_id_${{ matrix.run_id }}
          path: |
            ./artifacts/*
            # including node.sock makes the upload fails so exclude them:
            !./artifacts/**/node.sock
            # exclude cardano tools, saving ~50mb of data:
            !./artifacts/devnet/cardano-cli
            !./artifacts/devnet/cardano-node
          if-no-files-found: error
  
  publish-tests-results:
    if: success() || failure()
    runs-on: ubuntu-22.04
    needs: 
      - test
    steps:
      - name: Download Tests Results (${{ runner.os }}-${{ runner.arch }})
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-${{ runner.os }}-${{ runner.arch }}

      - name: Download Tests Results (macOS-X64)
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-macOS-X64

      - name: Download Tests Results (Windows-X64)
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-Windows-X64

      - name: Publish Unit Test Results
        if: success() || failure()
        uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          junit_files: ./**/test-results-*.xml

  docker-mithril:
    runs-on: ubuntu-22.04
    needs:
      - build
      - check
      - test
      - run-test-lab
    strategy:
      fail-fast: false
      matrix:
        project: [ mithril-aggregator, mithril-client-cli, mithril-signer, mithril-relay ]

        include:
          - project: mithril-client-cli
            package: mithril-client
    
    permissions:
      contents: read
      packages: write

    env:
      PUSH_PACKAGES: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref)) }}
      REGISTRY: ghcr.io
      PACKAGE: ${{ github.repository_owner }}/${{ matrix.package != '' && matrix.package || matrix.project }}
      DOCKER_FILE: ./${{ matrix.project }}/Dockerfile.ci
      CONTEXT: .

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.PACKAGE }}
          tags: |
            unstable
            type=raw,value=${{ github.base_ref || github.ref_name }}-{{sha}}

      - name: Download built artifacts
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
          path: ${{ matrix.project }}

      - name: Build and push Docker image
        uses: docker/build-push-action@v3
        with:
          context: ${{ env.CONTEXT }}
          file: ${{ env.DOCKER_FILE }}
          push: ${{ env.PUSH_PACKAGES }}
          tags: ${{ steps.meta.outputs.tags }}

  publish-crate-test:
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        package: [ mithril-stm, mithril-common, mithril-client ]
          
    runs-on: ubuntu-22.04
    needs:
      - build
      - test
      - run-test-lab
      - check
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
        
      - name: Publish package to crates.io
        uses: ./.github/workflows/actions/publish-crate-package
        with:
          dry_run: "true"
          package: ${{ matrix.package }}

  publish-wasm-test:
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        package: [ mithril-client-wasm ]
        include:
          - package: mithril-client-wasm
            scope: mithril-dev
            tag: latest
            access: public
            api_token_secret_name: NPM_API_TOKEN_MITHRIL_CLIENT_WASM
          
    runs-on: ubuntu-22.04
    needs:
      - build-test-wasm
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}-wasm
          cargo-tools: wasm-pack
        
      - name: Publish package to npm
        uses: ./.github/workflows/actions/publish-npm-package
        with:
          dry_run: "true"
          package: ${{ matrix.package }}
          scope: ${{ matrix.scope }}
          access: ${{ matrix.access }}
          api_token: ${{ secrets[matrix.api_token_secret_name] }}

  unstable-release:
    if: vars.PUBLISH_UNSTABLE_RELEASE_IN_CI == 'true' && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref))
    runs-on: ubuntu-22.04
    needs:
      - build
      - build-test-wasm
      - test
      - run-test-lab
      - check
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3
      
      - name: Prepare packaging
        run: mkdir package

      - name: Get short SHA
        id: slug
        run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
      
      - name: Download built artifacts (Linux-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-Linux-X64
          path: ./package-Linux-X64

      - name: Download Debian packages (Linux-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-deb-packages-Linux-X64
          path: ./package

      - name: Download built artifacts (macOS-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-macOS-X64
          path: ./package-macOS-X64

      - name: Download built artifacts (Windows-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-Windows-X64
          path: ./package-Windows-X64

      - name: Prepare distribution package
        uses: ./.github/workflows/actions/prepare-distribution
        with:
          version-name: unstable-${{ steps.slug.outputs.sha8 }}
          download-url-base: ${{ github.server_url }}/${{ github.repository }}/releases/download/unstable
          gpg-secret-key: ${{ secrets.GPG_SECRET_KEY }}
          compatibility-table: '{ "release-mainnet": "⛔", "release-preprod": "⛔", "pre-release-preview": "⛔", "testing-preview": "✔" }'
      
      - name: Update unstable release
        uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          automatic_release_tag: unstable
          prerelease: true
          title: Unstable Development Builds
          files: package/*

      - name: Update unstable release body with release notes addon
        uses: tubone24/update_release@v1.3.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG_NAME: unstable 
        with:
          is_append_body: true
          body_path: ./release-notes-addon.txt

  deploy-testing:
    if: vars.DEPLOY_NETWORKS_IN_CI == 'true' && (github.event_name == 'push' || github.event_name == 'pull_request')
    strategy:
      fail-fast: false
      matrix:
        environment: [ testing-preview ]
        include:
        - environment: testing-preview
          environment_prefix: testing
          cardano_network: preview
          mithril_use_p2p_network: false
          mithril_api_domain: api.mithril.network
          mithril_protocol_parameters: |
            {
              k     = 2422
              m     = 20973
              phi_f = 0.2
            }
          mithril_signers: |
            {
              "1" = {
                type    = "verified",
                pool_id = "",
              },
              "2" = {
                type    = "verified",
                pool_id = "",
              },
              "3" = {
                type    = "verified",
                pool_id = "",
              },
            }
          terraform_backend_bucket: hydra-terraform-admin
          google_region: europe-west1
          google_zone: europe-west1-b
          google_machine_type: e2-highmem-4
          google_compute_instance_boot_disk_size: 200
          google_compute_instance_data_disk_size: 250
    environment: ${{ matrix.environment }}
    runs-on: ubuntu-22.04
    needs:
      - docker-mithril
    defaults:
      run:
        working-directory: mithril-infra
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Get Docker image id
        run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV

      - name: ${{ env.DEPLOY == 'true' && 'Apply' || 'Plan' }} terraform infrastructure
        uses: ./.github/workflows/actions/deploy-terraform-infrastructure
        env:
          DEPLOY: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
        with:
          dry_run: ${{ env.DEPLOY == 'true' && 'false' || 'true' }}
          terraform_backend_bucket: ${{ matrix.terraform_backend_bucket }}
          environment_prefix: ${{ matrix.environment_prefix }}
          environment: ${{ matrix.environment }}
          cardano_network: ${{ matrix.cardano_network }}
          cardano_node_version: ${{ vars.CARDANO_NODE_VERSION }}
          google_region: ${{ matrix.google_region }}
          google_zone: ${{ matrix.google_zone }}
          google_machine_type: ${{ matrix.google_machine_type }}
          google_compute_instance_boot_disk_size: ${{ matrix.google_compute_instance_boot_disk_size }}
          google_compute_instance_data_disk_size: ${{ matrix.google_compute_instance_data_disk_size }}
          google_application_credentials: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
          mithril_use_p2p_network: ${{ matrix.mithril_use_p2p_network }}
          mithril_api_domain: ${{ matrix.mithril_api_domain }}
          mithril_image_id: ${{ env.DOCKER_IMAGE_ID }}
          mithril_protocol_parameters: ${{ toJSON(matrix.mithril_protocol_parameters) }}
          mithril_signers: ${{ toJSON(matrix.mithril_signers) }}
          mithril_genesis_secret_key: ${{ secrets.GENESIS_SECRET_KEY }}
          mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
          mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
          mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
          mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
          mithril_aggregator_signed_entity_types: ${{ vars.AGGREGATOR_SIGNED_ENTITY_TYPES }}
          mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }}
          mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }}
          mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
          mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
          mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
          mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
          prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
          prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
          prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}
          prometheus_ingest_username: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
          prometheus_ingest_password: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
          loki_auth_username: ${{ secrets.LOKI_AUTH_USERNAME }}
          loki_auth_password: ${{ secrets.LOKI_AUTH_PASSWORD }}
          loki_ingest_host: ${{ vars.LOKI_INGEST_HOST }}
          loki_ingest_username: ${{ secrets.LOKI_INGEST_USERNAME }}
          loki_ingest_password: ${{ secrets.LOKI_INGEST_PASSWORD }}