Skip to content

get datums using pallas observer #29

get datums using pallas observer

get datums using pallas observer #29

Workflow file for this run

name: CI
on:
push:
branches:
- 'main'
- 'hotfix**'
pull_request:
types: [opened, reopened, synchronize]
paths-ignore: # ignore docs only changes since they use a dedicated workflows: docs.yml
- 'docs/**'
- 'mithril-explorer/**'
- '.github/workflows/docs.yml'
branches-ignore:
- 'hotfix**' # hotfix are handled by the push trigger
concurrency:
group: ci-build-test-${{ github.ref }}
cancel-in-progress: true
jobs:
build-ubuntu-X64:
runs-on: ubuntu-20.04
outputs:
eras: ${{ steps.eras-test-lab.outputs.eras }}
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain, tools, and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: 20.04-${{ secrets.CACHE_VERSION }}
cargo-tools: cargo-deb
# We separate the build in 2 steps as we want to avoid side effects with Rust feature unification.
- name: Cargo build - Tooling
shell: bash
run: |
cargo build --features portable --release --workspace \
--exclude mithril-aggregator --exclude mithril-client-cli \
--exclude mithril-client --exclude mithril-signer \
--exclude mithril-stm
- name: Build Mithril workspace & publish artifacts
uses: ./.github/workflows/actions/build-upload-mithril-artifact
with:
binaries-build-args: --bin mithril-aggregator --bin mithril-signer --bin mithril-client --bin mithril-relay --features bundle_openssl,full
libraries-build-args: --package mithril-stm --package mithril-client --features full
- name: Build Debian packages
shell: bash
run: |
cargo deb --no-build --package mithril-aggregator
cargo deb --no-build --package mithril-signer
cargo deb --no-build --package mithril-client-cli
cargo deb --no-build --package mithril-relay
- name: Publish Debian packages
uses: actions/upload-artifact@v3
with:
name: mithril-deb-packages-${{ runner.os }}-${{ runner.arch }}
path: target/debian/*.deb
if-no-files-found: error
- name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }})
uses: actions/upload-artifact@v3
with:
name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
path: target/release/mithril-end-to-end
if-no-files-found: error
- name: Prepare test lab eras
id: eras-test-lab
run: |
ERAS=$(./target/release/mithril-aggregator era list --json)
echo "Test Lab Eras: $ERAS"
echo "eras=$ERAS" >> $GITHUB_OUTPUT
build:
strategy:
fail-fast: false
matrix:
os: [ macos-12, windows-latest ]
include:
# Only build client on windows & mac
- os: macos-12
binaries-build-args: --bin mithril-client --features bundle_openssl
libraries-build-args: --package mithril-stm --package mithril-client --features full
- os: windows-latest
# Use `--bins --package <package>` instead of `--bin <package>`, otherwise the 'windows' compatibility
# hack in mithril common cargo.toml doesn't apply (we have no idea why).
binaries-build-args: --bins --package mithril-client-cli --features bundle_openssl
libraries-build-args: --package mithril-stm --package mithril-client --no-default-features --features num-integer-backend,full
runs-on: ${{ matrix.os }}
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: ${{ secrets.CACHE_VERSION }}
- name: Build Mithril workspace & publish artifacts
uses: ./.github/workflows/actions/build-upload-mithril-artifact
with:
binaries-build-args: ${{ matrix.binaries-build-args }}
libraries-build-args: ${{ matrix.libraries-build-args }}
common-build-args: ${{ matrix.common-build-args }}
build-test-wasm:
runs-on: ubuntu-22.04
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain, tools, and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: ${{ secrets.CACHE_VERSION }}-wasm
cargo-tools: wasm-pack
- name: Build 'mithril-client-wasm' library
shell: bash
run: |
cp LICENSE ./mithril-client-wasm/
wasm-pack build mithril-client-wasm --release --target bundler --out-dir ./pkg --scope mithril-dev --out-name index
- name: Prepare 'mithril-client-wasm' package
shell: bash
run: |
cp LICENSE ./mithril-client-wasm/
wasm-pack pack mithril-client-wasm/pkg
- name: Test 'mithril-client-wasm' - Chrome
shell: bash
run: |
wasm-pack test --headless --chrome mithril-client-wasm --release
- name: Test 'mithril-client-wasm' - Firefox
shell: bash
run: |
wasm-pack test --headless --firefox mithril-client-wasm --release
- name: Test 'mithril-client-wasm' - NodeJS
shell: bash
run: |
wasm-pack test --node mithril-client-wasm --release
- name: Publish Mithril Distribution (WASM)
uses: actions/upload-artifact@v3
with:
name: mithril-distribution-wasm
path: |
mithril-client-wasm/**/*-mithril-client-wasm-*.tgz
if-no-files-found: error
test:
strategy:
fail-fast: false
matrix:
os: [ ubuntu-22.04, macos-12, windows-latest ]
include:
- os: ubuntu-22.04
test-args: --features portable,full --workspace
# Only test client on windows & mac (since its the only binaries supported for those os for now)
- os: macos-12
test-args: --package mithril-client --package mithril-client-cli --features full
- os: windows-latest
test-args: --package mithril-client --package mithril-client-cli --features full
runs-on: ${{ matrix.os }}
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain, tools, and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: ${{ secrets.CACHE_VERSION }}
cargo-tools: cargo-nextest
- name: Build tests
run: cargo nextest run --no-run ${{ matrix.test-args }}
- name: Run tests
run: cargo nextest run --profile ci ${{ matrix.test-args }}
- name: Run doc tests
run: cargo test --doc ${{ matrix.test-args }}
- name: Ensure examples build
run: cargo build --examples ${{ matrix.test-args }}
- name: Rename junit file to include runner info
shell: bash
if: success() || failure()
run: |
mv target/nextest/ci/tests-result.junit.xml test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml
- name: Upload Tests Results
uses: actions/upload-artifact@v3
if: success() || failure()
with:
name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}
path: |
./test-results-*.xml
check:
runs-on: ubuntu-22.04
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain, tools, and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: ${{ secrets.CACHE_VERSION }}
cargo-tools: cargo-sort clippy-sarif sarif-fmt
- name: Clippy Check
if: success() || failure()
run: |
cargo clippy \
--all-features --all-targets --no-deps --message-format=json \
| clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
# Make this step fail if any warning has been found
if [[ $(cat rust-clippy-results.sarif | jq '.runs[0].results') != "[]" ]]; then
false
fi
- name: Upload clippy analysis results to GitHub
if: success() || failure()
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: rust-clippy-results.sarif
wait-for-processing: true
- name: Cargo fmt
if: success() || failure()
shell: bash
run: cargo fmt --check
- name: Cargo sort
if: success() || failure()
shell: bash
run: cargo sort -w -c
- name: Dependency & Vulnerabilities Review
if: github.event_name == 'pull_request'
uses: actions/dependency-review-action@v3
with:
base-ref: ${{ github.event.pull_request.base.sha || 'main' }}
head-ref: ${{ github.event.pull_request.head.sha || github.ref }}
run-test-lab:
runs-on: ubuntu-22.04
needs: [ build-ubuntu-X64 ]
strategy:
fail-fast: false
matrix:
era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }}
run_id: [1,2,3]
extra_args: [""]
include:
- era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }}
run_id: 3
extra_args: "--use-p2p-network"
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Download binaries
uses: actions/download-artifact@v3
with:
name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
path: ./bin
- name: Download rust test runner
uses: actions/download-artifact@v3
with:
name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
path: ./
- run: |
chmod +x ./bin/mithril-aggregator
chmod +x ./bin/mithril-client
chmod +x ./bin/mithril-signer
chmod +x ./bin/mithril-relay
chmod +x ./mithril-end-to-end
mkdir artifacts
- name: Test
run: ./mithril-end-to-end -vvv --bin-directory ./bin --work-directory=./artifacts --devnet-scripts-directory=./mithril-test-lab/mithril-devnet --mithril-era=${{ matrix.era }} ${{ matrix.extra_args }}
- name: Upload E2E Tests Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-era_${{ matrix.era }}-run_id_${{ matrix.run_id }}
path: |
./artifacts/*
# including node.sock makes the upload fails so exclude them:
!./artifacts/**/node.sock
# exclude cardano tools, saving ~50mb of data:
!./artifacts/devnet/cardano-cli
!./artifacts/devnet/cardano-node
if-no-files-found: error
publish-tests-results:
if: success() || failure()
runs-on: ubuntu-22.04
needs:
- test
steps:
- name: Download Tests Results (${{ runner.os }}-${{ runner.arch }})
if: success() || failure()
uses: actions/download-artifact@v3
with:
name: test-results-${{ runner.os }}-${{ runner.arch }}
- name: Download Tests Results (macOS-X64)
if: success() || failure()
uses: actions/download-artifact@v3
with:
name: test-results-macOS-X64
- name: Download Tests Results (Windows-X64)
if: success() || failure()
uses: actions/download-artifact@v3
with:
name: test-results-Windows-X64
- name: Publish Unit Test Results
if: success() || failure()
uses: EnricoMi/publish-unit-test-result-action@v2
with:
junit_files: ./**/test-results-*.xml
docker-mithril:
runs-on: ubuntu-22.04
needs:
- build
- check
- test
- run-test-lab
strategy:
fail-fast: false
matrix:
project: [ mithril-aggregator, mithril-client-cli, mithril-signer, mithril-relay ]
include:
- project: mithril-client-cli
package: mithril-client
permissions:
contents: read
packages: write
env:
PUSH_PACKAGES: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref)) }}
REGISTRY: ghcr.io
PACKAGE: ${{ github.repository_owner }}/${{ matrix.package != '' && matrix.package || matrix.project }}
DOCKER_FILE: ./${{ matrix.project }}/Dockerfile.ci
CONTEXT: .
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY }}/${{ env.PACKAGE }}
tags: |
unstable
type=raw,value=${{ github.base_ref || github.ref_name }}-{{sha}}
- name: Download built artifacts
uses: actions/download-artifact@v3
with:
name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
path: ${{ matrix.project }}
- name: Build and push Docker image
uses: docker/build-push-action@v3
with:
context: ${{ env.CONTEXT }}
file: ${{ env.DOCKER_FILE }}
push: ${{ env.PUSH_PACKAGES }}
tags: ${{ steps.meta.outputs.tags }}
publish-crate-test:
strategy:
fail-fast: false
max-parallel: 1
matrix:
package: [ mithril-stm, mithril-common, mithril-client ]
runs-on: ubuntu-22.04
needs:
- build
- test
- run-test-lab
- check
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Publish package to crates.io
uses: ./.github/workflows/actions/publish-crate-package
with:
dry_run: "true"
package: ${{ matrix.package }}
publish-wasm-test:
strategy:
fail-fast: false
max-parallel: 1
matrix:
package: [ mithril-client-wasm ]
include:
- package: mithril-client-wasm
scope: mithril-dev
tag: latest
access: public
api_token_secret_name: NPM_API_TOKEN_MITHRIL_CLIENT_WASM
runs-on: ubuntu-22.04
needs:
- build-test-wasm
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain, tools, and restore cache
uses: ./.github/workflows/actions/toolchain-and-cache
with:
cache-version: ${{ secrets.CACHE_VERSION }}-wasm
cargo-tools: wasm-pack
- name: Publish package to npm
uses: ./.github/workflows/actions/publish-npm-package
with:
dry_run: "true"
package: ${{ matrix.package }}
scope: ${{ matrix.scope }}
access: ${{ matrix.access }}
api_token: ${{ secrets[matrix.api_token_secret_name] }}
unstable-release:
if: vars.PUBLISH_UNSTABLE_RELEASE_IN_CI == 'true' && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref))
runs-on: ubuntu-22.04
needs:
- build
- build-test-wasm
- test
- run-test-lab
- check
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Prepare packaging
run: mkdir package
- name: Get short SHA
id: slug
run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
- name: Download built artifacts (Linux-X64)
uses: actions/download-artifact@v3
with:
name: mithril-distribution-Linux-X64
path: ./package-Linux-X64
- name: Download Debian packages (Linux-X64)
uses: actions/download-artifact@v3
with:
name: mithril-deb-packages-Linux-X64
path: ./package
- name: Download built artifacts (macOS-X64)
uses: actions/download-artifact@v3
with:
name: mithril-distribution-macOS-X64
path: ./package-macOS-X64
- name: Download built artifacts (Windows-X64)
uses: actions/download-artifact@v3
with:
name: mithril-distribution-Windows-X64
path: ./package-Windows-X64
- name: Prepare distribution package
uses: ./.github/workflows/actions/prepare-distribution
with:
version-name: unstable-${{ steps.slug.outputs.sha8 }}
download-url-base: ${{ github.server_url }}/${{ github.repository }}/releases/download/unstable
gpg-secret-key: ${{ secrets.GPG_SECRET_KEY }}
compatibility-table: '{ "release-mainnet": "⛔", "release-preprod": "⛔", "pre-release-preview": "⛔", "testing-preview": "✔" }'
- name: Update unstable release
uses: marvinpinto/action-automatic-releases@latest
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
automatic_release_tag: unstable
prerelease: true
title: Unstable Development Builds
files: package/*
- name: Update unstable release body with release notes addon
uses: tubone24/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG_NAME: unstable
with:
is_append_body: true
body_path: ./release-notes-addon.txt
deploy-testing:
if: vars.DEPLOY_NETWORKS_IN_CI == 'true' && (github.event_name == 'push' || github.event_name == 'pull_request')
strategy:
fail-fast: false
matrix:
environment: [ testing-preview ]
include:
- environment: testing-preview
environment_prefix: testing
cardano_network: preview
mithril_use_p2p_network: false
mithril_api_domain: api.mithril.network
mithril_protocol_parameters: |
{
k = 2422
m = 20973
phi_f = 0.2
}
mithril_signers: |
{
"1" = {
type = "verified",
pool_id = "",
},
"2" = {
type = "verified",
pool_id = "",
},
"3" = {
type = "verified",
pool_id = "",
},
}
terraform_backend_bucket: hydra-terraform-admin
google_region: europe-west1
google_zone: europe-west1-b
google_machine_type: e2-highmem-4
google_compute_instance_boot_disk_size: 200
google_compute_instance_data_disk_size: 250
environment: ${{ matrix.environment }}
runs-on: ubuntu-22.04
needs:
- docker-mithril
defaults:
run:
working-directory: mithril-infra
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Get Docker image id
run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV
- name: ${{ env.DEPLOY == 'true' && 'Apply' || 'Plan' }} terraform infrastructure
uses: ./.github/workflows/actions/deploy-terraform-infrastructure
env:
DEPLOY: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
with:
dry_run: ${{ env.DEPLOY == 'true' && 'false' || 'true' }}
terraform_backend_bucket: ${{ matrix.terraform_backend_bucket }}
environment_prefix: ${{ matrix.environment_prefix }}
environment: ${{ matrix.environment }}
cardano_network: ${{ matrix.cardano_network }}
cardano_node_version: ${{ vars.CARDANO_NODE_VERSION }}
google_region: ${{ matrix.google_region }}
google_zone: ${{ matrix.google_zone }}
google_machine_type: ${{ matrix.google_machine_type }}
google_compute_instance_boot_disk_size: ${{ matrix.google_compute_instance_boot_disk_size }}
google_compute_instance_data_disk_size: ${{ matrix.google_compute_instance_data_disk_size }}
google_application_credentials: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
mithril_use_p2p_network: ${{ matrix.mithril_use_p2p_network }}
mithril_api_domain: ${{ matrix.mithril_api_domain }}
mithril_image_id: ${{ env.DOCKER_IMAGE_ID }}
mithril_protocol_parameters: ${{ toJSON(matrix.mithril_protocol_parameters) }}
mithril_signers: ${{ toJSON(matrix.mithril_signers) }}
mithril_genesis_secret_key: ${{ secrets.GENESIS_SECRET_KEY }}
mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
mithril_aggregator_signed_entity_types: ${{ vars.AGGREGATOR_SIGNED_ENTITY_TYPES }}
mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }}
mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }}
mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}
prometheus_ingest_username: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
prometheus_ingest_password: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
loki_auth_username: ${{ secrets.LOKI_AUTH_USERNAME }}
loki_auth_password: ${{ secrets.LOKI_AUTH_PASSWORD }}
loki_ingest_host: ${{ vars.LOKI_INGEST_HOST }}
loki_ingest_username: ${{ secrets.LOKI_INGEST_USERNAME }}
loki_ingest_password: ${{ secrets.LOKI_INGEST_PASSWORD }}