From 4326696f73a60d859a7a1b24817e338662a12aee Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 10:27:28 +0200 Subject: [PATCH 01/54] More univeral lcov prunning --- .github/workflows/solidity-foundry.yml | 219 ++++++++++++++---- contracts/scripts/ccip_lcov_prune | 29 --- .../scripts/ci/generate_slither_report.sh | 80 +++++++ contracts/scripts/ci/modify_remappings.sh | 28 +++ contracts/scripts/ci/select_solc_version.sh | 114 +++++++++ contracts/scripts/lcov_prune | 77 ++++++ 6 files changed, 470 insertions(+), 77 deletions(-) delete mode 100755 contracts/scripts/ccip_lcov_prune create mode 100755 contracts/scripts/ci/generate_slither_report.sh create mode 100755 contracts/scripts/ci/modify_remappings.sh create mode 100755 contracts/scripts/ci/select_solc_version.sh create mode 100755 contracts/scripts/lcov_prune diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 4ec9e424471..1360d8de027 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -11,28 +11,55 @@ jobs: name: Detect changes runs-on: ubuntu-latest outputs: - changes: ${{ steps.changes.outputs.src }} + src_changes: ${{ steps.changes.outputs.src }} + sol_changes: ${{ steps.changes.outputs.sol }} + sol_files: ${{ steps.changes.outputs.sol_files }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes with: + list-files: 'shell' filters: | src: - 'contracts/src/v0.8/**/*' - '.github/workflows/solidity-foundry.yml' - 'contracts/foundry.toml' - 'contracts/gas-snapshots/*.gas-snapshot' + sol: + - 'contracts/src/v0.8/**/*.sol' tests: + if: needs.changes.outputs.src_changes == 'true' strategy: fail-fast: false matrix: - product: [automation, ccip, functions, keystone, l2ep, liquiditymanager, llo-feeds, operatorforwarder, shared, vrf] + product: + # 98.5 is the aspirational code coverage once we migrate all tests to Foundry + # product that have that minimum coverage of 98.5 in the matrix below are currently excluded from the check + - name: automation + min-coverage: 98.5 + - name: ccip + min-coverage: 98.5 + - name: functions + min-coverage: 98.5 + - name: keystone + min-coverage: 74.1 + - name: l2ep + min-coverage: 65.6 + - name: liquiditymanager + min-coverage: 46.3 + - name: llo-feeds + min-coverage: 49.3 + - name: operatorforwarder + min-coverage: 55.7 + - name: shared + min-coverage: 37.1 + - name: vrf + min-coverage: 98.5 needs: [changes] - name: Foundry Tests ${{ matrix.product }} - # See https://github.com/foundry-rs/foundry/issues/3827 + name: Foundry Tests ${{ matrix.product.name }} runs-on: ubuntu-22.04 # The if statements for steps after checkout repo is workaround for @@ -47,127 +74,223 @@ jobs: # and not native Foundry. This is to make sure the dependencies # stay in sync. - name: Setup NodeJS - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.src_changes == 'true' uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.src_changes == 'true' uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge build - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.src_changes == 'true' run: | forge --version forge build id: build working-directory: contracts env: - FOUNDRY_PROFILE: ${{ matrix.product }} + FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge tests - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.src_changes == 'true' run: | forge test -vvv id: test working-directory: contracts env: - FOUNDRY_PROFILE: ${{ matrix.product }} + FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge snapshot - if: ${{ !contains(fromJson('["vrf"]'), matrix.product) && !contains(fromJson('["automation"]'), matrix.product) && !contains(fromJson('["keystone"]'), matrix.product) && needs.changes.outputs.changes == 'true' }} + if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["keystone"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} run: | - forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product }}.gas-snapshot + forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot id: snapshot working-directory: contracts env: - FOUNDRY_PROFILE: ${{ matrix.product }} + FOUNDRY_PROFILE: ${{ matrix.product.name }} - - name: Run coverage - if: ${{ contains(fromJson('["ccip"]'), matrix.product) && needs.changes.outputs.changes == 'true' }} + # required for code coverage report generation + - name: Setup LCOV + if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 + + - name: Run coverage for ${{ matrix.product.name }} + if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} working-directory: contracts run: forge coverage --report lcov env: - FOUNDRY_PROFILE: ${{ matrix.product }} + FOUNDRY_PROFILE: ${{ matrix.product.name }} - - name: Prune report - if: ${{ contains(fromJson('["ccip"]'), matrix.product) && needs.changes.outputs.changes == 'true' }} + - name: Prune lcov report + if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} run: | sudo apt-get install lcov - ./contracts/scripts/ccip_lcov_prune ./contracts/lcov.info ./lcov.info.pruned + ./contracts/scripts/lcov_prune ${{ matrix.product.name }} ./contracts/lcov.info ./contracts/lcov.info.pruned - - name: Report code coverage - if: ${{ contains(fromJson('["ccip"]'), matrix.product) && needs.changes.outputs.changes == 'true' }} + - name: Report code coverage for ${{ matrix.product.name }} + if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: update-comment: true - coverage-files: lcov.info.pruned - minimum-coverage: 98.5 - artifact-name: code-coverage-report + coverage-files: ./contracts/lcov.info.pruned + minimum-coverage: ${{ matrix.product.min-coverage }} + artifact-name: code-coverage-report-${{ matrix.product.name }} working-directory: ./contracts github-token: ${{ secrets.GITHUB_TOKEN }} - name: Collect Metrics - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.src_changes == 'true' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: - id: solidity-foundry + id: ${{ matrix.product.name }}-solidity-foundry org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Foundry Tests ${{ matrix.product }} + this-job-name: Foundry Tests ${{ matrix.product.name }} continue-on-error: true - solidity-forge-fmt: - strategy: - fail-fast: false - matrix: - product: [ ccip ] + analyze: needs: [ changes ] - name: Forge fmt ${{ matrix.product }} - # See https://github.com/foundry-rs/foundry/issues/3827 + name: Run static analysis + if: needs.changes.outputs.sol_changes == 'true' runs-on: ubuntu-22.04 + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive - # The if statements for steps after checkout repo is workaround for - # passing required check for PRs that don't have filtered changes. + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + + - name: Install semver + shell: bash + run: | + npm install -g semver + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 + with: + version: ${{ env.FOUNDRY_VERSION }} + + - name: Set up Python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f #v5.1.1 + with: + python-version: '3.8' + + - name: Install solc-select and solc + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install solc-select + sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select + solc-select install 0.8.19 + solc-select use 0.8.19 + + - name: Install Slither + run: | + python -m pip install --upgrade pip + pip install slither-analyzer + + - name: Run Slither + shell: bash + env: + FOUNDRY_PROFILE: ${{ matrix.product.name }} + run: | + CHANGED="${{ needs.changes.outputs.sol_files }}" + CHANGED_ARRAY=$(echo "$CHANGED" | tr ' ' ',') + + # modify remappings so that solc can find dependencies + ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt + mv remappings_modified.txt remappings.txt + + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-artifacts.json "." "$CHANGED_ARRAY" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + + - name: Print Slither summary + shell: bash + run: | + echo "# Static analysis results " >> $GITHUB_STEP_SUMMARY + for file in "contracts/slither-reports"/*.md; do + if [ -e "$file" ]; then + cat "$file" >> $GITHUB_STEP_SUMMARY + fi + done + + - name: Upload Slither report + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 10 + continue-on-error: true + with: + name: slither-reports-${{ github.sha }} + path: | + contracts/slither-reports + retention-days: 7 + + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 + with: + id: solidity-foundry-slither + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Run static analysis + continue-on-error: true + + fmt: + needs: [ changes ] + name: Check formatting for modified files + if: needs.changes.outputs.sol_changes == 'true' + runs-on: ubuntu-22.04 steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - # Only needed because we use the NPM versions of packages - # and not native Foundry. This is to make sure the dependencies - # stay in sync. - name: Setup NodeJS - if: needs.changes.outputs.changes == 'true' uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: needs.changes.outputs.changes == 'true' uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - if: needs.changes.outputs.changes == 'true' + env: + FOUNDRY_PROFILE: ${{ matrix.product.name }} run: | - forge fmt --check + SOL_FILES="${{ needs.changes.outputs.sol_files }}" + FILE_LIST=$(echo "$SOL_FILES" | tr ' ' '\n') + format_errors=0 + for file in $FILE_LIST; do + sanitized_file="${file/contracts\//}" + PROFILE=$(echo "$sanitized_file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) + if [ -z "$PROFILE" ]; then + PROFILE="${{ env.FOUNDRY_PROFILE }}" + fi + echo "::debug::Running forge fmt $sanitized_file with FOUNDRY_PROFILE=$PROFILE" + FOUNDRY_PROFILE=$PROFILE forge fmt --check "$sanitized_file" || format_errors=$((format_errors+1)) + done + if [ $format_errors -ne 0 ]; then + echo "There were $format_errors formatting errors." + exit 1 + else + echo "All files are properly formatted." + fi id: fmt working-directory: contracts - env: - FOUNDRY_PROFILE: ${{ matrix.product }} - name: Collect Metrics - if: needs.changes.outputs.changes == 'true' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: - id: solidity-forge-fmt + id: solidity-foundry-fmt org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Foundry Tests ${{ matrix.product }} + this-job-name: Foundry Tests fmt continue-on-error: true diff --git a/contracts/scripts/ccip_lcov_prune b/contracts/scripts/ccip_lcov_prune deleted file mode 100755 index 002e5a3f133..00000000000 --- a/contracts/scripts/ccip_lcov_prune +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# src/v0.8/ccip/libraries/Internal.sol -# src/v0.8/ccip/libraries/RateLimiter.sol -# src/v0.8/ccip/libraries/USDPriceWith18Decimals.sol -# src/v0.8/ccip/libraries/MerkleMultiProof.sol -# src/v0.8/ccip/libraries/Pool.sol -# excluded because Foundry doesn't support coverage on library files - -# BurnWithFromMintTokenPool is excluded because Forge doesn't seem to -# register coverage, even though it is 100% covered. - -lcov --remove $1 -o $2 \ - '*/ccip/test/*' \ - '*/vendor/*' \ - '*/shared/*' \ - 'src/v0.8/ccip/ocr/OCR2Abstract.sol' \ - 'src/v0.8/ccip/libraries/Internal.sol' \ - 'src/v0.8/ccip/libraries/RateLimiter.sol' \ - 'src/v0.8/ccip/libraries/USDPriceWith18Decimals.sol' \ - 'src/v0.8/ccip/libraries/MerkleMultiProof.sol' \ - 'src/v0.8/ccip/libraries/Pool.sol' \ - 'src/v0.8/ConfirmedOwnerWithProposal.sol' \ - 'src/v0.8/tests/MockV3Aggregator.sol' \ - 'src/v0.8/ccip/applications/CCIPClientExample.sol' \ - 'src/v0.8/ccip/pools/BurnWithFromMintTokenPool.sol' \ - --rc lcov_branch_coverage=1 diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh new file mode 100755 index 00000000000..50fc991e894 --- /dev/null +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +function check_chainlink_dir() { + local param_dir="chainlink" + current_dir=$(pwd) + + current_base=$(basename "$current_dir") + + if [ "$current_base" != "$param_dir" ]; then + >&2 echo "The script must be run from the root of $param_dir directory" + exit 1 + fi +} + +check_chainlink_dir + +if [ "$#" -lt 5 ]; then + >&2 echo "Generates Markdown Slither reports and saves them to a target directory." + >&2 echo "Usage: $0 [slither extra params]" +exit 1 +fi + +REPO_URL=$1 +CONFIG_FILE=$2 +SOURCE_DIR=$3 +FILES=${4// /} # Remove any spaces from the list of files +TARGET_DIR=$5 +SLITHER_EXTRA_PARAMS=$6 + +run_slither() { + local FILE=$1 + local TARGET_DIR=$2 + + # needed, because the action we use returns all modified files, also deleted ones and we must skip those + if [ ! -f "$FILE" ]; then + echo "Warning: File not found: $FILE" + echo "Skipping..." + return + fi + + source ./contracts/scripts/ci/select_solc_version.sh "$FILE" + if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to select Solc version for $FILE" + exit 1 + fi + + SLITHER_OUTPUT_FILE="$TARGET_DIR/$(basename "${FILE%.sol}")-slither-report.md" + + output=$(slither --config-file "$CONFIG_FILE" "$FILE" --checklist --markdown-root "$REPO_URL" --fail-none $SLITHER_EXTRA_PARAMS) + if [ $? -ne 0 ]; then + >&2 echo "Slither failed for $FILE" + exit 1 + fi + output=$(echo "$output" | sed '/\*\*THIS CHECKLIST IS NOT COMPLETE\*\*. Use `--show-ignored-findings` to show all the results./d' | sed '/Summary/d') + + echo "# Summary for $FILE" > "$SLITHER_OUTPUT_FILE" + echo "$output" >> "$SLITHER_OUTPUT_FILE" +} + +process_files() { + local SOURCE_DIR=$1 + local TARGET_DIR=$2 + local FILES=(${3//,/ }) # Split the comma-separated list into an array + + mkdir -p "$TARGET_DIR" + + for FILE in "${FILES[@]}"; do + FILE=${FILE//\"/} + run_slither "$SOURCE_DIR/$FILE" "$TARGET_DIR" + done +} + +process_files "$SOURCE_DIR" "$TARGET_DIR" "${FILES[@]}" + +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to generate Slither reports" + exit 1 +fi + +echo "Slither reports saved in $TARGET_DIR folder" diff --git a/contracts/scripts/ci/modify_remappings.sh b/contracts/scripts/ci/modify_remappings.sh new file mode 100755 index 00000000000..72394a2fe89 --- /dev/null +++ b/contracts/scripts/ci/modify_remappings.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + >&2 echo "Usage: $0 " + exit 1 +fi + +DIR_PREFIX=$1 +REMAPPINGS_FILE=$2 + +if [ ! -f "$REMAPPINGS_FILE" ]; then + >&2 echo "Error: Remappings file '$REMAPPINGS_FILE' not found." + exit 1 +fi + +OUTPUT_FILE="remappings_modified.txt" + +while IFS= read -r line; do + if [[ "$line" =~ ^[^=]+= ]]; then + REMAPPED_PATH="${line#*=}" + MODIFIED_LINE="${line%=*}=${DIR_PREFIX}/${REMAPPED_PATH}" + echo "$MODIFIED_LINE" >> "$OUTPUT_FILE" + else + echo "$line" >> "$OUTPUT_FILE" + fi +done < "$REMAPPINGS_FILE" + +echo "Modified remappings have been saved to: $OUTPUT_FILE" diff --git a/contracts/scripts/ci/select_solc_version.sh b/contracts/scripts/ci/select_solc_version.sh new file mode 100755 index 00000000000..316b261616a --- /dev/null +++ b/contracts/scripts/ci/select_solc_version.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +function check_chainlink_dir() { + local param_dir="chainlink" + current_dir=$(pwd) + + current_base=$(basename "$current_dir") + + if [ "$current_base" != "$param_dir" ]; then + echo "The script must be run from the root of $param_dir directory" + exit 1 + fi +} + +check_chainlink_dir + +FILE="$1" + +if [ "$#" -lt 1 ]; then + echo "Detects the Solidity version of a file and selects the appropriate Solc version." + echo "If the version is not installed, it will be installed and selected." + echo "Will prefer to use the version from Foundry profile if it satisfies the version in the file." + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$FILE" ]; then + echo "Error: File not provided." + exit 1 +fi + +extract_product() { + local path=$1 + + echo "$path" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1 +} + +extract_pragma() { + local FILE=$1 + + if [[ -f "$FILE" ]]; then + SOLCVER="$(grep --no-filename '^pragma solidity' "$FILE" | cut -d' ' -f3)" + else + echo "$FILE is not a file or it could not be found. Exiting." + return 1 + fi + SOLCVER="$(echo "$SOLCVER" | sed 's/[^0-9\.^]//g')" + >&2 echo "::debug::Detected Solidity version in pragma: $SOLCVER" + echo "$SOLCVER" +} + +echo "Detecting Solc version for $FILE" + +# Set FOUNDRY_PROFILE to the product name only if it is set; otherwise either already set value will be used or it will be empty +PRODUCT=$(extract_product "$FILE") +if [ -n "$PRODUCT" ]; then + FOUNDRY_PROFILE="$PRODUCT" +fi +SOLC_IN_PROFILE=$(forge config --json --root contracts | jq ".solc") +SOLC_IN_PROFILE=$(echo "$SOLC_IN_PROFILE" | tr -d "'\"") +echo "::debug::Detected Solidity version in profile: $SOLC_IN_PROFILE" + +SOLCVER=$(extract_pragma "$FILE") + +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "Error: Failed to extract the Solidity version from $FILE." + return 1 +fi + +SOLCVER=$(echo "$SOLCVER" | tr -d "'\"") + +if [[ "$SOLC_IN_PROFILE" != "null" && -n "$SOLCVER" ]]; then + set +e + COMPAT_SOLC_VERSION=$(npx semver "$SOLC_IN_PROFILE" -r "$SOLCVER") + exit_code=$? + set -e + if [[ $exit_code -eq 0 && -n "$COMPAT_SOLC_VERSION" ]]; then + echo "::debug::Version $SOLC_IN_PROFILE satisfies the constraint $SOLCVER" + SOLC_TO_USE="$SOLC_IN_PROFILE" + else + echo "::debug::Version $SOLC_IN_PROFILE does not satisfy the constraint $SOLCVER" + SOLC_TO_USE="$SOLCVER" + fi + elif [[ "$SOLC_IN_PROFILE" != "null" && -z "$SOLCVER" ]]; then + >&2 echo "No version found in the Solidity file. Exiting" + return 1 + elif [[ "$SOLC_IN_PROFILE" == "null" && -n "$SOLCVER" ]]; then + echo "::debug::Using the version from the file: $SOLCVER" + SOLC_TO_USE="$SOLCVER" + else + >&2 echo "No version found in the profile or the Solidity file." + return 1 +fi + +echo "Will use $SOLC_TO_USE" +SOLC_TO_USE=$(echo "$SOLC_TO_USE" | tr -d "'\"") +SOLC_TO_USE="$(echo "$SOLC_TO_USE" | sed 's/[^0-9\.]//g')" + +INSTALLED_VERSIONS=$(solc-select versions) + +if echo "$INSTALLED_VERSIONS" | grep -q "$SOLC_TO_USE"; then + echo "::debug::Version $SOLCVER is already installed." + if echo "$INSTALLED_VERSIONS" | grep "$SOLC_TO_USE" | grep -q "current"; then + echo "::debug::Version $SOLCVER is already selected." + else + echo "::debug::Selecting $SOLC_TO_USE" + solc-select use "$SOLC_TO_USE" + fi +else + echo "::debug::Version $SOLC_TO_USE is not installed." + solc-select install "$SOLC_TO_USE" + solc-select use "$SOLC_TO_USE" +fi diff --git a/contracts/scripts/lcov_prune b/contracts/scripts/lcov_prune new file mode 100755 index 00000000000..0f16715cb2e --- /dev/null +++ b/contracts/scripts/lcov_prune @@ -0,0 +1,77 @@ +#!/bin/bash + +if [ "$#" -ne 3 ]; then + >&2 echo "Usage: $0 " + exit 1 +fi + +set -e + +product_name=$1 +input_coverage_file=$2 +output_coverage_file=$3 + +# src/v0.8/ccip/libraries/Internal.sol +# src/v0.8/ccip/libraries/RateLimiter.sol +# src/v0.8/ccip/libraries/USDPriceWith18Decimals.sol +# src/v0.8/ccip/libraries/MerkleMultiProof.sol +# src/v0.8/ccip/libraries/Pool.sol +# excluded because Foundry doesn't support coverage on library files + +# BurnWithFromMintTokenPool is excluded because Forge doesn't seem to +# register coverage, even though it is 100% covered. +exclusion_list_ccip=( + "src/v0.8/ccip/ocr/OCR2Abstract.sol" + "src/v0.8/ccip/libraries/Internal.sol" + "src/v0.8/ccip/libraries/RateLimiter.sol" + "src/v0.8/ccip/libraries/USDPriceWith18Decimals.sol" + "src/v0.8/ccip/libraries/MerkleMultiProof.sol" + "src/v0.8/ccip/libraries/Pool.sol" + "src/v0.8/ConfirmedOwnerWithProposal.sol" + "src/v0.8/tests/MockV3Aggregator.sol" + "src/v0.8/ccip/applications/CCIPClientExample.sol" + "src/v0.8/ccip/pools/BurnWithFromMintTokenPool.sol" +) + +exclusion_list_shared=( + "*/shared/*" +) + +exclusion_list_common=( + "*/$product_name/test/*" + "*/vendor/*" +) + +all_exclusions=() + +case "$product_name" in + "ccip") + all_exclusions+=("${exclusion_list_ccip[@]}") + ;; + "shared") + # No product-specific exclusions for shared + ;; + *) + ;; +esac + +all_exclusions+=("${exclusion_list_common[@]}") + +if [ "$product_name" != "shared" ]; then + all_exclusions+=("${exclusion_list_shared[@]}") +fi + +echo "Excluding the following files for product $product_name:" +for exclusion in "${all_exclusions[@]}"; do + echo "$exclusion" +done + +lcov_command="lcov --remove $input_coverage_file -o $output_coverage_file" + +for exclusion in "${all_exclusions[@]}"; do + lcov_command+=" \"$exclusion\"" +done + +lcov_command+=" --rc lcov_branch_coverage=1" + +eval $lcov_command From 8d9ce269e94c0d4564e0aac2e4810ca3aaa1a7ed Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 13:00:39 +0200 Subject: [PATCH 02/54] update Shared code cov --- .github/workflows/solidity-foundry.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 1360d8de027..4b39f6f3288 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -55,7 +55,7 @@ jobs: - name: operatorforwarder min-coverage: 55.7 - name: shared - min-coverage: 37.1 + min-coverage: 32.6 - name: vrf min-coverage: 98.5 needs: [changes] From 3b8049d608bc2d8be5ae9cf053ea448c93c6435b Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 14:05:54 +0200 Subject: [PATCH 03/54] exclude deleted files from Slither --- .github/workflows/solidity-foundry.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 4b39f6f3288..62e3e209007 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -28,7 +28,7 @@ jobs: - 'contracts/foundry.toml' - 'contracts/gas-snapshots/*.gas-snapshot' sol: - - 'contracts/src/v0.8/**/*.sol' + - modified|added: 'contracts/src/v0.8/**/*.sol' tests: if: needs.changes.outputs.src_changes == 'true' From 27b80094096029f22639aa4d0ae8a83414f83e61 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:26:12 +0200 Subject: [PATCH 04/54] use single source of truth for all Solidity Foundry jobs --- .github/workflows/solidity-foundry.yml | 112 ++++++++++++------------- 1 file changed, 52 insertions(+), 60 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 62e3e209007..5d6bd0caf3e 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -7,6 +7,32 @@ env: FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 jobs: + define-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.define-matrix.outputs.matrix }} + steps: + - name: Define test matrix + id: define-matrix + shell: bash + run: | + cat < matrix.json + { + { "name": "automation", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "ccip", "setup": { "run-coverage": true, "min-coverage": 98.5, "run-gas-snapshot": true, "run-forge-fmt": true }}, + { "name": "functions", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "keystone", "setup": { "run-coverage": true, "min-coverage": 74.1, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "l2ep", "setup": { "run-coverage": true, "min-coverage": 65.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "liquiditymanager", "setup": { "run-coverage": true, "min-coverage": 46.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "llo-feeds", "setup": { "run-coverage": true, "min-coverage": 49.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "operatorforwarder", "setup": { "run-coverage": true, "min-coverage": 55.7, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "shared", "setup": { "run-coverage": true, "min-coverage": 32.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "vrf", "setup": { "run-coverage": false, "run-gas-snapshot": false, "run-forge-fmt": false }} + } + EOF + + cat matrix.json >> $GITHUB_OUTPUT + changes: name: Detect changes runs-on: ubuntu-latest @@ -35,30 +61,8 @@ jobs: strategy: fail-fast: false matrix: - product: - # 98.5 is the aspirational code coverage once we migrate all tests to Foundry - # product that have that minimum coverage of 98.5 in the matrix below are currently excluded from the check - - name: automation - min-coverage: 98.5 - - name: ccip - min-coverage: 98.5 - - name: functions - min-coverage: 98.5 - - name: keystone - min-coverage: 74.1 - - name: l2ep - min-coverage: 65.6 - - name: liquiditymanager - min-coverage: 46.3 - - name: llo-feeds - min-coverage: 49.3 - - name: operatorforwarder - min-coverage: 55.7 - - name: shared - min-coverage: 32.6 - - name: vrf - min-coverage: 98.5 - needs: [changes] + product: ${{fromJson(needs.define-matrix.outputs.matrix)}} + needs: [define-matrix, changes] name: Foundry Tests ${{ matrix.product.name }} runs-on: ubuntu-22.04 @@ -103,7 +107,7 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge snapshot - if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["keystone"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-gas-snapshot }} run: | forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot id: snapshot @@ -113,32 +117,31 @@ jobs: # required for code coverage report generation - name: Setup LCOV - if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - name: Run coverage for ${{ matrix.product.name }} - if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} working-directory: contracts run: forge coverage --report lcov env: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Prune lcov report - if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} run: | sudo apt-get install lcov ./contracts/scripts/lcov_prune ${{ matrix.product.name }} ./contracts/lcov.info ./contracts/lcov.info.pruned - name: Report code coverage for ${{ matrix.product.name }} - if: ${{ !contains(fromJson('["vrf"]'), matrix.product.name) && !contains(fromJson('["automation"]'), matrix.product.name) && !contains(fromJson('["functions"]'), matrix.product.name) && needs.changes.outputs.src_changes == 'true' }} + if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: - update-comment: true + update-comment: false coverage-files: ./contracts/lcov.info.pruned minimum-coverage: ${{ matrix.product.min-coverage }} artifact-name: code-coverage-report-${{ matrix.product.name }} working-directory: ./contracts - github-token: ${{ secrets.GITHUB_TOKEN }} - name: Collect Metrics if: needs.changes.outputs.src_changes == 'true' @@ -197,8 +200,6 @@ jobs: - name: Run Slither shell: bash - env: - FOUNDRY_PROFILE: ${{ matrix.product.name }} run: | CHANGED="${{ needs.changes.outputs.sol_files }}" CHANGED_ARRAY=$(echo "$CHANGED" | tr ' ' ',') @@ -240,57 +241,48 @@ jobs: this-job-name: Run static analysis continue-on-error: true - fmt: - needs: [ changes ] - name: Check formatting for modified files - if: needs.changes.outputs.sol_changes == 'true' + solidity-forge-fmt: + if: needs.changes.outputs.changes == 'true' + needs: [define-matrix, changes] + strategy: + fail-fast: false + matrix: + product: ${{fromJson(needs.define-matrix.outputs.matrix)}} + name: Forge fmt ${{ matrix.product.name }} runs-on: ubuntu-22.04 steps: - name: Checkout the repo + if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS + if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry + if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - env: - FOUNDRY_PROFILE: ${{ matrix.product.name }} - run: | - SOL_FILES="${{ needs.changes.outputs.sol_files }}" - FILE_LIST=$(echo "$SOL_FILES" | tr ' ' '\n') - format_errors=0 - for file in $FILE_LIST; do - sanitized_file="${file/contracts\//}" - PROFILE=$(echo "$sanitized_file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) - if [ -z "$PROFILE" ]; then - PROFILE="${{ env.FOUNDRY_PROFILE }}" - fi - echo "::debug::Running forge fmt $sanitized_file with FOUNDRY_PROFILE=$PROFILE" - FOUNDRY_PROFILE=$PROFILE forge fmt --check "$sanitized_file" || format_errors=$((format_errors+1)) - done - if [ $format_errors -ne 0 ]; then - echo "There were $format_errors formatting errors." - exit 1 - else - echo "All files are properly formatted." - fi + if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + run: forge fmt --check id: fmt working-directory: contracts + env: + FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics + if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: - id: solidity-foundry-fmt + id: solidity-foundry-fmt-${{ matrix.product }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Foundry Tests fmt + this-job-name: Foundry fmt ${{ matrix.product }} continue-on-error: true From 7c782ee5357e3839612346cf48d00c5a758bb856 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:29:41 +0200 Subject: [PATCH 05/54] fix json --- .github/workflows/solidity-foundry.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 5d6bd0caf3e..b9467df5d49 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -17,7 +17,7 @@ jobs: shell: bash run: | cat < matrix.json - { + [ { "name": "automation", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "ccip", "setup": { "run-coverage": true, "min-coverage": 98.5, "run-gas-snapshot": true, "run-forge-fmt": true }}, { "name": "functions", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, @@ -28,10 +28,11 @@ jobs: { "name": "operatorforwarder", "setup": { "run-coverage": true, "min-coverage": 55.7, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "shared", "setup": { "run-coverage": true, "min-coverage": 32.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "vrf", "setup": { "run-coverage": false, "run-gas-snapshot": false, "run-forge-fmt": false }} - } + ] EOF - cat matrix.json >> $GITHUB_OUTPUT + matrix=$(cat matrix.json) + echo "matrix=$matrix" >> $GITHUB_OUTPUT changes: name: Detect changes From 30aa2a68b2033cfa40370c385d23fcb077ed8019 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:31:48 +0200 Subject: [PATCH 06/54] compact output with jq --- .github/workflows/solidity-foundry.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index b9467df5d49..ea7d2862293 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -8,6 +8,7 @@ env: jobs: define-matrix: + name: Define test matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.define-matrix.outputs.matrix }} @@ -31,7 +32,7 @@ jobs: ] EOF - matrix=$(cat matrix.json) + matrix=$(cat matrix.json | jq -c .) echo "matrix=$matrix" >> $GITHUB_OUTPUT changes: From 82d1f07057e2ede72aaaf427e39d9461cba67130 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:36:57 +0200 Subject: [PATCH 07/54] fix condition for fmt --- .github/workflows/solidity-foundry.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index ea7d2862293..0ad3240ccfb 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -19,10 +19,10 @@ jobs: run: | cat < matrix.json [ - { "name": "automation", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "automation", "setup": { "run-coverage": false, "run-gas-snapshot": false, "run-forge-fmt": false }}, { "name": "ccip", "setup": { "run-coverage": true, "min-coverage": 98.5, "run-gas-snapshot": true, "run-forge-fmt": true }}, { "name": "functions", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, - { "name": "keystone", "setup": { "run-coverage": true, "min-coverage": 74.1, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "keystone", "setup": { "run-coverage": true, "min-coverage": 74.1, "run-gas-snapshot": false, "run-forge-fmt": false }}, { "name": "l2ep", "setup": { "run-coverage": true, "min-coverage": 65.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "liquiditymanager", "setup": { "run-coverage": true, "min-coverage": 46.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "llo-feeds", "setup": { "run-coverage": true, "min-coverage": 49.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, @@ -244,7 +244,7 @@ jobs: continue-on-error: true solidity-forge-fmt: - if: needs.changes.outputs.changes == 'true' + if: needs.changes.outputs.sol_changes == 'true' needs: [define-matrix, changes] strategy: fail-fast: false @@ -254,23 +254,23 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout the repo - if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS - if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} run: forge fmt --check id: fmt working-directory: contracts @@ -278,7 +278,7 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics - if: ${{ needs.changes.outputs.changes == 'true' && matrix.product.setup.run-forge-fmt }} + if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: From eed761e2e7d9b82c84b85cf794212f34644bb291 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:53:43 +0200 Subject: [PATCH 08/54] try to scope tests to changes --- .github/workflows/solidity-foundry.yml | 69 ++++++++++++++++---------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 0ad3240ccfb..59880a6dfbc 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -39,9 +39,10 @@ jobs: name: Detect changes runs-on: ubuntu-latest outputs: - src_changes: ${{ steps.changes.outputs.src }} - sol_changes: ${{ steps.changes.outputs.sol }} - sol_files: ${{ steps.changes.outputs.sol_files }} + non_src_changes: ${{ steps.changes.outputs.non_src }} + sol_modified: ${{ steps.changes.outputs.sol }} + sol_modified_files: ${{ steps.changes.outputs.sol_files }} + all_changes: ${{ steps.changes.outputs.changes }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -50,16 +51,41 @@ jobs: with: list-files: 'shell' filters: | - src: - - 'contracts/src/v0.8/**/*' + non_src: - '.github/workflows/solidity-foundry.yml' - 'contracts/foundry.toml' - 'contracts/gas-snapshots/*.gas-snapshot' sol: - modified|added: 'contracts/src/v0.8/**/*.sol' + automation: + - 'contracts/src/v0.8/automation/**/*.sol' + ccip: + - 'contracts/src/v0.8/ccip/**/*.sol' + functions: + - 'contracts/src/v0.8/functions/**/*.sol' + keystone: + - 'contracts/src/v0.8/keystone/**/*.sol' + l2ep: + - 'contracts/src/v0.8/l2ep/**/*.sol' + liquiditymanager: + - 'contracts/src/v0.8/liquiditymanager/**/*.sol' + ll-feeds: + - 'contracts/src/v0.8/llo-feeds/**/*.sol' + operatorforwarder: + - 'contracts/src/v0.8/operatorforwarder/**/*.sol' + vrf: + - 'contracts/src/v0.8/vrf/**/*.sol' + shared: + - 'contracts/src/v0.8/shared/**/*.sol' + - 'contracts/src/v0.8/*.sol' + - 'contracts/src/v0.8/mocks/**/*.sol' + - 'contracts/src/v0.8/tests/**/*.sol' + - 'contracts/src/v0.8/vendor/**/*.sol' tests: - if: needs.changes.outputs.src_changes == 'true' + if: ${{ needs.changes.outputs.non_src_changes == 'true' + || contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')}} strategy: fail-fast: false matrix: @@ -80,17 +106,14 @@ jobs: # and not native Foundry. This is to make sure the dependencies # stay in sync. - name: Setup NodeJS - if: needs.changes.outputs.src_changes == 'true' uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: needs.changes.outputs.src_changes == 'true' uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge build - if: needs.changes.outputs.src_changes == 'true' run: | forge --version forge build @@ -100,7 +123,6 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge tests - if: needs.changes.outputs.src_changes == 'true' run: | forge test -vvv id: test @@ -109,7 +131,7 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge snapshot - if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-gas-snapshot }} + if: ${{ matrix.product.setup.run-gas-snapshot }} run: | forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot id: snapshot @@ -119,24 +141,24 @@ jobs: # required for code coverage report generation - name: Setup LCOV - if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} + if: ${{ matrix.product.setup.run-coverage }} uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - name: Run coverage for ${{ matrix.product.name }} - if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} + if: ${{ matrix.product.setup.run-coverage }} working-directory: contracts run: forge coverage --report lcov env: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Prune lcov report - if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} + if: ${{ matrix.product.setup.run-coverage }} run: | sudo apt-get install lcov ./contracts/scripts/lcov_prune ${{ matrix.product.name }} ./contracts/lcov.info ./contracts/lcov.info.pruned - name: Report code coverage for ${{ matrix.product.name }} - if: ${{ needs.changes.outputs.src_changes == 'true' && matrix.product.setup.run-coverage }} + if: ${{ matrix.product.setup.run-coverage }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: update-comment: false @@ -146,7 +168,6 @@ jobs: working-directory: ./contracts - name: Collect Metrics - if: needs.changes.outputs.src_changes == 'true' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -160,7 +181,7 @@ jobs: analyze: needs: [ changes ] name: Run static analysis - if: needs.changes.outputs.sol_changes == 'true' + if: needs.changes.outputs.sol_modified == 'true' runs-on: ubuntu-22.04 steps: - name: Checkout the repo @@ -203,7 +224,7 @@ jobs: - name: Run Slither shell: bash run: | - CHANGED="${{ needs.changes.outputs.sol_files }}" + CHANGED="${{ needs.changes.outputs.sol_modified_files }}" CHANGED_ARRAY=$(echo "$CHANGED" | tr ' ' ',') # modify remappings so that solc can find dependencies @@ -244,33 +265,32 @@ jobs: continue-on-error: true solidity-forge-fmt: - if: needs.changes.outputs.sol_changes == 'true' + name: Forge fmt ${{ matrix.product.name }} + if: ${{ needs.changes.outputs.non_src_changes == 'true' + || contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') + && matrix.product.setup.run-forge-fmt}} needs: [define-matrix, changes] strategy: fail-fast: false matrix: product: ${{fromJson(needs.define-matrix.outputs.matrix)}} - name: Forge fmt ${{ matrix.product.name }} runs-on: ubuntu-22.04 steps: - name: Checkout the repo - if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS - if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} run: forge fmt --check id: fmt working-directory: contracts @@ -278,7 +298,6 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics - if: ${{ needs.changes.outputs.sol_changes == 'true' && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: From f9e116be20e81bd3d46532a80654667403312788 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 16:57:14 +0200 Subject: [PATCH 09/54] move matrix check to step level --- .github/workflows/solidity-foundry.yml | 56 ++++++++++++++++++++------ 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 59880a6dfbc..6af36351f1b 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -83,9 +83,7 @@ jobs: - 'contracts/src/v0.8/vendor/**/*.sol' tests: - if: ${{ needs.changes.outputs.non_src_changes == 'true' - || contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')}} + if: ${{ needs.changes.outputs.non_src_changes == 'true' }} strategy: fail-fast: false matrix: @@ -98,6 +96,8 @@ jobs: # passing required check for PRs that don't have filtered changes. steps: - name: Checkout the repo + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive @@ -106,14 +106,20 @@ jobs: # and not native Foundry. This is to make sure the dependencies # stay in sync. - name: Setup NodeJS + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} uses: ./.github/actions/setup-nodejs - name: Install Foundry + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge build + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} run: | forge --version forge build @@ -123,6 +129,8 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge tests + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} run: | forge test -vvv id: test @@ -131,7 +139,9 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge snapshot - if: ${{ matrix.product.setup.run-gas-snapshot }} + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) && + matrix.product.setup.run-gas-snapshot }} run: | forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot id: snapshot @@ -141,24 +151,32 @@ jobs: # required for code coverage report generation - name: Setup LCOV - if: ${{ matrix.product.setup.run-coverage }} + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-coverage }} uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - name: Run coverage for ${{ matrix.product.name }} - if: ${{ matrix.product.setup.run-coverage }} + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-coverage }} working-directory: contracts run: forge coverage --report lcov env: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Prune lcov report - if: ${{ matrix.product.setup.run-coverage }} + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-coverage }} run: | sudo apt-get install lcov ./contracts/scripts/lcov_prune ${{ matrix.product.name }} ./contracts/lcov.info ./contracts/lcov.info.pruned - name: Report code coverage for ${{ matrix.product.name }} - if: ${{ matrix.product.setup.run-coverage }} + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-coverage }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: update-comment: false @@ -168,6 +186,8 @@ jobs: working-directory: ./contracts - name: Collect Metrics + if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')}} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -266,10 +286,7 @@ jobs: solidity-forge-fmt: name: Forge fmt ${{ matrix.product.name }} - if: ${{ needs.changes.outputs.non_src_changes == 'true' - || contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') - && matrix.product.setup.run-forge-fmt}} + if: ${{ needs.changes.outputs.non_src_changes == 'true'}} needs: [define-matrix, changes] strategy: fail-fast: false @@ -278,19 +295,31 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout the repo + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-forge-fmt }} run: forge fmt --check id: fmt working-directory: contracts @@ -298,6 +327,9 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics + if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: From 5bef6d9b0afede006694a9c6ebddeac59080a8f3 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:00:26 +0200 Subject: [PATCH 10/54] fix outputs path --- .github/workflows/solidity-foundry.yml | 64 +++++++++++++------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 6af36351f1b..3a4be8ae317 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -96,8 +96,8 @@ jobs: # passing required check for PRs that don't have filtered changes. steps: - name: Checkout the repo - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive @@ -106,20 +106,20 @@ jobs: # and not native Foundry. This is to make sure the dependencies # stay in sync. - name: Setup NodeJS - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge build - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} run: | forge --version forge build @@ -129,8 +129,8 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge tests - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared') }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} run: | forge test -vvv id: test @@ -139,8 +139,8 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Run Forge snapshot - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) && + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-gas-snapshot }} run: | forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot @@ -151,14 +151,14 @@ jobs: # required for code coverage report generation - name: Setup LCOV - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-coverage }} uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - name: Run coverage for ${{ matrix.product.name }} - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-coverage }} working-directory: contracts run: forge coverage --report lcov @@ -166,16 +166,16 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Prune lcov report - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-coverage }} run: | sudo apt-get install lcov ./contracts/scripts/lcov_prune ${{ matrix.product.name }} ./contracts/lcov.info ./contracts/lcov.info.pruned - name: Report code coverage for ${{ matrix.product.name }} - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-coverage }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: @@ -186,8 +186,8 @@ jobs: working-directory: ./contracts - name: Collect Metrics - if: ${{ contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')}} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')}} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -295,30 +295,30 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout the repo - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-forge-fmt }} run: forge fmt --check id: fmt @@ -327,8 +327,8 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics - if: ${{ (contains(fromJson(needs.define-matrix.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.define-matrix.outputs.all_changes), 'shared')) + if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 From d1f5f2cc6ef5280b099a39a06f8068a9335d5ce7 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:01:36 +0200 Subject: [PATCH 11/54] trigger From 6f5f25e430b4a162c7a7f73e910d1b4f2b202b18 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:02:43 +0200 Subject: [PATCH 12/54] test with Automation change --- .../src/v0.8/automation/AutomationBase2.sol | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 contracts/src/v0.8/automation/AutomationBase2.sol diff --git a/contracts/src/v0.8/automation/AutomationBase2.sol b/contracts/src/v0.8/automation/AutomationBase2.sol new file mode 100644 index 00000000000..72ee8aaeff2 --- /dev/null +++ b/contracts/src/v0.8/automation/AutomationBase2.sol @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract AutomationBase { + error OnlySimulatedBackend(); + + /** + * @notice method that allows it to be simulated via eth_call by checking that + * the sender is the zero address. + */ + function _preventExecution() internal view { + // solhint-disable-next-line avoid-tx-origin + if (tx.origin != address(0) && tx.origin != address(0x1111111111111111111111111111111111111111)) { + revert OnlySimulatedBackend(); + } + } + + /** + * @notice modifier that allows it to be simulated via eth_call by checking + * that the sender is the zero address. + */ + modifier cannotExecute() { + _preventExecution(); + _; + } +} From 3c0688135d998d811378831ae0476a185ac0e7aa Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:07:17 +0200 Subject: [PATCH 13/54] try with shared --- .github/workflows/solidity-foundry.yml | 2 +- contracts/.slither.config-pr.json | 4 + .../src/v0.8/automation/AutomationBase2.sol | 26 ------ .../v0.8/shared/access/AuthorizedCallers2.sol | 82 +++++++++++++++++++ 4 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 contracts/.slither.config-pr.json delete mode 100644 contracts/src/v0.8/automation/AutomationBase2.sol create mode 100644 contracts/src/v0.8/shared/access/AuthorizedCallers2.sol diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 3a4be8ae317..dcafa026d5c 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -251,7 +251,7 @@ jobs: ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt - ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-artifacts.json "." "$CHANGED_ARRAY" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-pr.json "." "$CHANGED_ARRAY" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" - name: Print Slither summary shell: bash diff --git a/contracts/.slither.config-pr.json b/contracts/.slither.config-pr.json new file mode 100644 index 00000000000..1ef145a7954 --- /dev/null +++ b/contracts/.slither.config-pr.json @@ -0,0 +1,4 @@ +{ + "filter_paths": "(openzeppelin|mocks/|test/|tests/|testhelpers)", + "detectors_to_exclude": "pragma" +} diff --git a/contracts/src/v0.8/automation/AutomationBase2.sol b/contracts/src/v0.8/automation/AutomationBase2.sol deleted file mode 100644 index 72ee8aaeff2..00000000000 --- a/contracts/src/v0.8/automation/AutomationBase2.sol +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.0; - -contract AutomationBase { - error OnlySimulatedBackend(); - - /** - * @notice method that allows it to be simulated via eth_call by checking that - * the sender is the zero address. - */ - function _preventExecution() internal view { - // solhint-disable-next-line avoid-tx-origin - if (tx.origin != address(0) && tx.origin != address(0x1111111111111111111111111111111111111111)) { - revert OnlySimulatedBackend(); - } - } - - /** - * @notice modifier that allows it to be simulated via eth_call by checking - * that the sender is the zero address. - */ - modifier cannotExecute() { - _preventExecution(); - _; - } -} diff --git a/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol b/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol new file mode 100644 index 00000000000..93102d1a97a --- /dev/null +++ b/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: BUSL-1.1 +pragma solidity ^0.8.0; + +import {OwnerIsCreator} from "./OwnerIsCreator.sol"; +import {EnumerableSet} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/utils/structs/EnumerableSet.sol"; + +/// @title The AuthorizedCallers contract +/// @notice A contract that manages multiple authorized callers. Enables restricting access to certain functions to a set of addresses. +contract AuthorizedCallers is OwnerIsCreator { + using EnumerableSet for EnumerableSet.AddressSet; + + event AuthorizedCallerAdded(address caller); + event AuthorizedCallerRemoved(address caller); + + error UnauthorizedCaller(address caller); + error ZeroAddressNotAllowed(); + + /// @notice Update args for changing the authorized callers + struct AuthorizedCallerArgs { + address[] addedCallers; + address[] removedCallers; + } + + /// @dev Set of authorized callers + EnumerableSet.AddressSet internal s_authorizedCallers; + + /// @param authorizedCallers the authorized callers to set + constructor(address[] memory authorizedCallers) { + _applyAuthorizedCallerUpdates( + AuthorizedCallerArgs({addedCallers: authorizedCallers, removedCallers: new address[](0)}) + ); + } + + /// @return authorizedCallers Returns all authorized callers + function getAllAuthorizedCallers() external view returns (address[] memory) { + return s_authorizedCallers.values(); + } + + /// @notice Updates the list of authorized callers + /// @param authorizedCallerArgs Callers to add and remove. Removals are performed first. + function applyAuthorizedCallerUpdates(AuthorizedCallerArgs memory authorizedCallerArgs) external onlyOwner { + _applyAuthorizedCallerUpdates(authorizedCallerArgs); + } + + /// @notice Updates the list of authorized callers + /// @param authorizedCallerArgs Callers to add and remove. Removals are performed first. + function _applyAuthorizedCallerUpdates(AuthorizedCallerArgs memory authorizedCallerArgs) internal { + address[] memory removedCallers = authorizedCallerArgs.removedCallers; + for (uint256 i = 0; i < removedCallers.length; ++i) { + address caller = removedCallers[i]; + + if (s_authorizedCallers.remove(caller)) { + emit AuthorizedCallerRemoved(caller); + } + } + + address[] memory addedCallers = authorizedCallerArgs.addedCallers; + for (uint256 i = 0; i < addedCallers.length; ++i) { + address caller = addedCallers[i]; + + if (caller == address(0)) { + revert ZeroAddressNotAllowed(); + } + + s_authorizedCallers.add(caller); + emit AuthorizedCallerAdded(caller); + } + } + + /// @notice Checks the sender and reverts if it is anyone other than a listed authorized caller. + function _validateCaller() internal view { + if (!s_authorizedCallers.contains(msg.sender)) { + revert UnauthorizedCaller(msg.sender); + } + } + + /// @notice Checks the sender and reverts if it is anyone other than a listed authorized caller. + modifier onlyAuthorizedCallers() { + _validateCaller(); + _; + } +} From 033a062097129e098b7f896f30b24fcf44e1f006 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:11:07 +0200 Subject: [PATCH 14/54] run fmt also if any sol files were modified --- .github/workflows/solidity-foundry.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index dcafa026d5c..a01dd179e94 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -286,7 +286,7 @@ jobs: solidity-forge-fmt: name: Forge fmt ${{ matrix.product.name }} - if: ${{ needs.changes.outputs.non_src_changes == 'true'}} + if: ${{ needs.changes.outputs.non_src_changes == 'true' || needs.changes.outputs.sol_modified == 'true' }} needs: [define-matrix, changes] strategy: fail-fast: false From 9d18fde017b2ac419216194f89e7958d4de4e706 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:12:34 +0200 Subject: [PATCH 15/54] fix job name in collect metrics --- .github/workflows/solidity-foundry.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index a01dd179e94..6498a4aafbe 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -333,9 +333,9 @@ jobs: id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: - id: solidity-foundry-fmt-${{ matrix.product }} + id: solidity-foundry-fmt-${{ matrix.product.name }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Foundry fmt ${{ matrix.product }} + this-job-name: Foundry fmt ${{ matrix.product.name }} continue-on-error: true From 94ffafab55c2293995082e6c4e2aa882c155f510 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:20:49 +0200 Subject: [PATCH 16/54] trigger pipeline only for localised change + update changes info --- .github/workflows/solidity-foundry.yml | 10 +- contracts/src/v0.8/l2ep/dev/Flags2.sol | 177 ++++++++++++++++++ .../v0.8/shared/access/AuthorizedCallers2.sol | 82 -------- 3 files changed, 184 insertions(+), 85 deletions(-) create mode 100644 contracts/src/v0.8/l2ep/dev/Flags2.sol delete mode 100644 contracts/src/v0.8/shared/access/AuthorizedCallers2.sol diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 6498a4aafbe..c5f14f717e7 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -6,6 +6,10 @@ env: # Has to match the `make foundry` version in `contracts/GNUmakefile` FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 +# Making changes: +# * use the top-level matrix to decide, which checks should run for each product. +# * when enabling code coverage, remember to adjust the minimum code coverage as it's set to 98.5% by default. + jobs: define-matrix: name: Define test matrix @@ -19,16 +23,16 @@ jobs: run: | cat < matrix.json [ - { "name": "automation", "setup": { "run-coverage": false, "run-gas-snapshot": false, "run-forge-fmt": false }}, + { "name": "automation", "setup": { "run-coverage": false, "min-coverage": 98.5, "run-gas-snapshot": false, "run-forge-fmt": false }}, { "name": "ccip", "setup": { "run-coverage": true, "min-coverage": 98.5, "run-gas-snapshot": true, "run-forge-fmt": true }}, - { "name": "functions", "setup": { "run-coverage": false, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "functions", "setup": { "run-coverage": false, "min-coverage": 98.5, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "keystone", "setup": { "run-coverage": true, "min-coverage": 74.1, "run-gas-snapshot": false, "run-forge-fmt": false }}, { "name": "l2ep", "setup": { "run-coverage": true, "min-coverage": 65.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "liquiditymanager", "setup": { "run-coverage": true, "min-coverage": 46.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "llo-feeds", "setup": { "run-coverage": true, "min-coverage": 49.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "operatorforwarder", "setup": { "run-coverage": true, "min-coverage": 55.7, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "shared", "setup": { "run-coverage": true, "min-coverage": 32.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, - { "name": "vrf", "setup": { "run-coverage": false, "run-gas-snapshot": false, "run-forge-fmt": false }} + { "name": "vrf", "setup": { "run-coverage": false, "min-coverage": 98.5, "run-gas-snapshot": false, "run-forge-fmt": false }} ] EOF diff --git a/contracts/src/v0.8/l2ep/dev/Flags2.sol b/contracts/src/v0.8/l2ep/dev/Flags2.sol new file mode 100644 index 00000000000..0fcd095ac8e --- /dev/null +++ b/contracts/src/v0.8/l2ep/dev/Flags2.sol @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.6; + +import {SimpleReadAccessController} from "../../shared/access/SimpleReadAccessController.sol"; +import {AccessControllerInterface} from "../../shared/interfaces/AccessControllerInterface.sol"; +import {ITypeAndVersion} from "../../shared/interfaces/ITypeAndVersion.sol"; + +/* dev dependencies - to be re/moved after audit */ +import {FlagsInterface} from "./interfaces/FlagsInterface.sol"; + +/** + * @title The Flags contract + * @notice Allows flags to signal to any reader on the access control list. + * The owner can set flags, or designate other addresses to set flags. + * Raise flag actions are controlled by its own access controller. + * Lower flag actions are controlled by its own access controller. + * An expected pattern is to allow addresses to raise flags on themselves, so if you are subscribing to + * FlagOn events you should filter for addresses you care about. + */ +// solhint-disable gas-custom-errors +contract Flags is ITypeAndVersion, FlagsInterface, SimpleReadAccessController { + AccessControllerInterface public raisingAccessController; + AccessControllerInterface public loweringAccessController; + + mapping(address => bool) private s_flags; + + event FlagRaised(address indexed subject); + event FlagLowered(address indexed subject); + event RaisingAccessControllerUpdated(address indexed previous, address indexed current); + event LoweringAccessControllerUpdated(address indexed previous, address indexed current); + + /** + * @param racAddress address for the raising access controller. + * @param lacAddress address for the lowering access controller. + */ + constructor(address racAddress, address lacAddress) { + setRaisingAccessController(racAddress); + setLoweringAccessController(lacAddress); + } + + /** + * @notice versions: + * + * - Flags 1.1.0: upgraded to solc 0.8, added lowering access controller + * - Flags 1.0.0: initial release + * + * @inheritdoc ITypeAndVersion + */ + function typeAndVersion() external pure virtual override returns (string memory) { + return "Flags 1.1.0"; + } + + /** + * @notice read the warning flag status of a contract address. + * @param subject The contract address being checked for a flag. + * @return A true value indicates that a flag was raised and a + * false value indicates that no flag was raised. + */ + function getFlag(address subject) external view override checkAccess returns (bool) { + return s_flags[subject]; + } + + /** + * @notice read the warning flag status of a contract address. + * @param subjects An array of addresses being checked for a flag. + * @return An array of bools where a true value for any flag indicates that + * a flag was raised and a false value indicates that no flag was raised. + */ + function getFlags(address[] calldata subjects) external view override checkAccess returns (bool[] memory) { + bool[] memory responses = new bool[](subjects.length); + for (uint256 i = 0; i < subjects.length; i++) { + responses[i] = s_flags[subjects[i]]; + } + return responses; + } + + /** + * @notice enable the warning flag for an address. + * Access is controlled by raisingAccessController, except for owner + * who always has access. + * @param subject The contract address whose flag is being raised + */ + function raiseFlag(address subject) external override { + require(_allowedToRaiseFlags(), "Not allowed to raise flags"); + + _tryToRaiseFlag(subject); + } + + /** + * @notice enable the warning flags for multiple addresses. + * Access is controlled by raisingAccessController, except for owner + * who always has access. + * @param subjects List of the contract addresses whose flag is being raised + */ + function raiseFlags(address[] calldata subjects) external override { + require(_allowedToRaiseFlags(), "Not allowed to raise flags"); + + for (uint256 i = 0; i < subjects.length; i++) { + _tryToRaiseFlag(subjects[i]); + } + } + + /** + * @notice allows owner to disable the warning flags for an addresses. + * Access is controlled by loweringAccessController, except for owner + * who always has access. + * @param subject The contract address whose flag is being lowered + */ + function lowerFlag(address subject) external override { + require(_allowedToLowerFlags(), "Not allowed to lower flags"); + + _tryToLowerFlag(subject); + } + + /** + * @notice allows owner to disable the warning flags for multiple addresses. + * Access is controlled by loweringAccessController, except for owner + * who always has access. + * @param subjects List of the contract addresses whose flag is being lowered + */ + function lowerFlags(address[] calldata subjects) external override { + require(_allowedToLowerFlags(), "Not allowed to lower flags"); + + for (uint256 i = 0; i < subjects.length; i++) { + address subject = subjects[i]; + + _tryToLowerFlag(subject); + } + } + + /** + * @notice allows owner to change the access controller for raising flags. + * @param racAddress new address for the raising access controller. + */ + function setRaisingAccessController(address racAddress) public override onlyOwner { + address previous = address(raisingAccessController); + + if (previous != racAddress) { + raisingAccessController = AccessControllerInterface(racAddress); + + emit RaisingAccessControllerUpdated(previous, racAddress); + } + } + + function setLoweringAccessController(address lacAddress) public override onlyOwner { + address previous = address(loweringAccessController); + + if (previous != lacAddress) { + loweringAccessController = AccessControllerInterface(lacAddress); + + emit LoweringAccessControllerUpdated(previous, lacAddress); + } + } + + // PRIVATE + function _allowedToRaiseFlags() private view returns (bool) { + return msg.sender == owner() || raisingAccessController.hasAccess(msg.sender, msg.data); + } + + function _allowedToLowerFlags() private view returns (bool) { + return msg.sender == owner() || loweringAccessController.hasAccess(msg.sender, msg.data); + } + + function _tryToRaiseFlag(address subject) private { + if (!s_flags[subject]) { + s_flags[subject] = true; + emit FlagRaised(subject); + } + } + + function _tryToLowerFlag(address subject) private { + if (s_flags[subject]) { + s_flags[subject] = false; + emit FlagLowered(subject); + } + } +} diff --git a/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol b/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol deleted file mode 100644 index 93102d1a97a..00000000000 --- a/contracts/src/v0.8/shared/access/AuthorizedCallers2.sol +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 -pragma solidity ^0.8.0; - -import {OwnerIsCreator} from "./OwnerIsCreator.sol"; -import {EnumerableSet} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/utils/structs/EnumerableSet.sol"; - -/// @title The AuthorizedCallers contract -/// @notice A contract that manages multiple authorized callers. Enables restricting access to certain functions to a set of addresses. -contract AuthorizedCallers is OwnerIsCreator { - using EnumerableSet for EnumerableSet.AddressSet; - - event AuthorizedCallerAdded(address caller); - event AuthorizedCallerRemoved(address caller); - - error UnauthorizedCaller(address caller); - error ZeroAddressNotAllowed(); - - /// @notice Update args for changing the authorized callers - struct AuthorizedCallerArgs { - address[] addedCallers; - address[] removedCallers; - } - - /// @dev Set of authorized callers - EnumerableSet.AddressSet internal s_authorizedCallers; - - /// @param authorizedCallers the authorized callers to set - constructor(address[] memory authorizedCallers) { - _applyAuthorizedCallerUpdates( - AuthorizedCallerArgs({addedCallers: authorizedCallers, removedCallers: new address[](0)}) - ); - } - - /// @return authorizedCallers Returns all authorized callers - function getAllAuthorizedCallers() external view returns (address[] memory) { - return s_authorizedCallers.values(); - } - - /// @notice Updates the list of authorized callers - /// @param authorizedCallerArgs Callers to add and remove. Removals are performed first. - function applyAuthorizedCallerUpdates(AuthorizedCallerArgs memory authorizedCallerArgs) external onlyOwner { - _applyAuthorizedCallerUpdates(authorizedCallerArgs); - } - - /// @notice Updates the list of authorized callers - /// @param authorizedCallerArgs Callers to add and remove. Removals are performed first. - function _applyAuthorizedCallerUpdates(AuthorizedCallerArgs memory authorizedCallerArgs) internal { - address[] memory removedCallers = authorizedCallerArgs.removedCallers; - for (uint256 i = 0; i < removedCallers.length; ++i) { - address caller = removedCallers[i]; - - if (s_authorizedCallers.remove(caller)) { - emit AuthorizedCallerRemoved(caller); - } - } - - address[] memory addedCallers = authorizedCallerArgs.addedCallers; - for (uint256 i = 0; i < addedCallers.length; ++i) { - address caller = addedCallers[i]; - - if (caller == address(0)) { - revert ZeroAddressNotAllowed(); - } - - s_authorizedCallers.add(caller); - emit AuthorizedCallerAdded(caller); - } - } - - /// @notice Checks the sender and reverts if it is anyone other than a listed authorized caller. - function _validateCaller() internal view { - if (!s_authorizedCallers.contains(msg.sender)) { - revert UnauthorizedCaller(msg.sender); - } - } - - /// @notice Checks the sender and reverts if it is anyone other than a listed authorized caller. - modifier onlyAuthorizedCallers() { - _validateCaller(); - _; - } -} From a7fac9f24f89c31298d03de74391cff71268f6ef Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:22:29 +0200 Subject: [PATCH 17/54] add changeset --- contracts/.changeset/itchy-deers-deny.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 contracts/.changeset/itchy-deers-deny.md diff --git a/contracts/.changeset/itchy-deers-deny.md b/contracts/.changeset/itchy-deers-deny.md new file mode 100644 index 00000000000..888d58ce311 --- /dev/null +++ b/contracts/.changeset/itchy-deers-deny.md @@ -0,0 +1,5 @@ +--- +"@chainlink/contracts": patch +--- + +More comprehensive & product-scoped Solidity Foundry pipeline From 4bb7e462916f07b652b076340416a50f77b958ce Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 1 Aug 2024 17:24:23 +0200 Subject: [PATCH 18/54] remove test change --- contracts/src/v0.8/l2ep/dev/Flags2.sol | 177 ------------------------- 1 file changed, 177 deletions(-) delete mode 100644 contracts/src/v0.8/l2ep/dev/Flags2.sol diff --git a/contracts/src/v0.8/l2ep/dev/Flags2.sol b/contracts/src/v0.8/l2ep/dev/Flags2.sol deleted file mode 100644 index 0fcd095ac8e..00000000000 --- a/contracts/src/v0.8/l2ep/dev/Flags2.sol +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.6; - -import {SimpleReadAccessController} from "../../shared/access/SimpleReadAccessController.sol"; -import {AccessControllerInterface} from "../../shared/interfaces/AccessControllerInterface.sol"; -import {ITypeAndVersion} from "../../shared/interfaces/ITypeAndVersion.sol"; - -/* dev dependencies - to be re/moved after audit */ -import {FlagsInterface} from "./interfaces/FlagsInterface.sol"; - -/** - * @title The Flags contract - * @notice Allows flags to signal to any reader on the access control list. - * The owner can set flags, or designate other addresses to set flags. - * Raise flag actions are controlled by its own access controller. - * Lower flag actions are controlled by its own access controller. - * An expected pattern is to allow addresses to raise flags on themselves, so if you are subscribing to - * FlagOn events you should filter for addresses you care about. - */ -// solhint-disable gas-custom-errors -contract Flags is ITypeAndVersion, FlagsInterface, SimpleReadAccessController { - AccessControllerInterface public raisingAccessController; - AccessControllerInterface public loweringAccessController; - - mapping(address => bool) private s_flags; - - event FlagRaised(address indexed subject); - event FlagLowered(address indexed subject); - event RaisingAccessControllerUpdated(address indexed previous, address indexed current); - event LoweringAccessControllerUpdated(address indexed previous, address indexed current); - - /** - * @param racAddress address for the raising access controller. - * @param lacAddress address for the lowering access controller. - */ - constructor(address racAddress, address lacAddress) { - setRaisingAccessController(racAddress); - setLoweringAccessController(lacAddress); - } - - /** - * @notice versions: - * - * - Flags 1.1.0: upgraded to solc 0.8, added lowering access controller - * - Flags 1.0.0: initial release - * - * @inheritdoc ITypeAndVersion - */ - function typeAndVersion() external pure virtual override returns (string memory) { - return "Flags 1.1.0"; - } - - /** - * @notice read the warning flag status of a contract address. - * @param subject The contract address being checked for a flag. - * @return A true value indicates that a flag was raised and a - * false value indicates that no flag was raised. - */ - function getFlag(address subject) external view override checkAccess returns (bool) { - return s_flags[subject]; - } - - /** - * @notice read the warning flag status of a contract address. - * @param subjects An array of addresses being checked for a flag. - * @return An array of bools where a true value for any flag indicates that - * a flag was raised and a false value indicates that no flag was raised. - */ - function getFlags(address[] calldata subjects) external view override checkAccess returns (bool[] memory) { - bool[] memory responses = new bool[](subjects.length); - for (uint256 i = 0; i < subjects.length; i++) { - responses[i] = s_flags[subjects[i]]; - } - return responses; - } - - /** - * @notice enable the warning flag for an address. - * Access is controlled by raisingAccessController, except for owner - * who always has access. - * @param subject The contract address whose flag is being raised - */ - function raiseFlag(address subject) external override { - require(_allowedToRaiseFlags(), "Not allowed to raise flags"); - - _tryToRaiseFlag(subject); - } - - /** - * @notice enable the warning flags for multiple addresses. - * Access is controlled by raisingAccessController, except for owner - * who always has access. - * @param subjects List of the contract addresses whose flag is being raised - */ - function raiseFlags(address[] calldata subjects) external override { - require(_allowedToRaiseFlags(), "Not allowed to raise flags"); - - for (uint256 i = 0; i < subjects.length; i++) { - _tryToRaiseFlag(subjects[i]); - } - } - - /** - * @notice allows owner to disable the warning flags for an addresses. - * Access is controlled by loweringAccessController, except for owner - * who always has access. - * @param subject The contract address whose flag is being lowered - */ - function lowerFlag(address subject) external override { - require(_allowedToLowerFlags(), "Not allowed to lower flags"); - - _tryToLowerFlag(subject); - } - - /** - * @notice allows owner to disable the warning flags for multiple addresses. - * Access is controlled by loweringAccessController, except for owner - * who always has access. - * @param subjects List of the contract addresses whose flag is being lowered - */ - function lowerFlags(address[] calldata subjects) external override { - require(_allowedToLowerFlags(), "Not allowed to lower flags"); - - for (uint256 i = 0; i < subjects.length; i++) { - address subject = subjects[i]; - - _tryToLowerFlag(subject); - } - } - - /** - * @notice allows owner to change the access controller for raising flags. - * @param racAddress new address for the raising access controller. - */ - function setRaisingAccessController(address racAddress) public override onlyOwner { - address previous = address(raisingAccessController); - - if (previous != racAddress) { - raisingAccessController = AccessControllerInterface(racAddress); - - emit RaisingAccessControllerUpdated(previous, racAddress); - } - } - - function setLoweringAccessController(address lacAddress) public override onlyOwner { - address previous = address(loweringAccessController); - - if (previous != lacAddress) { - loweringAccessController = AccessControllerInterface(lacAddress); - - emit LoweringAccessControllerUpdated(previous, lacAddress); - } - } - - // PRIVATE - function _allowedToRaiseFlags() private view returns (bool) { - return msg.sender == owner() || raisingAccessController.hasAccess(msg.sender, msg.data); - } - - function _allowedToLowerFlags() private view returns (bool) { - return msg.sender == owner() || loweringAccessController.hasAccess(msg.sender, msg.data); - } - - function _tryToRaiseFlag(address subject) private { - if (!s_flags[subject]) { - s_flags[subject] = true; - emit FlagRaised(subject); - } - } - - function _tryToLowerFlag(address subject) private { - if (s_flags[subject]) { - s_flags[subject] = false; - emit FlagLowered(subject); - } - } -} From 67c8a5d3b81bd95bc3056e92f40c7f4386c525ca Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Fri, 2 Aug 2024 13:37:32 +0200 Subject: [PATCH 19/54] do not run forge fmt if shared contracts have changed --- .github/workflows/solidity-foundry.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index c5f14f717e7..e7ce6b9532b 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -299,31 +299,23 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout the repo - if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) - && matrix.product.setup.run-forge-fmt }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive - name: Setup NodeJS - if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) - && matrix.product.setup.run-forge-fmt }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} uses: ./.github/actions/setup-nodejs - name: Install Foundry - if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) - && matrix.product.setup.run-forge-fmt }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge fmt - if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) - && matrix.product.setup.run-forge-fmt }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} run: forge fmt --check id: fmt working-directory: contracts @@ -331,9 +323,7 @@ jobs: FOUNDRY_PROFILE: ${{ matrix.product.name }} - name: Collect Metrics - if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) - && matrix.product.setup.run-forge-fmt }} + if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: From f9b56c49267524229cc63887d9a72896e32f44fd Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Fri, 2 Aug 2024 15:17:53 +0200 Subject: [PATCH 20/54] try artifact pipeline by hijacking hardhat --- .github/workflows/solidity-hardhat.yml | 518 ++++++++++++++++-- contracts/.slither.config-artifacts.json | 0 .../scripts/ci/generate_slither_report.sh | 80 +++ contracts/scripts/ci/generate_uml.sh | 118 ++++ contracts/scripts/ci/modify_remappings.sh | 28 + contracts/scripts/ci/select_solc_version.sh | 114 ++++ 6 files changed, 803 insertions(+), 55 deletions(-) create mode 100644 contracts/.slither.config-artifacts.json create mode 100755 contracts/scripts/ci/generate_slither_report.sh create mode 100755 contracts/scripts/ci/generate_uml.sh create mode 100755 contracts/scripts/ci/modify_remappings.sh create mode 100755 contracts/scripts/ci/select_solc_version.sh diff --git a/.github/workflows/solidity-hardhat.yml b/.github/workflows/solidity-hardhat.yml index f28cf499072..b525b5d9b52 100644 --- a/.github/workflows/solidity-hardhat.yml +++ b/.github/workflows/solidity-hardhat.yml @@ -1,87 +1,495 @@ name: Solidity-Hardhat - on: - merge_group: push: + workflow_dispatch: + inputs: + product: + type: choice + description: 'product for which to generate artifacts; should be the same as Foundry profile' + required: true + options: + - "automation" + - "ccip" + - "functions" + - "keystone" + - "l2ep" + - "liquiditymanager" + - "llo-feeds" + - "operatorforwarder" + - "transmission" + - "vrf" + base_ref: + description: 'commit or tag to be used as base reference, when looking for modified Solidity files' + required: true env: - NODE_OPTIONS: --max_old_space_size=8192 - -defaults: - run: - shell: bash + FOUNDRY_PROFILE: ci + # Has to match the `make foundry` version. + FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 + PRODUCT: "functions" + BASE_REF: "v2.13.0" jobs: changes: name: Detect changes runs-on: ubuntu-latest outputs: - changes: ${{ steps.changes.outputs.src }} + changes: ${{ steps.changes.outputs.sol }} + product_changes: ${{ steps.changes.outputs.product }} + shared_changes: ${{ steps.changes.outputs.shared }} + files: ${{ steps.changes.outputs.included_files }} + changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} + changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - name: Find modified contracts + uses: AurorNZ/paths-filter@3b1f3abc3371cca888d8eb03dfa70bc8a9867629 # v4.0.0 id: changes with: + list-files: 'csv' + base: ${{ env.BASE_REF }} #TODO change to inputs.base_ref after testing + filters: | + ignored: &ignored + - '!contracts/src/v0.8/**/test/**' + - '!contracts/src/v0.8/**/tests/**' + - '!contracts/src/v0.8/**/*.t.sol' + - '!contracts/src/v0.8/*.t.sol' + - '!contracts/src/v0.8/**/testhelpers/**' + - '!contracts/src/v0.8/testhelpers/**' + - '!contracts/src/v0.8/vendor/**' + shared: &shared + - 'contracts/src/v0.8/shared/**/*.sol' + - 'contracts/src/v0.8/interfaces/**/*.sol' + - 'contracts/src/v0.8/*.sol' + - *ignored + sol: + - 'contracts/src/v0.8/**/*.sol' + - *ignored + product: &product + - 'contracts/src/v0.8/${{ env.PRODUCT }}/**/*.sol' + - *ignored + included: + - *product + - *shared + - *ignored + + - name: Find modified changesets + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes-dorny + with: + list-files: 'shell' + base: ${{ env.BASE_REF }} #TODO change to inputs.base_ref after testing filters: | - src: - - 'contracts/src/!(v0.8/(ccip|functions|keystone|l2ep|liquiditymanager|llo-feeds|transmission|vrf)/**)/**/*' - - 'contracts/test/**/*' - - 'contracts/package.json' - - 'contracts/pnpm-lock.yaml' - - 'contracts/hardhat.config.ts' - - '.github/workflows/solidity-hardhat.yml' - - hardhat-test: + changeset: + - modified|added: 'contracts/.changeset/!(README)*.md' + + - name: Check for changes outside of artifact scope + if: ${{ steps.changes.outputs.sol == 'true' }} + shell: bash + run: | + echo "::debug::All modified contracts:" + echo ${{ steps.changes.outputs.all_sol_files }} | tr ',' '\n' + echo "::debug::Filtered modified contracts:" + echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/shared/ && !/^contracts\/src\/v0\.8\/${{ env.PRODUCT}}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + echo "::debug::Excluded paths: $excluded_paths_pattern" + unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") + set -e + set -o pipefail + if [[ -n "$unexpected_files" ]]; then + products=() + productsStr="" + IFS=$'\n' read -r -d '' -a files <<< "$unexpected_files" || true + echo "Files: ${files[@]}" + + for file in "${files[@]}"; do + product=$(echo "$file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) + if [[ ! " ${products[@]} " =~ " ${product} " ]]; then + products+=("$product") + productsStr+="$product, " + fi + done + productsStr=${productsStr%, } + + set +e + set +o pipefail + + echo "Error: Found modified contracts outside of the expected scope: ${{ env.PRODUCT }}, shared" + echo "Files:" + echo "$unexpected_files" + echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" + + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found modified contracts outside of the expected scope: ${{ env.PRODUCT }}, shared" >> $GITHUB_STEP_SUMMARY + echo "### Files:" >> $GITHUB_STEP_SUMMARY + echo "$unexpected_files" >> $GITHUB_STEP_SUMMARY + echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY + else + echo "No unexpected files found." + fi + + - name: Check for changes only in shared + if: ${{ steps.changes.outputs.product == 'false' && steps.changes.outputs.shared == 'true' }} + shell: bash + run: | + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found No modified contracts for product ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY + echo "## Action required: Please check if you chose correct product and base reference" >> $GITHUB_STEP_SUMMARY + echo "## Product: ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY + #TODO change to inputs.base_ref after testing + echo "## Base Ref: ${{ env.BASE_REF }}" >> $GITHUB_STEP_SUMMARY + + gather-basic-info: + if: ${{ needs.changes.outputs.changeset_changes == 'true' }} + name: Gather basic info + runs-on: ubuntu-22.04 + needs: [ changes ] + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + fetch-depth: 0 + + - name: Copy modified changesets + run: | + mkdir -p contracts/changesets/changesets + for changeset in "${{ needs.changes.outputs.changeset_files }}"; do + echo "::debug:: Copying $changeset" + cp $changeset contracts/changesets/changesets + done + + - name: Generate basic info + shell: bash + run: | + echo "Commit SHA used to generate artifacts: ${{ github.sha }}" > contracts/commit_sha_base_ref.txt + #TODO change to inputs.base_ref after testing + echo "Base reference SHA used to find modified contracts: ${{ env.BASE_REF }}" >> contracts/commit_sha_base_ref.txt + contract_list="${{ needs.changes.outputs.files }}" + new_line_list=$(echo "$contract_list" | tr ',' '\n' | sed 's/,$//') + printf "%s\n" "$new_line_list" > contracts/modified_contracts.txt + + cat contracts/modified_contracts.txt + + - name: Generate modified contracts list + shell: bash + run: | + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" + echo "# Modified contracts:" > contracts/modified_contracts.md + for file in "${modified_files[@]}"; do + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ env.BASE_REF }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping." + continue + fi + + echo " - [$file](${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/$file)" >> contracts/modified_contracts.md + done + + - name: Upload basic info + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 + continue-on-error: true + with: + name: tmp-basic-info + path: | + contracts/modified_contracts.md + contracts/modified_contracts.txt + contracts/commit_sha_base_ref.txt + contracts/changesets + retention-days: 7 + + # some of the artifacts can only be generated on product level, and we cannot scope them to single contracts + # some product-level modifications might also require shared contracts changes, so if these happened we need to generate artifacts for shared contracts as well + coverage-and-book: + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + name: Generate Docs and Code Coverage reports + runs-on: ubuntu-22.04 needs: [changes] - if: needs.changes.outputs.changes == 'true' - name: Solidity ${{ fromJSON('["(skipped)", ""]')[needs.changes.outputs.changes == 'true'] }} - runs-on: ubuntu-latest steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + - name: Setup NodeJS uses: ./.github/actions/setup-nodejs - - name: Setup Hardhat - uses: ./.github/actions/setup-hardhat + + - name: Create directories + shell: bash + run: | + mkdir -p contracts/code-coverage + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - namespace: coverage - - name: Run tests + version: ${{ env.FOUNDRY_VERSION }} + + # required for code coverage report generation + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 + + - name: Run Forge build for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + run: | + forge --version + forge build + working-directory: contracts + env: + FOUNDRY_PROFILE: ${{ env.PRODUCT }} + + - name: Run Forge build for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + run: | + forge --version + forge build + working-directory: contracts + env: + FOUNDRY_PROFILE: "shared" + + - name: Run coverage for product contracts + if: ${{ !contains(fromJson('["vrf"]'), env.PRODUCT) && !contains(fromJson('["automation"]'), env.PRODUCT) && !contains(fromJson('["functions"]'), env.PRODUCT) && needs.changes.outputs.product_changes == 'true' }} + working-directory: contracts + run: forge coverage --report lcov --report-file code-coverage/lcov-${{ env.PRODUCT }}.info + env: + FOUNDRY_PROFILE: ${{ env.PRODUCT }} + + - name: Run coverage for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + working-directory: contracts + run: forge coverage --report lcov --report-file code-coverage/lcov-shared.info + env: + FOUNDRY_PROFILE: "shared" + + - name: Generate Code Coverage HTML report for product contracts + if: ${{ !contains(fromJson('["vrf"]'), env.PRODUCT) && !contains(fromJson('["automation"]'), env.PRODUCT) && !contains(fromJson('["functions"]'), env.PRODUCT) && needs.changes.outputs.product_changes == 'true' }} + shell: bash + run: cd contracts && genhtml code-coverage/lcov-${{ env.PRODUCT }}.info --branch-coverage --output-directory code-coverage/${{ env.PRODUCT }} + + - name: Generate Code Coverage HTML report for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + shell: bash + run: cd contracts && genhtml code-coverage/lcov-shared.info --branch-coverage --output-directory code-coverage/shared + + - name: Run Forge doc for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + run: forge doc --build -o docs/${{ env.PRODUCT }} + working-directory: contracts + env: + FOUNDRY_PROFILE: ${{ env.PRODUCT }} + + - name: Run Forge doc for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + run: forge doc --build -o docs/shared working-directory: contracts - run: pnpm test - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + env: + FOUNDRY_PROFILE: "shared" + + - name: Upload Artifacts for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 + continue-on-error: true with: - id: hardhat-test - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + name: tmp-${{ env.PRODUCT }}-artifacts + path: | + contracts/docs/${{ env.PRODUCT }} + contracts/code-coverage/lcov-${{ env.PRODUCT }}.info + contracts/code-coverage/${{ env.PRODUCT }} + retention-days: 7 + + - name: Upload Artifacts for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 continue-on-error: true + with: + name: tmp-shared-artifacts + path: | + contracts/docs/shared + contracts/code-coverage/lcov-shared.info + contracts/code-coverage/shared + retention-days: 7 - solidity: - needs: [changes, hardhat-test] - name: Solidity - runs-on: ubuntu-latest - if: always() + # Generates UML diagrams and Slither reports for modified contracts + uml-static-analysis: + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + name: Generate UML and Slither reports for modified contracts + runs-on: ubuntu-22.04 + needs: [changes] + env: + GH_TOKEN: ${{ github.token }} steps: - - run: echo 'Solidity tests finished!' - - name: Check test results + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + fetch-depth: 0 + + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 + with: + version: ${{ env.FOUNDRY_VERSION }} + + - name: Install Sol2uml run: | - if [[ "${{ needs.changes.result }}" = "failure" || "${{ needs.solidity-splits.result }}" = "failure" ]]; then - echo "One or more changes / solidity-splits jobs failed" - exit 1 - else - echo "All test jobs passed successfully" + npm link sol2uml --only=production + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Install solc-select and solc + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install solc-select + sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select + solc-select install 0.8.19 + solc-select use 0.8.19 + + - name: Install Slither + run: | + python -m pip install --upgrade pip + pip install slither-analyzer + + - name: Generate UML + shell: bash + run: | + contract_list="${{ needs.changes.outputs.files }}" + + # modify remappings so that solc can find dependencies + ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt + mv remappings_modified.txt remappings.txt + + ./contracts/scripts/ci/generate_uml.sh "./" "contracts/uml-diagrams" "$contract_list" + + - name: Generate Slither Markdown reports + run: | + contract_list="${{ needs.changes.outputs.files }}" + + echo "::debug::Processing contracts: $contract_list" + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-artifacts.json "." "$contract_list" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + + - name: Upload UMLs and Slither reports + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 10 + continue-on-error: true + with: + name: tmp-contracts-artifacts + path: | + contracts/uml-diagrams + contracts/slither-reports + retention-days: 7 + + - name: Validate UMLs and Slither reports + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + run: | + echo "Validating UMLs and Slither reports" + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" + missing_svgs=() + missing_reports=() + for file in "${modified_files[@]}"; do + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ env.BASE_REF }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + + svg_file="$(basename "${file%.sol}").svg" + if [ ! -f "contracts/uml-diagrams/$svg_file" ]; then + echo "Error: UML diagram for $file not found" + missing_svgs+=("$file") + fi + + report_file="$(basename "${file%.sol}")-slither-report.md" + if [ ! -f "contracts/slither-reports/$report_file" ]; then + echo "Error: Slither report for $file not found" + missing_reports+=("$file") + fi + done + + if [ ${#missing_svgs[@]} -gt 0 ]; then + echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_svgs[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY + else + echo "All UML diagrams generated successfully" + fi + + if [ ${#missing_reports[@]} -gt 0 ]; then + echo "Error: Missing Slither reports for files: ${missing_reports[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_reports[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY + else + echo "All Slither reports generated successfully" fi - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + + gather-all-artifacts: + name: Gather all artifacts + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + runs-on: ubuntu-latest + needs: [coverage-and-book, uml-static-analysis, gather-basic-info, changes] + steps: + - name: Download all artifacts + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - id: solidity-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Solidity - continue-on-error: true \ No newline at end of file + path: review_artifacts + merge-multiple: true + + - name: Upload all artifacts as single package + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + with: + name: review-artifacts-${{ env.PRODUCT }}-${{ github.sha }} + path: review_artifacts + + - name: Remove temporary artifacts + uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1 # v5.0 + with: + name: tmp-* + + - name: Print Artifact URL in job summary + env: + GH_TOKEN: ${{ github.token }} + run: | + ARTIFACTS=$(gh api -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts) + ARTIFACT_ID=$(echo "$ARTIFACTS" | jq '.artifacts[] | select(.name=="review-artifacts-${{ env.PRODUCT }}-${{ github.sha }}") | .id') + echo "Artifact ID: $ARTIFACT_ID" + + echo "# Solidity Review Artifact Generated" >> $GITHUB_STEP_SUMMARY + # TODO change to inputs.base_ref after testing + echo "Base Ref used: **${{ env.BASE_REF }}**" >> $GITHUB_STEP_SUMMARY + echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY + echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY + + notify-no-changes: + if: ${{ needs.changes.outputs.product_changes == 'false' && needs.changes.outputs.shared_changes == 'false' }} + needs: [changes] + runs-on: ubuntu-latest + steps: + - name: Print warning in job summary + shell: bash + run: | + echo "# Solidity Review Artifact NOT Generated" >> $GITHUB_STEP_SUMMARY + #TODO change to inputs.base_ref after testing + echo "Base Ref used: **${{ env.BASE_REF }}**" >> $GITHUB_STEP_SUMMARY + echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY + echo "## Reason: No modified Solidity files found for ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY + #TODO change to inputs.base_ref after testing + echo "No modified Solidity files found between ${{ env.BASE_REF }} and ${{ github.sha }} commits or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY + exit 1 diff --git a/contracts/.slither.config-artifacts.json b/contracts/.slither.config-artifacts.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh new file mode 100755 index 00000000000..50fc991e894 --- /dev/null +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +function check_chainlink_dir() { + local param_dir="chainlink" + current_dir=$(pwd) + + current_base=$(basename "$current_dir") + + if [ "$current_base" != "$param_dir" ]; then + >&2 echo "The script must be run from the root of $param_dir directory" + exit 1 + fi +} + +check_chainlink_dir + +if [ "$#" -lt 5 ]; then + >&2 echo "Generates Markdown Slither reports and saves them to a target directory." + >&2 echo "Usage: $0 [slither extra params]" +exit 1 +fi + +REPO_URL=$1 +CONFIG_FILE=$2 +SOURCE_DIR=$3 +FILES=${4// /} # Remove any spaces from the list of files +TARGET_DIR=$5 +SLITHER_EXTRA_PARAMS=$6 + +run_slither() { + local FILE=$1 + local TARGET_DIR=$2 + + # needed, because the action we use returns all modified files, also deleted ones and we must skip those + if [ ! -f "$FILE" ]; then + echo "Warning: File not found: $FILE" + echo "Skipping..." + return + fi + + source ./contracts/scripts/ci/select_solc_version.sh "$FILE" + if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to select Solc version for $FILE" + exit 1 + fi + + SLITHER_OUTPUT_FILE="$TARGET_DIR/$(basename "${FILE%.sol}")-slither-report.md" + + output=$(slither --config-file "$CONFIG_FILE" "$FILE" --checklist --markdown-root "$REPO_URL" --fail-none $SLITHER_EXTRA_PARAMS) + if [ $? -ne 0 ]; then + >&2 echo "Slither failed for $FILE" + exit 1 + fi + output=$(echo "$output" | sed '/\*\*THIS CHECKLIST IS NOT COMPLETE\*\*. Use `--show-ignored-findings` to show all the results./d' | sed '/Summary/d') + + echo "# Summary for $FILE" > "$SLITHER_OUTPUT_FILE" + echo "$output" >> "$SLITHER_OUTPUT_FILE" +} + +process_files() { + local SOURCE_DIR=$1 + local TARGET_DIR=$2 + local FILES=(${3//,/ }) # Split the comma-separated list into an array + + mkdir -p "$TARGET_DIR" + + for FILE in "${FILES[@]}"; do + FILE=${FILE//\"/} + run_slither "$SOURCE_DIR/$FILE" "$TARGET_DIR" + done +} + +process_files "$SOURCE_DIR" "$TARGET_DIR" "${FILES[@]}" + +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to generate Slither reports" + exit 1 +fi + +echo "Slither reports saved in $TARGET_DIR folder" diff --git a/contracts/scripts/ci/generate_uml.sh b/contracts/scripts/ci/generate_uml.sh new file mode 100755 index 00000000000..b9ce3d00ca1 --- /dev/null +++ b/contracts/scripts/ci/generate_uml.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +function check_chainlink_dir() { + local param_dir="chainlink" + current_dir=$(pwd) + + current_base=$(basename "$current_dir") + + if [ "$current_base" != "$param_dir" ]; then + >&2 echo "The script must be run from the root of $param_dir directory" + exit 1 + fi +} + +check_chainlink_dir + +if [ "$#" -lt 2 ]; then + >&2 echo "Generates UML diagrams for all contracts in a directory after flattening them to avoid call stack overflows." + >&2 echo "Usage: $0 [comma-separated list of files]" + exit 1 +fi + +SOURCE_DIR="$1" +TARGET_DIR="$2" +FILES=${3// /} # Remove any spaces from the list of files +FAILED_FILES=() + +flatten_and_generate_uml() { + local FILE=$1 + local TARGET_DIR=$2 + + FLATTENED_FILE="$TARGET_DIR/flattened_$(basename "$FILE")" + echo "::debug::Flattening $FILE to $FLATTENED_FILE" + forge flatten "$FILE" -o "$FLATTENED_FILE" --root contracts + if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to flatten $FILE" + FAILED_FILES+=("$FILE") + return + fi + + OUTPUT_FILE=${FLATTENED_FILE//"flattened_"/""} + OUTPUT_FILE_SVG="${OUTPUT_FILE%.sol}.svg" + echo "::debug::Generating SVG UML for $FLATTENED_FILE to $OUTPUT_FILE_SVG" + sol2uml "$FLATTENED_FILE" -o "$OUTPUT_FILE_SVG" + if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to generate UML diagram in SVG format for $FILE" + FAILED_FILES+=("$FILE") + rm "$FLATTENED_FILE" + return + fi + OUTPUT_FILE_DOT="${OUTPUT_FILE%.sol}.dot" + echo "::debug::Generating DOT UML for $FLATTENED_FILE to $OUTPUT_FILE_DOT" + sol2uml "$FLATTENED_FILE" -o "$OUTPUT_FILE_DOT" -f dot + if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to generate UML diagram in DOT format for $FILE" + FAILED_FILES+=("$FILE") + rm "$FLATTENED_FILE" + return + fi + + rm "$FLATTENED_FILE" +} + +flatten_contracts_in_directory() { + local SOURCE_DIR=$1 + local TARGET_DIR=$2 + + mkdir -p "$TARGET_DIR" + + find "$SOURCE_DIR" -type f -name '*.sol' | while read -r ITEM; do + flatten_and_generate_uml "$ITEM" "$TARGET_DIR" + done +} + +process_files() { + local SOURCE_DIR=$1 + local TARGET_DIR=$2 + local FILES=(${3//,/ }) # Split the comma-separated list into an array + + mkdir -p "$TARGET_DIR" + + for FILE in "${FILES[@]}"; do + FILE=${FILE//\"/} + MATCHES=($(find "$SOURCE_DIR" -type f -path "*/$FILE")) + + if [ ${#MATCHES[@]} -gt 1 ]; then + >&2 echo "Error: Multiple matches found for $FILE:" + for MATCH in "${MATCHES[@]}"; do + >&2 echo " $MATCH" + done + exit 1 + elif [ ${#MATCHES[@]} -eq 1 ]; then + >&2 echo "File found: ${MATCHES[0]}" + flatten_and_generate_uml "${MATCHES[0]}" "$TARGET_DIR" + else + # needed, because the action we use returns all modified files, also deleted ones and we must skip those + echo "File $FILE does not exist within the source directory $SOURCE_DIR." + echo "Skipping..." + continue + fi + done +} + +if [ -z "$FILES" ]; then + flatten_contracts_in_directory "$SOURCE_DIR" "$TARGET_DIR" +else + process_files "$SOURCE_DIR" "$TARGET_DIR" "$FILES" +fi + +if [ "${#FAILED_FILES[@]}" -gt 0 ]; then + >&2 echo "Error: Failed to generate UML diagrams for ${#FAILED_FILES[@]} files:" + for FILE in "${FAILED_FILES[@]}"; do + >&2 echo " $FILE" + echo "$FILE" >> "$TARGET_DIR/uml_generation_failures.txt" + done +fi + +echo "UML diagrams saved in $TARGET_DIR folder" diff --git a/contracts/scripts/ci/modify_remappings.sh b/contracts/scripts/ci/modify_remappings.sh new file mode 100755 index 00000000000..72394a2fe89 --- /dev/null +++ b/contracts/scripts/ci/modify_remappings.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + >&2 echo "Usage: $0 " + exit 1 +fi + +DIR_PREFIX=$1 +REMAPPINGS_FILE=$2 + +if [ ! -f "$REMAPPINGS_FILE" ]; then + >&2 echo "Error: Remappings file '$REMAPPINGS_FILE' not found." + exit 1 +fi + +OUTPUT_FILE="remappings_modified.txt" + +while IFS= read -r line; do + if [[ "$line" =~ ^[^=]+= ]]; then + REMAPPED_PATH="${line#*=}" + MODIFIED_LINE="${line%=*}=${DIR_PREFIX}/${REMAPPED_PATH}" + echo "$MODIFIED_LINE" >> "$OUTPUT_FILE" + else + echo "$line" >> "$OUTPUT_FILE" + fi +done < "$REMAPPINGS_FILE" + +echo "Modified remappings have been saved to: $OUTPUT_FILE" diff --git a/contracts/scripts/ci/select_solc_version.sh b/contracts/scripts/ci/select_solc_version.sh new file mode 100755 index 00000000000..316b261616a --- /dev/null +++ b/contracts/scripts/ci/select_solc_version.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +function check_chainlink_dir() { + local param_dir="chainlink" + current_dir=$(pwd) + + current_base=$(basename "$current_dir") + + if [ "$current_base" != "$param_dir" ]; then + echo "The script must be run from the root of $param_dir directory" + exit 1 + fi +} + +check_chainlink_dir + +FILE="$1" + +if [ "$#" -lt 1 ]; then + echo "Detects the Solidity version of a file and selects the appropriate Solc version." + echo "If the version is not installed, it will be installed and selected." + echo "Will prefer to use the version from Foundry profile if it satisfies the version in the file." + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$FILE" ]; then + echo "Error: File not provided." + exit 1 +fi + +extract_product() { + local path=$1 + + echo "$path" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1 +} + +extract_pragma() { + local FILE=$1 + + if [[ -f "$FILE" ]]; then + SOLCVER="$(grep --no-filename '^pragma solidity' "$FILE" | cut -d' ' -f3)" + else + echo "$FILE is not a file or it could not be found. Exiting." + return 1 + fi + SOLCVER="$(echo "$SOLCVER" | sed 's/[^0-9\.^]//g')" + >&2 echo "::debug::Detected Solidity version in pragma: $SOLCVER" + echo "$SOLCVER" +} + +echo "Detecting Solc version for $FILE" + +# Set FOUNDRY_PROFILE to the product name only if it is set; otherwise either already set value will be used or it will be empty +PRODUCT=$(extract_product "$FILE") +if [ -n "$PRODUCT" ]; then + FOUNDRY_PROFILE="$PRODUCT" +fi +SOLC_IN_PROFILE=$(forge config --json --root contracts | jq ".solc") +SOLC_IN_PROFILE=$(echo "$SOLC_IN_PROFILE" | tr -d "'\"") +echo "::debug::Detected Solidity version in profile: $SOLC_IN_PROFILE" + +SOLCVER=$(extract_pragma "$FILE") + +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "Error: Failed to extract the Solidity version from $FILE." + return 1 +fi + +SOLCVER=$(echo "$SOLCVER" | tr -d "'\"") + +if [[ "$SOLC_IN_PROFILE" != "null" && -n "$SOLCVER" ]]; then + set +e + COMPAT_SOLC_VERSION=$(npx semver "$SOLC_IN_PROFILE" -r "$SOLCVER") + exit_code=$? + set -e + if [[ $exit_code -eq 0 && -n "$COMPAT_SOLC_VERSION" ]]; then + echo "::debug::Version $SOLC_IN_PROFILE satisfies the constraint $SOLCVER" + SOLC_TO_USE="$SOLC_IN_PROFILE" + else + echo "::debug::Version $SOLC_IN_PROFILE does not satisfy the constraint $SOLCVER" + SOLC_TO_USE="$SOLCVER" + fi + elif [[ "$SOLC_IN_PROFILE" != "null" && -z "$SOLCVER" ]]; then + >&2 echo "No version found in the Solidity file. Exiting" + return 1 + elif [[ "$SOLC_IN_PROFILE" == "null" && -n "$SOLCVER" ]]; then + echo "::debug::Using the version from the file: $SOLCVER" + SOLC_TO_USE="$SOLCVER" + else + >&2 echo "No version found in the profile or the Solidity file." + return 1 +fi + +echo "Will use $SOLC_TO_USE" +SOLC_TO_USE=$(echo "$SOLC_TO_USE" | tr -d "'\"") +SOLC_TO_USE="$(echo "$SOLC_TO_USE" | sed 's/[^0-9\.]//g')" + +INSTALLED_VERSIONS=$(solc-select versions) + +if echo "$INSTALLED_VERSIONS" | grep -q "$SOLC_TO_USE"; then + echo "::debug::Version $SOLCVER is already installed." + if echo "$INSTALLED_VERSIONS" | grep "$SOLC_TO_USE" | grep -q "current"; then + echo "::debug::Version $SOLCVER is already selected." + else + echo "::debug::Selecting $SOLC_TO_USE" + solc-select use "$SOLC_TO_USE" + fi +else + echo "::debug::Version $SOLC_TO_USE is not installed." + solc-select install "$SOLC_TO_USE" + solc-select use "$SOLC_TO_USE" +fi From cb083ee2af3fd2aa6f2a524f1e0cf6d0fdada062 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 11:00:56 +0200 Subject: [PATCH 21/54] # This is a combination of 2 commits. # This is the 1st commit message: CR changes + test them # This is the commit message #2: use shell array --- .github/workflows/solidity-foundry.yml | 16 +- .../slither/.slither.config-ccip-pr.json | 4 + .../slither/.slither.config-default-pr.json | 4 + .../src/v0.8/ccip/AggregateRateLimiterNew.sol | 92 +++ .../v0.4.0/DestinationFeeManagerNew.sol | 557 ++++++++++++++++++ 5 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 contracts/configs/slither/.slither.config-ccip-pr.json create mode 100644 contracts/configs/slither/.slither.config-default-pr.json create mode 100644 contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol create mode 100644 contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index e7ce6b9532b..afbc753c6a5 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -73,7 +73,7 @@ jobs: - 'contracts/src/v0.8/l2ep/**/*.sol' liquiditymanager: - 'contracts/src/v0.8/liquiditymanager/**/*.sol' - ll-feeds: + llo-feeds: - 'contracts/src/v0.8/llo-feeds/**/*.sol' operatorforwarder: - 'contracts/src/v0.8/operatorforwarder/**/*.sol' @@ -248,14 +248,22 @@ jobs: - name: Run Slither shell: bash run: | - CHANGED="${{ needs.changes.outputs.sol_modified_files }}" - CHANGED_ARRAY=$(echo "$CHANGED" | tr ' ' ',') + CHANGED_ARRAY="${{ needs.changes.outputs.sol_modified_files }}" # modify remappings so that solc can find dependencies ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt - ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-pr.json "." "$CHANGED_ARRAY" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + for FILE in $CHANGED_ARRAY; do + PRODUCT=$(echo "$FILE" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) + echo "Running Slither for $FILE in $PRODUCT" + SLITHER_CONFIG="contracts/configs/slither/.slither.config-$PRODUCT-pr.json" + if [ ! -f $SLITHER_CONFIG ]; then + echo "No Slither config found for $PRODUCT, using default" + SLITHER_CONFIG="contracts/configs/slither/.slither.config-default-pr.json" + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" "$SLITHER_CONFIG" "." "$FILE" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + fi + done - name: Print Slither summary shell: bash diff --git a/contracts/configs/slither/.slither.config-ccip-pr.json b/contracts/configs/slither/.slither.config-ccip-pr.json new file mode 100644 index 00000000000..84d231ea07b --- /dev/null +++ b/contracts/configs/slither/.slither.config-ccip-pr.json @@ -0,0 +1,4 @@ +{ + "filter_paths": "(openzeppelin|mocks/|test/|tests/|testhelpers)", + "detectors_to_exclude": "pragma,solc-version,naming-convention,assembly,reentrancy-events,timestamp,calls-loop,unused-return" +} diff --git a/contracts/configs/slither/.slither.config-default-pr.json b/contracts/configs/slither/.slither.config-default-pr.json new file mode 100644 index 00000000000..1ef145a7954 --- /dev/null +++ b/contracts/configs/slither/.slither.config-default-pr.json @@ -0,0 +1,4 @@ +{ + "filter_paths": "(openzeppelin|mocks/|test/|tests/|testhelpers)", + "detectors_to_exclude": "pragma" +} diff --git a/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol b/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol new file mode 100644 index 00000000000..7401df2ed49 --- /dev/null +++ b/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BUSL-1.1 +pragma solidity 0.8.24; + +import {IPriceRegistry} from "./interfaces/IPriceRegistry.sol"; + +import {OwnerIsCreator} from "./../shared/access/OwnerIsCreator.sol"; +import {Client} from "./libraries/Client.sol"; +import {RateLimiter} from "./libraries/RateLimiter.sol"; +import {USDPriceWith18Decimals} from "./libraries/USDPriceWith18Decimals.sol"; + +/// @notice The aggregate rate limiter is a wrapper of the token bucket rate limiter +/// which permits rate limiting based on the aggregate value of a group of +/// token transfers, using a price registry to convert to a numeraire asset (e.g. USD). +contract AggregateRateLimiter is OwnerIsCreator { + using RateLimiter for RateLimiter.TokenBucket; + using USDPriceWith18Decimals for uint224; + + error PriceNotFoundForToken(address token); + + event AdminSet(address newAdmin); + + // The address of the token limit admin that has the same permissions as the owner. + address internal s_admin; + + // The token bucket object that contains the bucket state. + RateLimiter.TokenBucket private s_rateLimiter; + + /// @param config The RateLimiter.Config + constructor(RateLimiter.Config memory config) { + s_rateLimiter = RateLimiter.TokenBucket({ + rate: config.rate, + capacity: config.capacity, + tokens: config.capacity, + lastUpdated: uint32(block.timestamp), + isEnabled: config.isEnabled + }); + } + + /// @notice Consumes value from the rate limiter bucket based on the token value given. + function _rateLimitValue(uint256 value) internal { + s_rateLimiter._consume(value, address(0)); + } + + function _getTokenValue( + Client.EVMTokenAmount memory tokenAmount, + IPriceRegistry priceRegistry + ) internal view returns (uint256) { + // not fetching validated price, as price staleness is not important for value-based rate limiting + // we only need to verify the price is not 0 + uint224 pricePerToken = priceRegistry.getTokenPrice(tokenAmount.token).value; + if (pricePerToken == 0) revert PriceNotFoundForToken(tokenAmount.token); + return pricePerToken._calcUSDValueFromTokenAmount(tokenAmount.amount); + } + + /// @notice Gets the token bucket with its values for the block it was requested at. + /// @return The token bucket. + function currentRateLimiterState() external view returns (RateLimiter.TokenBucket memory) { + return s_rateLimiter._currentTokenBucketState(); + } + + /// @notice Sets the rate limited config. + /// @param config The new rate limiter config. + /// @dev should only be callable by the owner or token limit admin. + function setRateLimiterConfig(RateLimiter.Config memory config) external onlyAdminOrOwner { + s_rateLimiter._setTokenBucketConfig(config); + } + + // ================================================================ + // │ Access │ + // ================================================================ + + /// @notice Gets the token limit admin address. + /// @return the token limit admin address. + function getTokenLimitAdmin() external view returns (address) { + return s_admin; + } + + /// @notice Sets the token limit admin address. + /// @param newAdmin the address of the new admin. + /// @dev setting this to address(0) indicates there is no active admin. + function setAdmin(address newAdmin) external onlyAdminOrOwner { + s_admin = newAdmin; + emit AdminSet(newAdmin); + } + + /// @notice a modifier that allows the owner or the s_tokenLimitAdmin call the functions + /// it is applied to. + modifier onlyAdminOrOwner() { + if (msg.sender != owner() && msg.sender != s_admin) revert RateLimiter.OnlyCallableByAdminOrOwner(); + _; + } +} diff --git a/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol b/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol new file mode 100644 index 00000000000..38d93de5cb7 --- /dev/null +++ b/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: MIT +pragma solidity 0.8.19; + +import {ConfirmedOwner} from "../../shared/access/ConfirmedOwner.sol"; +import {TypeAndVersionInterface} from "../../interfaces/TypeAndVersionInterface.sol"; +import {IERC165} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC165.sol"; +import {Common} from "../libraries/Common.sol"; +import {IWERC20} from "../../shared/interfaces/IWERC20.sol"; +import {IERC20} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC20.sol"; +import {Math} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/utils/math/Math.sol"; +import {SafeERC20} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/token/ERC20/utils/SafeERC20.sol"; +import {IDestinationRewardManager} from "./interfaces/IDestinationRewardManager.sol"; +import {IDestinationFeeManager} from "./interfaces/IDestinationFeeManager.sol"; + +/** + * @title FeeManager + * @author Michael Fletcher + * @author Austin Born + * @notice This contract is used for the handling of fees required for users verifying reports. + */ +contract DestinationFeeManager is IDestinationFeeManager, ConfirmedOwner, TypeAndVersionInterface { + using SafeERC20 for IERC20; + + /// @notice list of subscribers and their discounts subscriberDiscounts[subscriber][feedId][token] + mapping(address => mapping(bytes32 => mapping(address => uint256))) public s_subscriberDiscounts; + + /// @notice keep track of any subsidised link that is owed to the reward manager. + mapping(bytes32 => uint256) public s_linkDeficit; + + /// @notice the total discount that can be applied to a fee, 1e18 = 100% discount + uint64 private constant PERCENTAGE_SCALAR = 1e18; + + /// @notice the LINK token address + address public immutable i_linkAddress; + + /// @notice the native token address + address public immutable i_nativeAddress; + + /// @notice the verifier address + mapping(address => address) public s_verifierAddressList; + + /// @notice the reward manager address + IDestinationRewardManager public i_rewardManager; + + // @notice the mask to apply to get the report version + bytes32 private constant REPORT_VERSION_MASK = 0xffff000000000000000000000000000000000000000000000000000000000000; + + // @notice the different report versions + bytes32 private constant REPORT_V1 = 0x0001000000000000000000000000000000000000000000000000000000000000; + + /// @notice the surcharge fee to be paid if paying in native + uint256 public s_nativeSurcharge; + + /// @notice the error thrown if the discount or surcharge is invalid + error InvalidSurcharge(); + + /// @notice the error thrown if the discount is invalid + error InvalidDiscount(); + + /// @notice the error thrown if the address is invalid + error InvalidAddress(); + + /// @notice thrown if msg.value is supplied with a bad quote + error InvalidDeposit(); + + /// @notice thrown if a report has expired + error ExpiredReport(); + + /// @notice thrown if a report has no quote + error InvalidQuote(); + + // @notice thrown when the caller is not authorized + error Unauthorized(); + + // @notice thrown when trying to clear a zero deficit + error ZeroDeficit(); + + /// @notice thrown when trying to pay an address that cannot except funds + error InvalidReceivingAddress(); + + /// @notice thrown when trying to bulk verify reports where theres not a matching number of poolIds + error PoolIdMismatch(); + + /// @notice Emitted whenever a subscriber's discount is updated + /// @param subscriber address of the subscriber to update discounts for + /// @param feedId Feed ID for the discount + /// @param token Token address for the discount + /// @param discount Discount to apply, in relation to the PERCENTAGE_SCALAR + event SubscriberDiscountUpdated(address indexed subscriber, bytes32 indexed feedId, address token, uint64 discount); + + /// @notice Emitted when updating the native surcharge + /// @param newSurcharge Surcharge amount to apply relative to PERCENTAGE_SCALAR + event NativeSurchargeUpdated(uint64 newSurcharge); + + /// @notice Emits when this contract does not have enough LINK to send to the reward manager when paying in native + /// @param rewards Config digest and link fees which could not be subsidised + event InsufficientLink(IDestinationRewardManager.FeePayment[] rewards); + + /// @notice Emitted when funds are withdrawn + /// @param adminAddress Address of the admin + /// @param recipient Address of the recipient + /// @param assetAddress Address of the asset withdrawn + /// @param quantity Amount of the asset withdrawn + event Withdraw(address adminAddress, address recipient, address assetAddress, uint192 quantity); + + /// @notice Emits when a deficit has been cleared for a particular config digest + /// @param configDigest Config digest of the deficit cleared + /// @param linkQuantity Amount of LINK required to pay the deficit + event LinkDeficitCleared(bytes32 indexed configDigest, uint256 linkQuantity); + + /// @notice Emits when a fee has been processed + /// @param configDigest Config digest of the fee processed + /// @param subscriber Address of the subscriber who paid the fee + /// @param fee Fee paid + /// @param reward Reward paid + /// @param appliedDiscount Discount applied to the fee + event DiscountApplied( + bytes32 indexed configDigest, + address indexed subscriber, + Common.Asset fee, + Common.Asset reward, + uint256 appliedDiscount + ); + + /** + * @notice Construct the FeeManager contract + * @param _linkAddress The address of the LINK token + * @param _nativeAddress The address of the wrapped ERC-20 version of the native token (represents fee in native or wrapped) + * @param _verifierAddress The address of the verifier contract + * @param _rewardManagerAddress The address of the reward manager contract + */ + constructor( + address _linkAddress, + address _nativeAddress, + address _verifierAddress, + address _rewardManagerAddress + ) ConfirmedOwner(msg.sender) { + if ( + _linkAddress == address(0) || + _nativeAddress == address(0) || + _verifierAddress == address(0) || + _rewardManagerAddress == address(0) + ) revert InvalidAddress(); + + i_linkAddress = _linkAddress; + i_nativeAddress = _nativeAddress; + s_verifierAddressList[_verifierAddress] = _verifierAddress; + i_rewardManager = IDestinationRewardManager(_rewardManagerAddress); + + IERC20(i_linkAddress).approve(address(i_rewardManager), type(uint256).max); + } + + modifier onlyOwnerOrVerifier() { + if (msg.sender != s_verifierAddressList[msg.sender] && msg.sender != owner()) revert Unauthorized(); + _; + } + + modifier onlyVerifier() { + if (msg.sender != s_verifierAddressList[msg.sender]) revert Unauthorized(); + _; + } + + /// @inheritdoc TypeAndVersionInterface + function typeAndVersion() external pure override returns (string memory) { + return "DestinationFeeManager 1.0.0"; + } + + /// @inheritdoc IERC165 + function supportsInterface(bytes4 interfaceId) external pure override returns (bool) { + //for each function in IDestinationFeeManager we need to check if it matches the selector + return + interfaceId == this.getFeeAndReward.selector || + interfaceId == this.setNativeSurcharge.selector || + interfaceId == this.updateSubscriberDiscount.selector || + interfaceId == this.withdraw.selector || + interfaceId == this.linkAvailableForPayment.selector || + interfaceId == this.payLinkDeficit.selector || + interfaceId == this.addVerifier.selector || + interfaceId == this.removeVerifier.selector || + interfaceId == this.processFee.selector || + interfaceId == this.processFeeBulk.selector || + interfaceId == this.setFeeRecipients.selector; + } + + function processFee( + bytes32 recipient, + bytes calldata payload, + bytes calldata parameterPayload, + address subscriber + ) external payable override onlyVerifier { + (Common.Asset memory fee, Common.Asset memory reward, uint256 appliedDiscount) = _calculateFee( + payload, + parameterPayload, + subscriber + ); + + if (fee.amount == 0) { + _tryReturnChange(subscriber, msg.value); + return; + } + + IDestinationFeeManager.FeeAndReward[] memory feeAndReward = new IDestinationFeeManager.FeeAndReward[](1); + feeAndReward[0] = IDestinationFeeManager.FeeAndReward(recipient, fee, reward, appliedDiscount); + + if (fee.assetAddress == i_linkAddress) { + _handleFeesAndRewards(subscriber, feeAndReward, 1, 0); + } else { + _handleFeesAndRewards(subscriber, feeAndReward, 0, 1); + } + } + + /// @inheritdoc IDestinationFeeManager + function processFeeBulk( + bytes32[] memory poolIds, + bytes[] calldata payloads, + bytes calldata parameterPayload, + address subscriber + ) external payable override onlyVerifier { + //poolIDs are mapped to payloads, so they should be the same length + if (poolIds.length != payloads.length) revert PoolIdMismatch(); + + IDestinationFeeManager.FeeAndReward[] memory feesAndRewards = new IDestinationFeeManager.FeeAndReward[]( + payloads.length + ); + + //keep track of the number of fees to prevent over initialising the FeePayment array within _convertToLinkAndNativeFees + uint256 numberOfLinkFees; + uint256 numberOfNativeFees; + + uint256 feesAndRewardsIndex; + for (uint256 i; i < payloads.length; ++i) { + if (poolIds[i] == bytes32(0)) revert InvalidAddress(); + + (Common.Asset memory fee, Common.Asset memory reward, uint256 appliedDiscount) = _calculateFee( + payloads[i], + parameterPayload, + subscriber + ); + + if (fee.amount != 0) { + feesAndRewards[feesAndRewardsIndex++] = IDestinationFeeManager.FeeAndReward( + poolIds[i], + fee, + reward, + appliedDiscount + ); + + unchecked { + //keep track of some tallys to make downstream calculations more efficient + if (fee.assetAddress == i_linkAddress) { + ++numberOfLinkFees; + } else { + ++numberOfNativeFees; + } + } + } + } + + if (numberOfLinkFees != 0 || numberOfNativeFees != 0) { + _handleFeesAndRewards(subscriber, feesAndRewards, numberOfLinkFees, numberOfNativeFees); + } else { + _tryReturnChange(subscriber, msg.value); + } + } + + /// @inheritdoc IDestinationFeeManager + function getFeeAndReward( + address subscriber, + bytes memory report, + address quoteAddress + ) public view returns (Common.Asset memory, Common.Asset memory, uint256) { + Common.Asset memory fee; + Common.Asset memory reward; + + //get the feedId from the report + bytes32 feedId = bytes32(report); + + //the report needs to be a support version + bytes32 reportVersion = _getReportVersion(feedId); + + //version 1 of the reports don't require quotes, so the fee will be 0 + if (reportVersion == REPORT_V1) { + fee.assetAddress = i_nativeAddress; + reward.assetAddress = i_linkAddress; + return (fee, reward, 0); + } + + //verify the quote payload is a supported token + if (quoteAddress != i_nativeAddress && quoteAddress != i_linkAddress) { + revert InvalidQuote(); + } + + //decode the report depending on the version + uint256 linkQuantity; + uint256 nativeQuantity; + uint256 expiresAt; + (, , , nativeQuantity, linkQuantity, expiresAt) = abi.decode( + report, + (bytes32, uint32, uint32, uint192, uint192, uint32) + ); + + //read the timestamp bytes from the report data and verify it has not expired + if (expiresAt < block.timestamp) { + revert ExpiredReport(); + } + + //get the discount being applied + uint256 discount = s_subscriberDiscounts[subscriber][feedId][quoteAddress]; + + //the reward is always set in LINK + reward.assetAddress = i_linkAddress; + reward.amount = Math.ceilDiv(linkQuantity * (PERCENTAGE_SCALAR - discount), PERCENTAGE_SCALAR); + + //calculate either the LINK fee or native fee if it's within the report + if (quoteAddress == i_linkAddress) { + fee.assetAddress = i_linkAddress; + fee.amount = reward.amount; + } else { + uint256 surchargedFee = Math.ceilDiv(nativeQuantity * (PERCENTAGE_SCALAR + s_nativeSurcharge), PERCENTAGE_SCALAR); + + fee.assetAddress = i_nativeAddress; + fee.amount = Math.ceilDiv(surchargedFee * (PERCENTAGE_SCALAR - discount), PERCENTAGE_SCALAR); + } + + //return the fee + return (fee, reward, discount); + } + + /// @inheritdoc IDestinationFeeManager + function setFeeRecipients( + bytes32 configDigest, + Common.AddressAndWeight[] calldata rewardRecipientAndWeights + ) external onlyOwnerOrVerifier { + i_rewardManager.setRewardRecipients(configDigest, rewardRecipientAndWeights); + } + + /// @inheritdoc IDestinationFeeManager + function setNativeSurcharge(uint64 surcharge) external onlyOwner { + if (surcharge > PERCENTAGE_SCALAR) revert InvalidSurcharge(); + + s_nativeSurcharge = surcharge; + + emit NativeSurchargeUpdated(surcharge); + } + + /// @inheritdoc IDestinationFeeManager + function updateSubscriberDiscount( + address subscriber, + bytes32 feedId, + address token, + uint64 discount + ) external onlyOwner { + //make sure the discount is not greater than the total discount that can be applied + if (discount > PERCENTAGE_SCALAR) revert InvalidDiscount(); + //make sure the token is either LINK or native + if (token != i_linkAddress && token != i_nativeAddress) revert InvalidAddress(); + + s_subscriberDiscounts[subscriber][feedId][token] = discount; + + emit SubscriberDiscountUpdated(subscriber, feedId, token, discount); + } + + /// @inheritdoc IDestinationFeeManager + function withdraw(address assetAddress, address recipient, uint192 quantity) external onlyOwner { + //address 0 is used to withdraw native in the context of withdrawing + if (assetAddress == address(0)) { + (bool success, ) = payable(recipient).call{value: quantity}(""); + + if (!success) revert InvalidReceivingAddress(); + return; + } + + //withdraw the requested asset + IERC20(assetAddress).safeTransfer(recipient, quantity); + + //emit event when funds are withdrawn + emit Withdraw(msg.sender, recipient, assetAddress, uint192(quantity)); + } + + /// @inheritdoc IDestinationFeeManager + function linkAvailableForPayment() external view returns (uint256) { + //return the amount of LINK this contact has available to pay rewards + return IERC20(i_linkAddress).balanceOf(address(this)); + } + + /** + * @notice Gets the current version of the report that is encoded as the last two bytes of the feed + * @param feedId feed id to get the report version for + */ + function _getReportVersion(bytes32 feedId) internal pure returns (bytes32) { + return REPORT_VERSION_MASK & feedId; + } + + function _calculateFee( + bytes calldata payload, + bytes calldata parameterPayload, + address subscriber + ) internal view returns (Common.Asset memory, Common.Asset memory, uint256) { + if (subscriber == address(this)) revert InvalidAddress(); + + //decode the report from the payload + (, bytes memory report) = abi.decode(payload, (bytes32[3], bytes)); + + //get the feedId from the report + bytes32 feedId = bytes32(report); + + //v1 doesn't need a quote payload, so skip the decoding + address quote; + if (_getReportVersion(feedId) != REPORT_V1) { + //decode the quote from the bytes + (quote) = abi.decode(parameterPayload, (address)); + } + + //decode the fee, it will always be native or LINK + return getFeeAndReward(subscriber, report, quote); + } + + function _handleFeesAndRewards( + address subscriber, + IDestinationFeeManager.FeeAndReward[] memory feesAndRewards, + uint256 numberOfLinkFees, + uint256 numberOfNativeFees + ) internal { + IDestinationRewardManager.FeePayment[] memory linkRewards = new IDestinationRewardManager.FeePayment[]( + numberOfLinkFees + ); + IDestinationRewardManager.FeePayment[] memory nativeFeeLinkRewards = new IDestinationRewardManager.FeePayment[]( + numberOfNativeFees + ); + + uint256 totalNativeFee; + uint256 totalNativeFeeLinkValue; + + uint256 linkRewardsIndex; + uint256 nativeFeeLinkRewardsIndex; + + uint256 totalNumberOfFees = numberOfLinkFees + numberOfNativeFees; + for (uint256 i; i < totalNumberOfFees; ++i) { + if (feesAndRewards[i].fee.assetAddress == i_linkAddress) { + linkRewards[linkRewardsIndex++] = IDestinationRewardManager.FeePayment( + feesAndRewards[i].configDigest, + uint192(feesAndRewards[i].reward.amount) + ); + } else { + nativeFeeLinkRewards[nativeFeeLinkRewardsIndex++] = IDestinationRewardManager.FeePayment( + feesAndRewards[i].configDigest, + uint192(feesAndRewards[i].reward.amount) + ); + totalNativeFee += feesAndRewards[i].fee.amount; + totalNativeFeeLinkValue += feesAndRewards[i].reward.amount; + } + + if (feesAndRewards[i].appliedDiscount != 0) { + emit DiscountApplied( + feesAndRewards[i].configDigest, + subscriber, + feesAndRewards[i].fee, + feesAndRewards[i].reward, + feesAndRewards[i].appliedDiscount + ); + } + } + + //keep track of change in case of any over payment + uint256 change; + + if (msg.value != 0) { + //there must be enough to cover the fee + if (totalNativeFee > msg.value) revert InvalidDeposit(); + + //wrap the amount required to pay the fee & approve as the subscriber paid in wrapped native + IWERC20(i_nativeAddress).deposit{value: totalNativeFee}(); + + unchecked { + //msg.value is always >= to fee.amount + change = msg.value - totalNativeFee; + } + } else { + if (totalNativeFee != 0) { + //subscriber has paid in wrapped native, so transfer the native to this contract + IERC20(i_nativeAddress).safeTransferFrom(subscriber, address(this), totalNativeFee); + } + } + + if (linkRewards.length != 0) { + i_rewardManager.onFeePaid(linkRewards, subscriber); + } + + if (nativeFeeLinkRewards.length != 0) { + //distribute subsidised fees paid in Native + if (totalNativeFeeLinkValue > IERC20(i_linkAddress).balanceOf(address(this))) { + // If not enough LINK on this contract to forward for rewards, tally the deficit to be paid by out-of-band LINK + for (uint256 i; i < nativeFeeLinkRewards.length; ++i) { + unchecked { + //we have previously tallied the fees, any overflows would have already reverted + s_linkDeficit[nativeFeeLinkRewards[i].poolId] += nativeFeeLinkRewards[i].amount; + } + } + + emit InsufficientLink(nativeFeeLinkRewards); + } else { + //distribute the fees + i_rewardManager.onFeePaid(nativeFeeLinkRewards, address(this)); + } + } + + // a refund may be needed if the payee has paid in excess of the fee + _tryReturnChange(subscriber, change); + } + + function _tryReturnChange(address subscriber, uint256 quantity) internal { + if (quantity != 0) { + payable(subscriber).transfer(quantity); + } + } + + /// @inheritdoc IDestinationFeeManager + function payLinkDeficit(bytes32 configDigest) external onlyOwner { + uint256 deficit = s_linkDeficit[configDigest]; + + if (deficit == 0) revert ZeroDeficit(); + + delete s_linkDeficit[configDigest]; + + IDestinationRewardManager.FeePayment[] memory deficitFeePayment = new IDestinationRewardManager.FeePayment[](1); + + deficitFeePayment[0] = IDestinationRewardManager.FeePayment(configDigest, uint192(deficit)); + + i_rewardManager.onFeePaid(deficitFeePayment, address(this)); + + emit LinkDeficitCleared(configDigest, deficit); + } + + /// @inheritdoc IDestinationFeeManager + function addVerifier(address verifierAddress) external onlyOwner { + if (verifierAddress == address(0)) revert InvalidAddress(); + //check doesn't already exist + if (s_verifierAddressList[verifierAddress] != address(0)) revert InvalidAddress(); + s_verifierAddressList[verifierAddress] = verifierAddress; + } + + /// @inheritdoc IDestinationFeeManager + function removeVerifier(address verifierAddress) external onlyOwner { + if (verifierAddress == address(0)) revert InvalidAddress(); + //check doesn't already exist + if (s_verifierAddressList[verifierAddress] == address(0)) revert InvalidAddress(); + delete s_verifierAddressList[verifierAddress]; + } + + /// @inheritdoc IDestinationFeeManager + function setRewardManager(address rewardManagerAddress) external onlyOwner { + if (rewardManagerAddress == address(0)) revert InvalidAddress(); + IERC20(i_linkAddress).approve(address(i_rewardManager), 0); + i_rewardManager = IDestinationRewardManager(rewardManagerAddress); + IERC20(i_linkAddress).approve(address(i_rewardManager), type(uint256).max); + } +} From 9ecb1560b8a3e1b641235b68d656ab09be9c0995 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 11:21:34 +0200 Subject: [PATCH 22/54] CR changes + test them use shell array init array CR changes + test them use shell array init array --- .github/workflows/solidity-foundry.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index afbc753c6a5..a7f1c6fb3fe 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -247,23 +247,24 @@ jobs: - name: Run Slither shell: bash - run: | - CHANGED_ARRAY="${{ needs.changes.outputs.sol_modified_files }}" - + run: | # modify remappings so that solc can find dependencies ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt + + # IFS=',' + FILES="${{ needs.changes.outputs.sol_modified_files }}" - for FILE in $CHANGED_ARRAY; do + for FILE in $FILES; do PRODUCT=$(echo "$FILE" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) echo "Running Slither for $FILE in $PRODUCT" SLITHER_CONFIG="contracts/configs/slither/.slither.config-$PRODUCT-pr.json" if [ ! -f $SLITHER_CONFIG ]; then echo "No Slither config found for $PRODUCT, using default" SLITHER_CONFIG="contracts/configs/slither/.slither.config-default-pr.json" - ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" "$SLITHER_CONFIG" "." "$FILE" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" fi - done + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" "$SLITHER_CONFIG" "." "$FILE" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + done - name: Print Slither summary shell: bash @@ -335,9 +336,9 @@ jobs: id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: - id: solidity-foundry-fmt-${{ matrix.product.name }} + id: solidity-forge-fmt-${{ matrix.product.name }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Foundry fmt ${{ matrix.product.name }} + this-job-name: Forge fmt ${{ matrix.product.name }} continue-on-error: true From 7eedda5298490ae774715dc7fcd1459f5accb069 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 11:39:29 +0200 Subject: [PATCH 23/54] remove test files --- .github/workflows/solidity-foundry.yml | 5 +- .../src/v0.8/ccip/AggregateRateLimiterNew.sol | 92 --- .../v0.4.0/DestinationFeeManagerNew.sol | 557 ------------------ 3 files changed, 2 insertions(+), 652 deletions(-) delete mode 100644 contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol delete mode 100644 contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index a7f1c6fb3fe..be3963259b3 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -252,15 +252,14 @@ jobs: ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt - # IFS=',' FILES="${{ needs.changes.outputs.sol_modified_files }}" for FILE in $FILES; do PRODUCT=$(echo "$FILE" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) - echo "Running Slither for $FILE in $PRODUCT" + echo "::debug::Running Slither for $FILE in $PRODUCT" SLITHER_CONFIG="contracts/configs/slither/.slither.config-$PRODUCT-pr.json" if [ ! -f $SLITHER_CONFIG ]; then - echo "No Slither config found for $PRODUCT, using default" + echo "::debug::No Slither config found for $PRODUCT, using default" SLITHER_CONFIG="contracts/configs/slither/.slither.config-default-pr.json" fi ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" "$SLITHER_CONFIG" "." "$FILE" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" diff --git a/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol b/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol deleted file mode 100644 index 7401df2ed49..00000000000 --- a/contracts/src/v0.8/ccip/AggregateRateLimiterNew.sol +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 -pragma solidity 0.8.24; - -import {IPriceRegistry} from "./interfaces/IPriceRegistry.sol"; - -import {OwnerIsCreator} from "./../shared/access/OwnerIsCreator.sol"; -import {Client} from "./libraries/Client.sol"; -import {RateLimiter} from "./libraries/RateLimiter.sol"; -import {USDPriceWith18Decimals} from "./libraries/USDPriceWith18Decimals.sol"; - -/// @notice The aggregate rate limiter is a wrapper of the token bucket rate limiter -/// which permits rate limiting based on the aggregate value of a group of -/// token transfers, using a price registry to convert to a numeraire asset (e.g. USD). -contract AggregateRateLimiter is OwnerIsCreator { - using RateLimiter for RateLimiter.TokenBucket; - using USDPriceWith18Decimals for uint224; - - error PriceNotFoundForToken(address token); - - event AdminSet(address newAdmin); - - // The address of the token limit admin that has the same permissions as the owner. - address internal s_admin; - - // The token bucket object that contains the bucket state. - RateLimiter.TokenBucket private s_rateLimiter; - - /// @param config The RateLimiter.Config - constructor(RateLimiter.Config memory config) { - s_rateLimiter = RateLimiter.TokenBucket({ - rate: config.rate, - capacity: config.capacity, - tokens: config.capacity, - lastUpdated: uint32(block.timestamp), - isEnabled: config.isEnabled - }); - } - - /// @notice Consumes value from the rate limiter bucket based on the token value given. - function _rateLimitValue(uint256 value) internal { - s_rateLimiter._consume(value, address(0)); - } - - function _getTokenValue( - Client.EVMTokenAmount memory tokenAmount, - IPriceRegistry priceRegistry - ) internal view returns (uint256) { - // not fetching validated price, as price staleness is not important for value-based rate limiting - // we only need to verify the price is not 0 - uint224 pricePerToken = priceRegistry.getTokenPrice(tokenAmount.token).value; - if (pricePerToken == 0) revert PriceNotFoundForToken(tokenAmount.token); - return pricePerToken._calcUSDValueFromTokenAmount(tokenAmount.amount); - } - - /// @notice Gets the token bucket with its values for the block it was requested at. - /// @return The token bucket. - function currentRateLimiterState() external view returns (RateLimiter.TokenBucket memory) { - return s_rateLimiter._currentTokenBucketState(); - } - - /// @notice Sets the rate limited config. - /// @param config The new rate limiter config. - /// @dev should only be callable by the owner or token limit admin. - function setRateLimiterConfig(RateLimiter.Config memory config) external onlyAdminOrOwner { - s_rateLimiter._setTokenBucketConfig(config); - } - - // ================================================================ - // │ Access │ - // ================================================================ - - /// @notice Gets the token limit admin address. - /// @return the token limit admin address. - function getTokenLimitAdmin() external view returns (address) { - return s_admin; - } - - /// @notice Sets the token limit admin address. - /// @param newAdmin the address of the new admin. - /// @dev setting this to address(0) indicates there is no active admin. - function setAdmin(address newAdmin) external onlyAdminOrOwner { - s_admin = newAdmin; - emit AdminSet(newAdmin); - } - - /// @notice a modifier that allows the owner or the s_tokenLimitAdmin call the functions - /// it is applied to. - modifier onlyAdminOrOwner() { - if (msg.sender != owner() && msg.sender != s_admin) revert RateLimiter.OnlyCallableByAdminOrOwner(); - _; - } -} diff --git a/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol b/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol deleted file mode 100644 index 38d93de5cb7..00000000000 --- a/contracts/src/v0.8/llo-feeds/v0.4.0/DestinationFeeManagerNew.sol +++ /dev/null @@ -1,557 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity 0.8.19; - -import {ConfirmedOwner} from "../../shared/access/ConfirmedOwner.sol"; -import {TypeAndVersionInterface} from "../../interfaces/TypeAndVersionInterface.sol"; -import {IERC165} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC165.sol"; -import {Common} from "../libraries/Common.sol"; -import {IWERC20} from "../../shared/interfaces/IWERC20.sol"; -import {IERC20} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC20.sol"; -import {Math} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/utils/math/Math.sol"; -import {SafeERC20} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/token/ERC20/utils/SafeERC20.sol"; -import {IDestinationRewardManager} from "./interfaces/IDestinationRewardManager.sol"; -import {IDestinationFeeManager} from "./interfaces/IDestinationFeeManager.sol"; - -/** - * @title FeeManager - * @author Michael Fletcher - * @author Austin Born - * @notice This contract is used for the handling of fees required for users verifying reports. - */ -contract DestinationFeeManager is IDestinationFeeManager, ConfirmedOwner, TypeAndVersionInterface { - using SafeERC20 for IERC20; - - /// @notice list of subscribers and their discounts subscriberDiscounts[subscriber][feedId][token] - mapping(address => mapping(bytes32 => mapping(address => uint256))) public s_subscriberDiscounts; - - /// @notice keep track of any subsidised link that is owed to the reward manager. - mapping(bytes32 => uint256) public s_linkDeficit; - - /// @notice the total discount that can be applied to a fee, 1e18 = 100% discount - uint64 private constant PERCENTAGE_SCALAR = 1e18; - - /// @notice the LINK token address - address public immutable i_linkAddress; - - /// @notice the native token address - address public immutable i_nativeAddress; - - /// @notice the verifier address - mapping(address => address) public s_verifierAddressList; - - /// @notice the reward manager address - IDestinationRewardManager public i_rewardManager; - - // @notice the mask to apply to get the report version - bytes32 private constant REPORT_VERSION_MASK = 0xffff000000000000000000000000000000000000000000000000000000000000; - - // @notice the different report versions - bytes32 private constant REPORT_V1 = 0x0001000000000000000000000000000000000000000000000000000000000000; - - /// @notice the surcharge fee to be paid if paying in native - uint256 public s_nativeSurcharge; - - /// @notice the error thrown if the discount or surcharge is invalid - error InvalidSurcharge(); - - /// @notice the error thrown if the discount is invalid - error InvalidDiscount(); - - /// @notice the error thrown if the address is invalid - error InvalidAddress(); - - /// @notice thrown if msg.value is supplied with a bad quote - error InvalidDeposit(); - - /// @notice thrown if a report has expired - error ExpiredReport(); - - /// @notice thrown if a report has no quote - error InvalidQuote(); - - // @notice thrown when the caller is not authorized - error Unauthorized(); - - // @notice thrown when trying to clear a zero deficit - error ZeroDeficit(); - - /// @notice thrown when trying to pay an address that cannot except funds - error InvalidReceivingAddress(); - - /// @notice thrown when trying to bulk verify reports where theres not a matching number of poolIds - error PoolIdMismatch(); - - /// @notice Emitted whenever a subscriber's discount is updated - /// @param subscriber address of the subscriber to update discounts for - /// @param feedId Feed ID for the discount - /// @param token Token address for the discount - /// @param discount Discount to apply, in relation to the PERCENTAGE_SCALAR - event SubscriberDiscountUpdated(address indexed subscriber, bytes32 indexed feedId, address token, uint64 discount); - - /// @notice Emitted when updating the native surcharge - /// @param newSurcharge Surcharge amount to apply relative to PERCENTAGE_SCALAR - event NativeSurchargeUpdated(uint64 newSurcharge); - - /// @notice Emits when this contract does not have enough LINK to send to the reward manager when paying in native - /// @param rewards Config digest and link fees which could not be subsidised - event InsufficientLink(IDestinationRewardManager.FeePayment[] rewards); - - /// @notice Emitted when funds are withdrawn - /// @param adminAddress Address of the admin - /// @param recipient Address of the recipient - /// @param assetAddress Address of the asset withdrawn - /// @param quantity Amount of the asset withdrawn - event Withdraw(address adminAddress, address recipient, address assetAddress, uint192 quantity); - - /// @notice Emits when a deficit has been cleared for a particular config digest - /// @param configDigest Config digest of the deficit cleared - /// @param linkQuantity Amount of LINK required to pay the deficit - event LinkDeficitCleared(bytes32 indexed configDigest, uint256 linkQuantity); - - /// @notice Emits when a fee has been processed - /// @param configDigest Config digest of the fee processed - /// @param subscriber Address of the subscriber who paid the fee - /// @param fee Fee paid - /// @param reward Reward paid - /// @param appliedDiscount Discount applied to the fee - event DiscountApplied( - bytes32 indexed configDigest, - address indexed subscriber, - Common.Asset fee, - Common.Asset reward, - uint256 appliedDiscount - ); - - /** - * @notice Construct the FeeManager contract - * @param _linkAddress The address of the LINK token - * @param _nativeAddress The address of the wrapped ERC-20 version of the native token (represents fee in native or wrapped) - * @param _verifierAddress The address of the verifier contract - * @param _rewardManagerAddress The address of the reward manager contract - */ - constructor( - address _linkAddress, - address _nativeAddress, - address _verifierAddress, - address _rewardManagerAddress - ) ConfirmedOwner(msg.sender) { - if ( - _linkAddress == address(0) || - _nativeAddress == address(0) || - _verifierAddress == address(0) || - _rewardManagerAddress == address(0) - ) revert InvalidAddress(); - - i_linkAddress = _linkAddress; - i_nativeAddress = _nativeAddress; - s_verifierAddressList[_verifierAddress] = _verifierAddress; - i_rewardManager = IDestinationRewardManager(_rewardManagerAddress); - - IERC20(i_linkAddress).approve(address(i_rewardManager), type(uint256).max); - } - - modifier onlyOwnerOrVerifier() { - if (msg.sender != s_verifierAddressList[msg.sender] && msg.sender != owner()) revert Unauthorized(); - _; - } - - modifier onlyVerifier() { - if (msg.sender != s_verifierAddressList[msg.sender]) revert Unauthorized(); - _; - } - - /// @inheritdoc TypeAndVersionInterface - function typeAndVersion() external pure override returns (string memory) { - return "DestinationFeeManager 1.0.0"; - } - - /// @inheritdoc IERC165 - function supportsInterface(bytes4 interfaceId) external pure override returns (bool) { - //for each function in IDestinationFeeManager we need to check if it matches the selector - return - interfaceId == this.getFeeAndReward.selector || - interfaceId == this.setNativeSurcharge.selector || - interfaceId == this.updateSubscriberDiscount.selector || - interfaceId == this.withdraw.selector || - interfaceId == this.linkAvailableForPayment.selector || - interfaceId == this.payLinkDeficit.selector || - interfaceId == this.addVerifier.selector || - interfaceId == this.removeVerifier.selector || - interfaceId == this.processFee.selector || - interfaceId == this.processFeeBulk.selector || - interfaceId == this.setFeeRecipients.selector; - } - - function processFee( - bytes32 recipient, - bytes calldata payload, - bytes calldata parameterPayload, - address subscriber - ) external payable override onlyVerifier { - (Common.Asset memory fee, Common.Asset memory reward, uint256 appliedDiscount) = _calculateFee( - payload, - parameterPayload, - subscriber - ); - - if (fee.amount == 0) { - _tryReturnChange(subscriber, msg.value); - return; - } - - IDestinationFeeManager.FeeAndReward[] memory feeAndReward = new IDestinationFeeManager.FeeAndReward[](1); - feeAndReward[0] = IDestinationFeeManager.FeeAndReward(recipient, fee, reward, appliedDiscount); - - if (fee.assetAddress == i_linkAddress) { - _handleFeesAndRewards(subscriber, feeAndReward, 1, 0); - } else { - _handleFeesAndRewards(subscriber, feeAndReward, 0, 1); - } - } - - /// @inheritdoc IDestinationFeeManager - function processFeeBulk( - bytes32[] memory poolIds, - bytes[] calldata payloads, - bytes calldata parameterPayload, - address subscriber - ) external payable override onlyVerifier { - //poolIDs are mapped to payloads, so they should be the same length - if (poolIds.length != payloads.length) revert PoolIdMismatch(); - - IDestinationFeeManager.FeeAndReward[] memory feesAndRewards = new IDestinationFeeManager.FeeAndReward[]( - payloads.length - ); - - //keep track of the number of fees to prevent over initialising the FeePayment array within _convertToLinkAndNativeFees - uint256 numberOfLinkFees; - uint256 numberOfNativeFees; - - uint256 feesAndRewardsIndex; - for (uint256 i; i < payloads.length; ++i) { - if (poolIds[i] == bytes32(0)) revert InvalidAddress(); - - (Common.Asset memory fee, Common.Asset memory reward, uint256 appliedDiscount) = _calculateFee( - payloads[i], - parameterPayload, - subscriber - ); - - if (fee.amount != 0) { - feesAndRewards[feesAndRewardsIndex++] = IDestinationFeeManager.FeeAndReward( - poolIds[i], - fee, - reward, - appliedDiscount - ); - - unchecked { - //keep track of some tallys to make downstream calculations more efficient - if (fee.assetAddress == i_linkAddress) { - ++numberOfLinkFees; - } else { - ++numberOfNativeFees; - } - } - } - } - - if (numberOfLinkFees != 0 || numberOfNativeFees != 0) { - _handleFeesAndRewards(subscriber, feesAndRewards, numberOfLinkFees, numberOfNativeFees); - } else { - _tryReturnChange(subscriber, msg.value); - } - } - - /// @inheritdoc IDestinationFeeManager - function getFeeAndReward( - address subscriber, - bytes memory report, - address quoteAddress - ) public view returns (Common.Asset memory, Common.Asset memory, uint256) { - Common.Asset memory fee; - Common.Asset memory reward; - - //get the feedId from the report - bytes32 feedId = bytes32(report); - - //the report needs to be a support version - bytes32 reportVersion = _getReportVersion(feedId); - - //version 1 of the reports don't require quotes, so the fee will be 0 - if (reportVersion == REPORT_V1) { - fee.assetAddress = i_nativeAddress; - reward.assetAddress = i_linkAddress; - return (fee, reward, 0); - } - - //verify the quote payload is a supported token - if (quoteAddress != i_nativeAddress && quoteAddress != i_linkAddress) { - revert InvalidQuote(); - } - - //decode the report depending on the version - uint256 linkQuantity; - uint256 nativeQuantity; - uint256 expiresAt; - (, , , nativeQuantity, linkQuantity, expiresAt) = abi.decode( - report, - (bytes32, uint32, uint32, uint192, uint192, uint32) - ); - - //read the timestamp bytes from the report data and verify it has not expired - if (expiresAt < block.timestamp) { - revert ExpiredReport(); - } - - //get the discount being applied - uint256 discount = s_subscriberDiscounts[subscriber][feedId][quoteAddress]; - - //the reward is always set in LINK - reward.assetAddress = i_linkAddress; - reward.amount = Math.ceilDiv(linkQuantity * (PERCENTAGE_SCALAR - discount), PERCENTAGE_SCALAR); - - //calculate either the LINK fee or native fee if it's within the report - if (quoteAddress == i_linkAddress) { - fee.assetAddress = i_linkAddress; - fee.amount = reward.amount; - } else { - uint256 surchargedFee = Math.ceilDiv(nativeQuantity * (PERCENTAGE_SCALAR + s_nativeSurcharge), PERCENTAGE_SCALAR); - - fee.assetAddress = i_nativeAddress; - fee.amount = Math.ceilDiv(surchargedFee * (PERCENTAGE_SCALAR - discount), PERCENTAGE_SCALAR); - } - - //return the fee - return (fee, reward, discount); - } - - /// @inheritdoc IDestinationFeeManager - function setFeeRecipients( - bytes32 configDigest, - Common.AddressAndWeight[] calldata rewardRecipientAndWeights - ) external onlyOwnerOrVerifier { - i_rewardManager.setRewardRecipients(configDigest, rewardRecipientAndWeights); - } - - /// @inheritdoc IDestinationFeeManager - function setNativeSurcharge(uint64 surcharge) external onlyOwner { - if (surcharge > PERCENTAGE_SCALAR) revert InvalidSurcharge(); - - s_nativeSurcharge = surcharge; - - emit NativeSurchargeUpdated(surcharge); - } - - /// @inheritdoc IDestinationFeeManager - function updateSubscriberDiscount( - address subscriber, - bytes32 feedId, - address token, - uint64 discount - ) external onlyOwner { - //make sure the discount is not greater than the total discount that can be applied - if (discount > PERCENTAGE_SCALAR) revert InvalidDiscount(); - //make sure the token is either LINK or native - if (token != i_linkAddress && token != i_nativeAddress) revert InvalidAddress(); - - s_subscriberDiscounts[subscriber][feedId][token] = discount; - - emit SubscriberDiscountUpdated(subscriber, feedId, token, discount); - } - - /// @inheritdoc IDestinationFeeManager - function withdraw(address assetAddress, address recipient, uint192 quantity) external onlyOwner { - //address 0 is used to withdraw native in the context of withdrawing - if (assetAddress == address(0)) { - (bool success, ) = payable(recipient).call{value: quantity}(""); - - if (!success) revert InvalidReceivingAddress(); - return; - } - - //withdraw the requested asset - IERC20(assetAddress).safeTransfer(recipient, quantity); - - //emit event when funds are withdrawn - emit Withdraw(msg.sender, recipient, assetAddress, uint192(quantity)); - } - - /// @inheritdoc IDestinationFeeManager - function linkAvailableForPayment() external view returns (uint256) { - //return the amount of LINK this contact has available to pay rewards - return IERC20(i_linkAddress).balanceOf(address(this)); - } - - /** - * @notice Gets the current version of the report that is encoded as the last two bytes of the feed - * @param feedId feed id to get the report version for - */ - function _getReportVersion(bytes32 feedId) internal pure returns (bytes32) { - return REPORT_VERSION_MASK & feedId; - } - - function _calculateFee( - bytes calldata payload, - bytes calldata parameterPayload, - address subscriber - ) internal view returns (Common.Asset memory, Common.Asset memory, uint256) { - if (subscriber == address(this)) revert InvalidAddress(); - - //decode the report from the payload - (, bytes memory report) = abi.decode(payload, (bytes32[3], bytes)); - - //get the feedId from the report - bytes32 feedId = bytes32(report); - - //v1 doesn't need a quote payload, so skip the decoding - address quote; - if (_getReportVersion(feedId) != REPORT_V1) { - //decode the quote from the bytes - (quote) = abi.decode(parameterPayload, (address)); - } - - //decode the fee, it will always be native or LINK - return getFeeAndReward(subscriber, report, quote); - } - - function _handleFeesAndRewards( - address subscriber, - IDestinationFeeManager.FeeAndReward[] memory feesAndRewards, - uint256 numberOfLinkFees, - uint256 numberOfNativeFees - ) internal { - IDestinationRewardManager.FeePayment[] memory linkRewards = new IDestinationRewardManager.FeePayment[]( - numberOfLinkFees - ); - IDestinationRewardManager.FeePayment[] memory nativeFeeLinkRewards = new IDestinationRewardManager.FeePayment[]( - numberOfNativeFees - ); - - uint256 totalNativeFee; - uint256 totalNativeFeeLinkValue; - - uint256 linkRewardsIndex; - uint256 nativeFeeLinkRewardsIndex; - - uint256 totalNumberOfFees = numberOfLinkFees + numberOfNativeFees; - for (uint256 i; i < totalNumberOfFees; ++i) { - if (feesAndRewards[i].fee.assetAddress == i_linkAddress) { - linkRewards[linkRewardsIndex++] = IDestinationRewardManager.FeePayment( - feesAndRewards[i].configDigest, - uint192(feesAndRewards[i].reward.amount) - ); - } else { - nativeFeeLinkRewards[nativeFeeLinkRewardsIndex++] = IDestinationRewardManager.FeePayment( - feesAndRewards[i].configDigest, - uint192(feesAndRewards[i].reward.amount) - ); - totalNativeFee += feesAndRewards[i].fee.amount; - totalNativeFeeLinkValue += feesAndRewards[i].reward.amount; - } - - if (feesAndRewards[i].appliedDiscount != 0) { - emit DiscountApplied( - feesAndRewards[i].configDigest, - subscriber, - feesAndRewards[i].fee, - feesAndRewards[i].reward, - feesAndRewards[i].appliedDiscount - ); - } - } - - //keep track of change in case of any over payment - uint256 change; - - if (msg.value != 0) { - //there must be enough to cover the fee - if (totalNativeFee > msg.value) revert InvalidDeposit(); - - //wrap the amount required to pay the fee & approve as the subscriber paid in wrapped native - IWERC20(i_nativeAddress).deposit{value: totalNativeFee}(); - - unchecked { - //msg.value is always >= to fee.amount - change = msg.value - totalNativeFee; - } - } else { - if (totalNativeFee != 0) { - //subscriber has paid in wrapped native, so transfer the native to this contract - IERC20(i_nativeAddress).safeTransferFrom(subscriber, address(this), totalNativeFee); - } - } - - if (linkRewards.length != 0) { - i_rewardManager.onFeePaid(linkRewards, subscriber); - } - - if (nativeFeeLinkRewards.length != 0) { - //distribute subsidised fees paid in Native - if (totalNativeFeeLinkValue > IERC20(i_linkAddress).balanceOf(address(this))) { - // If not enough LINK on this contract to forward for rewards, tally the deficit to be paid by out-of-band LINK - for (uint256 i; i < nativeFeeLinkRewards.length; ++i) { - unchecked { - //we have previously tallied the fees, any overflows would have already reverted - s_linkDeficit[nativeFeeLinkRewards[i].poolId] += nativeFeeLinkRewards[i].amount; - } - } - - emit InsufficientLink(nativeFeeLinkRewards); - } else { - //distribute the fees - i_rewardManager.onFeePaid(nativeFeeLinkRewards, address(this)); - } - } - - // a refund may be needed if the payee has paid in excess of the fee - _tryReturnChange(subscriber, change); - } - - function _tryReturnChange(address subscriber, uint256 quantity) internal { - if (quantity != 0) { - payable(subscriber).transfer(quantity); - } - } - - /// @inheritdoc IDestinationFeeManager - function payLinkDeficit(bytes32 configDigest) external onlyOwner { - uint256 deficit = s_linkDeficit[configDigest]; - - if (deficit == 0) revert ZeroDeficit(); - - delete s_linkDeficit[configDigest]; - - IDestinationRewardManager.FeePayment[] memory deficitFeePayment = new IDestinationRewardManager.FeePayment[](1); - - deficitFeePayment[0] = IDestinationRewardManager.FeePayment(configDigest, uint192(deficit)); - - i_rewardManager.onFeePaid(deficitFeePayment, address(this)); - - emit LinkDeficitCleared(configDigest, deficit); - } - - /// @inheritdoc IDestinationFeeManager - function addVerifier(address verifierAddress) external onlyOwner { - if (verifierAddress == address(0)) revert InvalidAddress(); - //check doesn't already exist - if (s_verifierAddressList[verifierAddress] != address(0)) revert InvalidAddress(); - s_verifierAddressList[verifierAddress] = verifierAddress; - } - - /// @inheritdoc IDestinationFeeManager - function removeVerifier(address verifierAddress) external onlyOwner { - if (verifierAddress == address(0)) revert InvalidAddress(); - //check doesn't already exist - if (s_verifierAddressList[verifierAddress] == address(0)) revert InvalidAddress(); - delete s_verifierAddressList[verifierAddress]; - } - - /// @inheritdoc IDestinationFeeManager - function setRewardManager(address rewardManagerAddress) external onlyOwner { - if (rewardManagerAddress == address(0)) revert InvalidAddress(); - IERC20(i_linkAddress).approve(address(i_rewardManager), 0); - i_rewardManager = IDestinationRewardManager(rewardManagerAddress); - IERC20(i_linkAddress).approve(address(i_rewardManager), type(uint256).max); - } -} From 9028d69a106e3251336f78ef7d19df47696f6275 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 12:49:54 +0200 Subject: [PATCH 24/54] do not run Slither for test files --- .github/workflows/solidity-foundry.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index be3963259b3..28dca9333fe 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -46,6 +46,8 @@ jobs: non_src_changes: ${{ steps.changes.outputs.non_src }} sol_modified: ${{ steps.changes.outputs.sol }} sol_modified_files: ${{ steps.changes.outputs.sol_files }} + not_test_sol_modified: ${{ steps.changes.outputs.not_test_sol }} + not_test_sol_modified_files: ${{ steps.changes.outputs.not_test_sol_files }} all_changes: ${{ steps.changes.outputs.changes }} steps: - name: Checkout the repo @@ -61,6 +63,8 @@ jobs: - 'contracts/gas-snapshots/*.gas-snapshot' sol: - modified|added: 'contracts/src/v0.8/**/*.sol' + not_test_sol: + - modified|added: 'contracts/src/v0.8/**/!(*.t).sol' automation: - 'contracts/src/v0.8/automation/**/*.sol' ccip: @@ -87,7 +91,7 @@ jobs: - 'contracts/src/v0.8/vendor/**/*.sol' tests: - if: ${{ needs.changes.outputs.non_src_changes == 'true' }} + if: ${{ needs.changes.outputs.non_src_changes == 'true' || needs.changes.outputs.sol_modified == 'true' }} strategy: fail-fast: false matrix: @@ -205,7 +209,7 @@ jobs: analyze: needs: [ changes ] name: Run static analysis - if: needs.changes.outputs.sol_modified == 'true' + if: needs.changes.outputs.not_test_sol_modified == 'true' runs-on: ubuntu-22.04 steps: - name: Checkout the repo @@ -252,7 +256,7 @@ jobs: ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt - FILES="${{ needs.changes.outputs.sol_modified_files }}" + FILES="${{ needs.changes.outputs.not_test_sol_files }}" for FILE in $FILES; do PRODUCT=$(echo "$FILE" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) From f8aafb38b8d88a2658253d923003531fbb066e09 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 12:51:59 +0200 Subject: [PATCH 25/54] do not run fmt if test files were modified --- .github/workflows/solidity-foundry.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 28dca9333fe..f8c1000d6bd 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -256,7 +256,7 @@ jobs: ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt mv remappings_modified.txt remappings.txt - FILES="${{ needs.changes.outputs.not_test_sol_files }}" + FILES="${{ needs.changes.outputs.not_test_sol_modified_files }}" for FILE in $FILES; do PRODUCT=$(echo "$FILE" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) @@ -302,7 +302,7 @@ jobs: solidity-forge-fmt: name: Forge fmt ${{ matrix.product.name }} - if: ${{ needs.changes.outputs.non_src_changes == 'true' || needs.changes.outputs.sol_modified == 'true' }} + if: ${{ needs.changes.outputs.non_src_changes == 'true' || needs.changes.outputs.not_test_sol_modified == 'true' }} needs: [define-matrix, changes] strategy: fail-fast: false From 468a15724bb229be43d6bb0cb576761bd4007280 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 13:26:23 +0200 Subject: [PATCH 26/54] remove unused config file --- contracts/.slither.config-pr.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 contracts/.slither.config-pr.json diff --git a/contracts/.slither.config-pr.json b/contracts/.slither.config-pr.json deleted file mode 100644 index 1ef145a7954..00000000000 --- a/contracts/.slither.config-pr.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "filter_paths": "(openzeppelin|mocks/|test/|tests/|testhelpers)", - "detectors_to_exclude": "pragma" -} From 3cfe0818a4c50623712b07e52485226793747759 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 13:44:51 +0200 Subject: [PATCH 27/54] restore old Hardhat pipeline --- .../workflows/solidity-foundry-artifacts.yml | 486 ++++++++++++++++ .github/workflows/solidity-hardhat.yml | 518 ++---------------- contracts/.slither.config-artifacts.json | 0 .../slither/.slither.config-artifacts.json | 3 + 4 files changed, 544 insertions(+), 463 deletions(-) create mode 100644 .github/workflows/solidity-foundry-artifacts.yml delete mode 100644 contracts/.slither.config-artifacts.json create mode 100644 contracts/configs/slither/.slither.config-artifacts.json diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml new file mode 100644 index 00000000000..bb1166ad641 --- /dev/null +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -0,0 +1,486 @@ +name: Solidity Foundry Artifact Generation +on: + workflow_dispatch: + inputs: + product: + type: choice + description: 'product for which to generate artifacts; should be the same as Foundry profile' + required: true + options: + - "automation" + - "ccip" + - "functions" + - "keystone" + - "l2ep" + - "liquiditymanager" + - "llo-feeds" + - "operatorforwarder" + - "transmission" + - "vrf" + base_ref: + description: 'commit or tag to be used as base reference, when looking for modified Solidity files' + required: true + +env: + FOUNDRY_PROFILE: ci + # Has to match the `make foundry` version. + FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 + +jobs: + changes: + name: Detect changes + runs-on: ubuntu-latest + outputs: + changes: ${{ steps.changes.outputs.sol }} + product_changes: ${{ steps.changes.outputs.product }} + shared_changes: ${{ steps.changes.outputs.shared }} + files: ${{ steps.changes.outputs.included_files }} + changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} + changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Find modified contracts + uses: AurorNZ/paths-filter@3b1f3abc3371cca888d8eb03dfa70bc8a9867629 # v4.0.0 + id: changes + with: + list-files: 'csv' + base: ${{ inputs.base_ref }} + filters: | + ignored: &ignored + - '!contracts/src/v0.8/**/test/**' + - '!contracts/src/v0.8/**/tests/**' + - '!contracts/src/v0.8/**/*.t.sol' + - '!contracts/src/v0.8/*.t.sol' + - '!contracts/src/v0.8/**/testhelpers/**' + - '!contracts/src/v0.8/testhelpers/**' + - '!contracts/src/v0.8/vendor/**' + shared: &shared + - 'contracts/src/v0.8/shared/**/*.sol' + - 'contracts/src/v0.8/interfaces/**/*.sol' + - 'contracts/src/v0.8/*.sol' + - *ignored + sol: + - 'contracts/src/v0.8/**/*.sol' + - *ignored + product: &product + - 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' + - *ignored + included: + - *product + - *shared + - *ignored + + - name: Find modified changesets + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes-dorny + with: + list-files: 'shell' + base: ${{ inputs.base_ref }} + filters: | + changeset: + - modified|added: 'contracts/.changeset/!(README)*.md' + + - name: Check for changes outside of artifact scope + if: ${{ steps.changes.outputs.sol == 'true' }} + shell: bash + run: | + echo "::debug::All modified contracts:" + echo ${{ steps.changes.outputs.all_sol_files }} | tr ',' '\n' + echo "::debug::Filtered modified contracts:" + echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/shared/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + echo "::debug::Excluded paths: $excluded_paths_pattern" + unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") + set -e + set -o pipefail + if [[ -n "$unexpected_files" ]]; then + products=() + productsStr="" + IFS=$'\n' read -r -d '' -a files <<< "$unexpected_files" || true + echo "Files: ${files[@]}" + + for file in "${files[@]}"; do + product=$(echo "$file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) + if [[ ! " ${products[@]} " =~ " ${product} " ]]; then + products+=("$product") + productsStr+="$product, " + fi + done + productsStr=${productsStr%, } + + set +e + set +o pipefail + + echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}, shared" + echo "Files:" + echo "$unexpected_files" + echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" + + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}, shared" >> $GITHUB_STEP_SUMMARY + echo "### Files:" >> $GITHUB_STEP_SUMMARY + echo "$unexpected_files" >> $GITHUB_STEP_SUMMARY + echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY + else + echo "No unexpected files found." + fi + + - name: Check for changes only in shared + if: ${{ steps.changes.outputs.product == 'false' && steps.changes.outputs.shared == 'true' }} + shell: bash + run: | + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found No modified contracts for product ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + echo "## Action required: Please check if you chose correct product and base reference" >> $GITHUB_STEP_SUMMARY + echo "## Product: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + echo "## Base Ref: ${{ inputs.base_ref }}" >> $GITHUB_STEP_SUMMARY + + gather-basic-info: + if: ${{ needs.changes.outputs.changeset_changes == 'true' }} + name: Gather basic info + runs-on: ubuntu-22.04 + needs: [ changes ] + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + fetch-depth: 0 + + - name: Copy modified changesets + run: | + mkdir -p contracts/changesets/changesets + for changeset in "${{ needs.changes.outputs.changeset_files }}"; do + echo "::debug:: Copying $changeset" + cp $changeset contracts/changesets/changesets + done + + - name: Generate basic info + shell: bash + run: | + echo "Commit SHA used to generate artifacts: ${{ github.sha }}" > contracts/commit_sha_base_ref.txt + #TODO change to inputs.base_ref after testing + echo "Base reference SHA used to find modified contracts: ${{ inputs.base_ref }}" >> contracts/commit_sha_base_ref.txt + contract_list="${{ needs.changes.outputs.files }}" + new_line_list=$(echo "$contract_list" | tr ',' '\n' | sed 's/,$//') + printf "%s\n" "$new_line_list" > contracts/modified_contracts.txt + + - name: Generate modified contracts list + shell: bash + run: | + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" + echo "# Modified contracts:" > contracts/modified_contracts.md + for file in "${modified_files[@]}"; do + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping." + continue + fi + + echo " - [$file](${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/$file)" >> contracts/modified_contracts.md + done + + - name: Upload basic info + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 + continue-on-error: true + with: + name: tmp-basic-info + path: | + contracts/modified_contracts.md + contracts/modified_contracts.txt + contracts/commit_sha_base_ref.txt + contracts/changesets + retention-days: 7 + + # some of the artifacts can only be generated on product level, and we cannot scope them to single contracts + # some product-level modifications might also require shared contracts changes, so if these happened we need to generate artifacts for shared contracts as well + coverage-and-book: + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + name: Generate Docs and Code Coverage reports + runs-on: ubuntu-22.04 + needs: [changes] + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + + - name: Create directories + shell: bash + run: | + mkdir -p contracts/code-coverage + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 + with: + version: ${{ env.FOUNDRY_VERSION }} + + # required for code coverage report generation + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 + + - name: Run Forge build for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + run: | + forge --version + forge build + working-directory: contracts + env: + FOUNDRY_PROFILE: ${{ inputs.product }} + + - name: Run Forge build for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + run: | + forge --version + forge build + working-directory: contracts + env: + FOUNDRY_PROFILE: "shared" + + - name: Run coverage for product contracts + if: ${{ !contains(fromJson('["vrf"]'), inputs.product) && !contains(fromJson('["automation"]'), inputs.product) && !contains(fromJson('["functions"]'), inputs.product) && needs.changes.outputs.product_changes == 'true' }} + working-directory: contracts + run: forge coverage --report lcov --report-file code-coverage/lcov-${{ inputs.product }}.info + env: + FOUNDRY_PROFILE: ${{ inputs.product }} + + - name: Run coverage for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + working-directory: contracts + run: forge coverage --report lcov --report-file code-coverage/lcov-shared.info + env: + FOUNDRY_PROFILE: "shared" + + - name: Generate Code Coverage HTML report for product contracts + if: ${{ !contains(fromJson('["vrf"]'), inputs.product) && !contains(fromJson('["automation"]'), inputs.product) && !contains(fromJson('["functions"]'), inputs.product) && needs.changes.outputs.product_changes == 'true' }} + shell: bash + run: cd contracts && genhtml code-coverage/lcov-${{ inputs.product }}.info --branch-coverage --output-directory code-coverage/${{ inputs.product }} + + - name: Generate Code Coverage HTML report for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + shell: bash + run: cd contracts && genhtml code-coverage/lcov-shared.info --branch-coverage --output-directory code-coverage/shared + + - name: Run Forge doc for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + run: forge doc --build -o docs/${{ inputs.product }} + working-directory: contracts + env: + FOUNDRY_PROFILE: ${{ inputs.product }} + + - name: Run Forge doc for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + run: forge doc --build -o docs/shared + working-directory: contracts + env: + FOUNDRY_PROFILE: "shared" + + - name: Upload Artifacts for product contracts + if: ${{ needs.changes.outputs.product_changes == 'true' }} + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 + continue-on-error: true + with: + name: tmp-${{ inputs.product }}-artifacts + path: | + contracts/docs/${{ inputs.product }} + contracts/code-coverage/lcov-${{ inputs.product }}.info + contracts/code-coverage/${{ inputs.product }} + retention-days: 7 + + - name: Upload Artifacts for shared contracts + if: ${{ needs.changes.outputs.shared_changes == 'true' }} + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 2 + continue-on-error: true + with: + name: tmp-shared-artifacts + path: | + contracts/docs/shared + contracts/code-coverage/lcov-shared.info + contracts/code-coverage/shared + retention-days: 7 + + # Generates UML diagrams and Slither reports for modified contracts + uml-static-analysis: + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + name: Generate UML and Slither reports for modified contracts + runs-on: ubuntu-22.04 + needs: [changes] + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + submodules: recursive + fetch-depth: 0 + + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 + with: + version: ${{ env.FOUNDRY_VERSION }} + + - name: Install Sol2uml + run: | + npm link sol2uml --only=production + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Install solc-select and solc + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install solc-select + sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select + solc-select install 0.8.19 + solc-select use 0.8.19 + + - name: Install Slither + run: | + python -m pip install --upgrade pip + pip install slither-analyzer + + - name: Generate UML + shell: bash + run: | + contract_list="${{ needs.changes.outputs.files }}" + + # modify remappings so that solc can find dependencies + ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt + mv remappings_modified.txt remappings.txt + + ./contracts/scripts/ci/generate_uml.sh "./" "contracts/uml-diagrams" "$contract_list" + + - name: Generate Slither Markdown reports + run: | + contract_list="${{ needs.changes.outputs.files }}" + + echo "::debug::Processing contracts: $contract_list" + ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/configs/slither/.slither.config-artifacts.json "." "$contract_list" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" + + - name: Upload UMLs and Slither reports + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + timeout-minutes: 10 + continue-on-error: true + with: + name: tmp-contracts-artifacts + path: | + contracts/uml-diagrams + contracts/slither-reports + retention-days: 7 + + - name: Validate UMLs and Slither reports + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + run: | + echo "Validating UMLs and Slither reports" + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" + missing_svgs=() + missing_reports=() + for file in "${modified_files[@]}"; do + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + + svg_file="$(basename "${file%.sol}").svg" + if [ ! -f "contracts/uml-diagrams/$svg_file" ]; then + echo "Error: UML diagram for $file not found" + missing_svgs+=("$file") + fi + + report_file="$(basename "${file%.sol}")-slither-report.md" + if [ ! -f "contracts/slither-reports/$report_file" ]; then + echo "Error: Slither report for $file not found" + missing_reports+=("$file") + fi + done + + if [ ${#missing_svgs[@]} -gt 0 ]; then + echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_svgs[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY + else + echo "All UML diagrams generated successfully" + fi + + if [ ${#missing_reports[@]} -gt 0 ]; then + echo "Error: Missing Slither reports for files: ${missing_reports[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_reports[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY + else + echo "All Slither reports generated successfully" + fi + + gather-all-artifacts: + name: Gather all artifacts + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + runs-on: ubuntu-latest + needs: [coverage-and-book, uml-static-analysis, gather-basic-info, changes] + steps: + - name: Download all artifacts + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + path: review_artifacts + merge-multiple: true + + - name: Upload all artifacts as single package + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + with: + name: review-artifacts-${{ inputs.product }}-${{ github.sha }} + path: review_artifacts + + - name: Remove temporary artifacts + uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1 # v5.0 + with: + name: tmp-* + + - name: Print Artifact URL in job summary + env: + GH_TOKEN: ${{ github.token }} + run: | + ARTIFACTS=$(gh api -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts) + ARTIFACT_ID=$(echo "$ARTIFACTS" | jq '.artifacts[] | select(.name=="review-artifacts-${{ inputs.product }}-${{ github.sha }}") | .id') + echo "Artifact ID: $ARTIFACT_ID" + + echo "# Solidity Review Artifact Generated" >> $GITHUB_STEP_SUMMARY + echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY + echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY + echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY + + notify-no-changes: + if: ${{ needs.changes.outputs.product_changes == 'false' && needs.changes.outputs.shared_changes == 'false' }} + needs: [changes] + runs-on: ubuntu-latest + steps: + - name: Print warning in job summary + shell: bash + run: | + echo "# Solidity Review Artifact NOT Generated" >> $GITHUB_STEP_SUMMARY + echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY + echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY + echo "## Reason: No modified Solidity files found for ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + echo "No modified Solidity files found between ${{ inputs.base_ref }} and ${{ github.sha }} commits or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY + exit 1 diff --git a/.github/workflows/solidity-hardhat.yml b/.github/workflows/solidity-hardhat.yml index b525b5d9b52..705fad3be60 100644 --- a/.github/workflows/solidity-hardhat.yml +++ b/.github/workflows/solidity-hardhat.yml @@ -1,495 +1,87 @@ name: Solidity-Hardhat + on: + merge_group: push: - workflow_dispatch: - inputs: - product: - type: choice - description: 'product for which to generate artifacts; should be the same as Foundry profile' - required: true - options: - - "automation" - - "ccip" - - "functions" - - "keystone" - - "l2ep" - - "liquiditymanager" - - "llo-feeds" - - "operatorforwarder" - - "transmission" - - "vrf" - base_ref: - description: 'commit or tag to be used as base reference, when looking for modified Solidity files' - required: true env: - FOUNDRY_PROFILE: ci - # Has to match the `make foundry` version. - FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 - PRODUCT: "functions" - BASE_REF: "v2.13.0" + NODE_OPTIONS: --max_old_space_size=8192 + +defaults: + run: + shell: bash jobs: changes: name: Detect changes runs-on: ubuntu-latest outputs: - changes: ${{ steps.changes.outputs.sol }} - product_changes: ${{ steps.changes.outputs.product }} - shared_changes: ${{ steps.changes.outputs.shared }} - files: ${{ steps.changes.outputs.included_files }} - changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} - changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} + changes: ${{ steps.changes.outputs.src }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Find modified contracts - uses: AurorNZ/paths-filter@3b1f3abc3371cca888d8eb03dfa70bc8a9867629 # v4.0.0 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes with: - list-files: 'csv' - base: ${{ env.BASE_REF }} #TODO change to inputs.base_ref after testing - filters: | - ignored: &ignored - - '!contracts/src/v0.8/**/test/**' - - '!contracts/src/v0.8/**/tests/**' - - '!contracts/src/v0.8/**/*.t.sol' - - '!contracts/src/v0.8/*.t.sol' - - '!contracts/src/v0.8/**/testhelpers/**' - - '!contracts/src/v0.8/testhelpers/**' - - '!contracts/src/v0.8/vendor/**' - shared: &shared - - 'contracts/src/v0.8/shared/**/*.sol' - - 'contracts/src/v0.8/interfaces/**/*.sol' - - 'contracts/src/v0.8/*.sol' - - *ignored - sol: - - 'contracts/src/v0.8/**/*.sol' - - *ignored - product: &product - - 'contracts/src/v0.8/${{ env.PRODUCT }}/**/*.sol' - - *ignored - included: - - *product - - *shared - - *ignored - - - name: Find modified changesets - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: changes-dorny - with: - list-files: 'shell' - base: ${{ env.BASE_REF }} #TODO change to inputs.base_ref after testing filters: | - changeset: - - modified|added: 'contracts/.changeset/!(README)*.md' - - - name: Check for changes outside of artifact scope - if: ${{ steps.changes.outputs.sol == 'true' }} - shell: bash - run: | - echo "::debug::All modified contracts:" - echo ${{ steps.changes.outputs.all_sol_files }} | tr ',' '\n' - echo "::debug::Filtered modified contracts:" - echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' - excluded_paths_pattern="!/^contracts\/src\/v0\.8\/shared/ && !/^contracts\/src\/v0\.8\/${{ env.PRODUCT}}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" - echo "::debug::Excluded paths: $excluded_paths_pattern" - unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") - set -e - set -o pipefail - if [[ -n "$unexpected_files" ]]; then - products=() - productsStr="" - IFS=$'\n' read -r -d '' -a files <<< "$unexpected_files" || true - echo "Files: ${files[@]}" - - for file in "${files[@]}"; do - product=$(echo "$file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) - if [[ ! " ${products[@]} " =~ " ${product} " ]]; then - products+=("$product") - productsStr+="$product, " - fi - done - productsStr=${productsStr%, } - - set +e - set +o pipefail - - echo "Error: Found modified contracts outside of the expected scope: ${{ env.PRODUCT }}, shared" - echo "Files:" - echo "$unexpected_files" - echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" - - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found modified contracts outside of the expected scope: ${{ env.PRODUCT }}, shared" >> $GITHUB_STEP_SUMMARY - echo "### Files:" >> $GITHUB_STEP_SUMMARY - echo "$unexpected_files" >> $GITHUB_STEP_SUMMARY - echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY - else - echo "No unexpected files found." - fi - - - name: Check for changes only in shared - if: ${{ steps.changes.outputs.product == 'false' && steps.changes.outputs.shared == 'true' }} - shell: bash - run: | - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found No modified contracts for product ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY - echo "## Action required: Please check if you chose correct product and base reference" >> $GITHUB_STEP_SUMMARY - echo "## Product: ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY - #TODO change to inputs.base_ref after testing - echo "## Base Ref: ${{ env.BASE_REF }}" >> $GITHUB_STEP_SUMMARY - - gather-basic-info: - if: ${{ needs.changes.outputs.changeset_changes == 'true' }} - name: Gather basic info - runs-on: ubuntu-22.04 - needs: [ changes ] - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - submodules: recursive - fetch-depth: 0 - - - name: Copy modified changesets - run: | - mkdir -p contracts/changesets/changesets - for changeset in "${{ needs.changes.outputs.changeset_files }}"; do - echo "::debug:: Copying $changeset" - cp $changeset contracts/changesets/changesets - done - - - name: Generate basic info - shell: bash - run: | - echo "Commit SHA used to generate artifacts: ${{ github.sha }}" > contracts/commit_sha_base_ref.txt - #TODO change to inputs.base_ref after testing - echo "Base reference SHA used to find modified contracts: ${{ env.BASE_REF }}" >> contracts/commit_sha_base_ref.txt - contract_list="${{ needs.changes.outputs.files }}" - new_line_list=$(echo "$contract_list" | tr ',' '\n' | sed 's/,$//') - printf "%s\n" "$new_line_list" > contracts/modified_contracts.txt - - cat contracts/modified_contracts.txt - - - name: Generate modified contracts list - shell: bash - run: | - IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" - echo "# Modified contracts:" > contracts/modified_contracts.md - for file in "${modified_files[@]}"; do - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ env.BASE_REF }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping." - continue - fi - - echo " - [$file](${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/$file)" >> contracts/modified_contracts.md - done - - - name: Upload basic info - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - timeout-minutes: 2 - continue-on-error: true - with: - name: tmp-basic-info - path: | - contracts/modified_contracts.md - contracts/modified_contracts.txt - contracts/commit_sha_base_ref.txt - contracts/changesets - retention-days: 7 - - # some of the artifacts can only be generated on product level, and we cannot scope them to single contracts - # some product-level modifications might also require shared contracts changes, so if these happened we need to generate artifacts for shared contracts as well - coverage-and-book: - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} - name: Generate Docs and Code Coverage reports - runs-on: ubuntu-22.04 + src: + - 'contracts/src/!(v0.8/(ccip|functions|keystone|l2ep|liquiditymanager|llo-feeds|transmission|vrf)/**)/**/*' + - 'contracts/test/**/*' + - 'contracts/package.json' + - 'contracts/pnpm-lock.yaml' + - 'contracts/hardhat.config.ts' + - '.github/workflows/solidity-hardhat.yml' + + hardhat-test: needs: [changes] + if: needs.changes.outputs.changes == 'true' + name: Solidity ${{ fromJSON('["(skipped)", ""]')[needs.changes.outputs.changes == 'true'] }} + runs-on: ubuntu-latest steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - submodules: recursive - - name: Setup NodeJS uses: ./.github/actions/setup-nodejs - - - name: Create directories - shell: bash - run: | - mkdir -p contracts/code-coverage - - - name: Install Foundry - uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 + - name: Setup Hardhat + uses: ./.github/actions/setup-hardhat with: - version: ${{ env.FOUNDRY_VERSION }} - - # required for code coverage report generation - - name: Setup LCOV - uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - - - name: Run Forge build for product contracts - if: ${{ needs.changes.outputs.product_changes == 'true' }} - run: | - forge --version - forge build - working-directory: contracts - env: - FOUNDRY_PROFILE: ${{ env.PRODUCT }} - - - name: Run Forge build for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - run: | - forge --version - forge build - working-directory: contracts - env: - FOUNDRY_PROFILE: "shared" - - - name: Run coverage for product contracts - if: ${{ !contains(fromJson('["vrf"]'), env.PRODUCT) && !contains(fromJson('["automation"]'), env.PRODUCT) && !contains(fromJson('["functions"]'), env.PRODUCT) && needs.changes.outputs.product_changes == 'true' }} - working-directory: contracts - run: forge coverage --report lcov --report-file code-coverage/lcov-${{ env.PRODUCT }}.info - env: - FOUNDRY_PROFILE: ${{ env.PRODUCT }} - - - name: Run coverage for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - working-directory: contracts - run: forge coverage --report lcov --report-file code-coverage/lcov-shared.info - env: - FOUNDRY_PROFILE: "shared" - - - name: Generate Code Coverage HTML report for product contracts - if: ${{ !contains(fromJson('["vrf"]'), env.PRODUCT) && !contains(fromJson('["automation"]'), env.PRODUCT) && !contains(fromJson('["functions"]'), env.PRODUCT) && needs.changes.outputs.product_changes == 'true' }} - shell: bash - run: cd contracts && genhtml code-coverage/lcov-${{ env.PRODUCT }}.info --branch-coverage --output-directory code-coverage/${{ env.PRODUCT }} - - - name: Generate Code Coverage HTML report for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - shell: bash - run: cd contracts && genhtml code-coverage/lcov-shared.info --branch-coverage --output-directory code-coverage/shared - - - name: Run Forge doc for product contracts - if: ${{ needs.changes.outputs.product_changes == 'true' }} - run: forge doc --build -o docs/${{ env.PRODUCT }} - working-directory: contracts - env: - FOUNDRY_PROFILE: ${{ env.PRODUCT }} - - - name: Run Forge doc for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - run: forge doc --build -o docs/shared + namespace: coverage + - name: Run tests working-directory: contracts - env: - FOUNDRY_PROFILE: "shared" - - - name: Upload Artifacts for product contracts - if: ${{ needs.changes.outputs.product_changes == 'true' }} - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - timeout-minutes: 2 - continue-on-error: true + run: pnpm test + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: - name: tmp-${{ env.PRODUCT }}-artifacts - path: | - contracts/docs/${{ env.PRODUCT }} - contracts/code-coverage/lcov-${{ env.PRODUCT }}.info - contracts/code-coverage/${{ env.PRODUCT }} - retention-days: 7 - - - name: Upload Artifacts for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - timeout-minutes: 2 + id: hardhat-test + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} continue-on-error: true - with: - name: tmp-shared-artifacts - path: | - contracts/docs/shared - contracts/code-coverage/lcov-shared.info - contracts/code-coverage/shared - retention-days: 7 - # Generates UML diagrams and Slither reports for modified contracts - uml-static-analysis: - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} - name: Generate UML and Slither reports for modified contracts - runs-on: ubuntu-22.04 - needs: [changes] - env: - GH_TOKEN: ${{ github.token }} + solidity: + needs: [changes, hardhat-test] + name: Solidity + runs-on: ubuntu-latest + if: always() steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - submodules: recursive - fetch-depth: 0 - - - name: Setup NodeJS - uses: ./.github/actions/setup-nodejs - - - name: Install Foundry - uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 - with: - version: ${{ env.FOUNDRY_VERSION }} - - - name: Install Sol2uml - run: | - npm link sol2uml --only=production - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - - name: Install solc-select and solc - run: | - sudo apt-get update - sudo apt-get install -y python3-pip - pip3 install solc-select - sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select - solc-select install 0.8.19 - solc-select use 0.8.19 - - - name: Install Slither + - run: echo 'Solidity tests finished!' + - name: Check test results run: | - python -m pip install --upgrade pip - pip install slither-analyzer - - - name: Generate UML - shell: bash - run: | - contract_list="${{ needs.changes.outputs.files }}" - - # modify remappings so that solc can find dependencies - ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt - mv remappings_modified.txt remappings.txt - - ./contracts/scripts/ci/generate_uml.sh "./" "contracts/uml-diagrams" "$contract_list" - - - name: Generate Slither Markdown reports - run: | - contract_list="${{ needs.changes.outputs.files }}" - - echo "::debug::Processing contracts: $contract_list" - ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/.slither.config-artifacts.json "." "$contract_list" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" - - - name: Upload UMLs and Slither reports - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - timeout-minutes: 10 - continue-on-error: true - with: - name: tmp-contracts-artifacts - path: | - contracts/uml-diagrams - contracts/slither-reports - retention-days: 7 - - - name: Validate UMLs and Slither reports - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} - run: | - echo "Validating UMLs and Slither reports" - IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" - missing_svgs=() - missing_reports=() - for file in "${modified_files[@]}"; do - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ env.BASE_REF }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue - fi - - svg_file="$(basename "${file%.sol}").svg" - if [ ! -f "contracts/uml-diagrams/$svg_file" ]; then - echo "Error: UML diagram for $file not found" - missing_svgs+=("$file") - fi - - report_file="$(basename "${file%.sol}")-slither-report.md" - if [ ! -f "contracts/slither-reports/$report_file" ]; then - echo "Error: Slither report for $file not found" - missing_reports+=("$file") - fi - done - - if [ ${#missing_svgs[@]} -gt 0 ]; then - echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_svgs[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY - else - echo "All UML diagrams generated successfully" - fi - - if [ ${#missing_reports[@]} -gt 0 ]; then - echo "Error: Missing Slither reports for files: ${missing_reports[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_reports[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY - else - echo "All Slither reports generated successfully" + if [[ "${{ needs.changes.result }}" = "failure" || "${{ needs.solidity-splits.result }}" = "failure" ]]; then + echo "One or more changes / solidity-splits jobs failed" + exit 1 + else + echo "All test jobs passed successfully" fi - - gather-all-artifacts: - name: Gather all artifacts - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} - runs-on: ubuntu-latest - needs: [coverage-and-book, uml-static-analysis, gather-basic-info, changes] - steps: - - name: Download all artifacts - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: - path: review_artifacts - merge-multiple: true - - - name: Upload all artifacts as single package - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - with: - name: review-artifacts-${{ env.PRODUCT }}-${{ github.sha }} - path: review_artifacts - - - name: Remove temporary artifacts - uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1 # v5.0 - with: - name: tmp-* - - - name: Print Artifact URL in job summary - env: - GH_TOKEN: ${{ github.token }} - run: | - ARTIFACTS=$(gh api -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts) - ARTIFACT_ID=$(echo "$ARTIFACTS" | jq '.artifacts[] | select(.name=="review-artifacts-${{ env.PRODUCT }}-${{ github.sha }}") | .id') - echo "Artifact ID: $ARTIFACT_ID" - - echo "# Solidity Review Artifact Generated" >> $GITHUB_STEP_SUMMARY - # TODO change to inputs.base_ref after testing - echo "Base Ref used: **${{ env.BASE_REF }}**" >> $GITHUB_STEP_SUMMARY - echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY - echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY - - notify-no-changes: - if: ${{ needs.changes.outputs.product_changes == 'false' && needs.changes.outputs.shared_changes == 'false' }} - needs: [changes] - runs-on: ubuntu-latest - steps: - - name: Print warning in job summary - shell: bash - run: | - echo "# Solidity Review Artifact NOT Generated" >> $GITHUB_STEP_SUMMARY - #TODO change to inputs.base_ref after testing - echo "Base Ref used: **${{ env.BASE_REF }}**" >> $GITHUB_STEP_SUMMARY - echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY - echo "## Reason: No modified Solidity files found for ${{ env.PRODUCT }}" >> $GITHUB_STEP_SUMMARY - #TODO change to inputs.base_ref after testing - echo "No modified Solidity files found between ${{ env.BASE_REF }} and ${{ github.sha }} commits or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY - exit 1 + id: solidity-tests + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Solidity + continue-on-error: true diff --git a/contracts/.slither.config-artifacts.json b/contracts/.slither.config-artifacts.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/contracts/configs/slither/.slither.config-artifacts.json b/contracts/configs/slither/.slither.config-artifacts.json new file mode 100644 index 00000000000..75071341f51 --- /dev/null +++ b/contracts/configs/slither/.slither.config-artifacts.json @@ -0,0 +1,3 @@ +{ + "filter_paths": "(openzeppelin|mocks/|test/|tests/|testhelpers)" +} From 0a49359a8e3ccc4c234e052939ea1561cba0cff1 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 5 Aug 2024 13:54:49 +0200 Subject: [PATCH 28/54] add missing transmission setup --- .github/workflows/solidity-foundry.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index f8c1000d6bd..266673cc69e 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -32,6 +32,7 @@ jobs: { "name": "llo-feeds", "setup": { "run-coverage": true, "min-coverage": 49.3, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "operatorforwarder", "setup": { "run-coverage": true, "min-coverage": 55.7, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "shared", "setup": { "run-coverage": true, "min-coverage": 32.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, + { "name": "transmission", "setup": { "run-coverage": true, "min-coverage": 65.6, "run-gas-snapshot": true, "run-forge-fmt": false }}, { "name": "vrf", "setup": { "run-coverage": false, "min-coverage": 98.5, "run-gas-snapshot": false, "run-forge-fmt": false }} ] EOF @@ -88,7 +89,9 @@ jobs: - 'contracts/src/v0.8/*.sol' - 'contracts/src/v0.8/mocks/**/*.sol' - 'contracts/src/v0.8/tests/**/*.sol' - - 'contracts/src/v0.8/vendor/**/*.sol' + - 'contracts/src/v0.8/vendor/**/*.sol' + transmission: + - 'contracts/src/v0.8/transmission/**/*.sol' tests: if: ${{ needs.changes.outputs.non_src_changes == 'true' || needs.changes.outputs.sol_modified == 'true' }} From f2b7bd7ebd071ca64c84fffbfc9fba1c64a8ec6f Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 11:41:08 +0200 Subject: [PATCH 29/54] fix basic info condition, join 2 steps into 1, define higher-level coverage exclusions --- .../workflows/solidity-foundry-artifacts.yml | 32 +++++++++++-------- contracts/scripts/ci/generate_uml.sh | 9 +++--- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index fed8c163bd4..bf5ce988520 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -137,8 +137,8 @@ jobs: echo "## Base Ref: ${{ inputs.base_ref }}" >> $GITHUB_STEP_SUMMARY gather-basic-info: - if: ${{ needs.changes.outputs.changeset_changes == 'true' }} name: Gather basic info + if: ${{ steps.changes.outputs.product == 'true' || steps.changes.outputs.shared == 'true' }} runs-on: ubuntu-22.04 needs: [ changes ] steps: @@ -149,6 +149,7 @@ jobs: fetch-depth: 0 - name: Copy modified changesets + if: ${{ needs.changes.outputs.changeset_changes == 'true' }} run: | mkdir -p contracts/changesets/changesets for changeset in "${{ needs.changes.outputs.changeset_files }}"; do @@ -156,19 +157,12 @@ jobs: cp $changeset contracts/changesets/changesets done - - name: Generate basic info + - name: Generate basic info and modified contracts list shell: bash run: | echo "Commit SHA used to generate artifacts: ${{ github.sha }}" > contracts/commit_sha_base_ref.txt - #TODO change to inputs.base_ref after testing - echo "Base reference SHA used to find modified contracts: ${{ inputs.base_ref }}" >> contracts/commit_sha_base_ref.txt - contract_list="${{ needs.changes.outputs.files }}" - new_line_list=$(echo "$contract_list" | tr ',' '\n' | sed 's/,$//') - printf "%s\n" "$new_line_list" > contracts/modified_contracts.txt - - - name: Generate modified contracts list - shell: bash - run: | + echo "Base reference SHA used to find modified contracts: ${{ inputs.base_ref }}" >> contracts/commit_sha_base_ref.txt + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" echo "# Modified contracts:" > contracts/modified_contracts.md for file in "${modified_files[@]}"; do @@ -180,9 +174,10 @@ jobs: fi echo " - [$file](${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/$file)" >> contracts/modified_contracts.md + echo "$file" >> contracts/modified_contracts.txt done - - name: Upload basic info + - name: Upload basic info and modified contracts list uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 timeout-minutes: 2 continue-on-error: true @@ -203,6 +198,15 @@ jobs: runs-on: ubuntu-22.04 needs: [changes] steps: + - name: Prepare exclusion list + id: prepare-exclusion-list + run: | + cat < coverage_exclusions.json + ["automation", "functions", "vrf"] + EOF + coverage_exclusions=$(cat coverage_exclusions.json | jq -c .) + echo "coverage_exclusions=$coverage_exclusions" >> $GITHUB_OUTPUT + - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: @@ -244,7 +248,7 @@ jobs: FOUNDRY_PROFILE: "shared" - name: Run coverage for product contracts - if: ${{ !contains(fromJson('["vrf"]'), inputs.product) && !contains(fromJson('["automation"]'), inputs.product) && !contains(fromJson('["functions"]'), inputs.product) && needs.changes.outputs.product_changes == 'true' }} + if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} working-directory: contracts run: forge coverage --report lcov --report-file code-coverage/lcov-${{ inputs.product }}.info env: @@ -258,7 +262,7 @@ jobs: FOUNDRY_PROFILE: "shared" - name: Generate Code Coverage HTML report for product contracts - if: ${{ !contains(fromJson('["vrf"]'), inputs.product) && !contains(fromJson('["automation"]'), inputs.product) && !contains(fromJson('["functions"]'), inputs.product) && needs.changes.outputs.product_changes == 'true' }} + if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} shell: bash run: cd contracts && genhtml code-coverage/lcov-${{ inputs.product }}.info --branch-coverage --output-directory code-coverage/${{ inputs.product }} diff --git a/contracts/scripts/ci/generate_uml.sh b/contracts/scripts/ci/generate_uml.sh index b9ce3d00ca1..fbab2724205 100755 --- a/contracts/scripts/ci/generate_uml.sh +++ b/contracts/scripts/ci/generate_uml.sh @@ -61,7 +61,7 @@ flatten_and_generate_uml() { rm "$FLATTENED_FILE" } -flatten_contracts_in_directory() { +process_all_files_in_directory() { local SOURCE_DIR=$1 local TARGET_DIR=$2 @@ -72,7 +72,7 @@ flatten_contracts_in_directory() { done } -process_files() { +process_selected_files() { local SOURCE_DIR=$1 local TARGET_DIR=$2 local FILES=(${3//,/ }) # Split the comma-separated list into an array @@ -101,10 +101,11 @@ process_files() { done } +# if FILES is empty, process all files in the directory, otherwise process only the selected files if [ -z "$FILES" ]; then - flatten_contracts_in_directory "$SOURCE_DIR" "$TARGET_DIR" + process_all_files_in_directory "$SOURCE_DIR" "$TARGET_DIR" else - process_files "$SOURCE_DIR" "$TARGET_DIR" "$FILES" + process_selected_files "$SOURCE_DIR" "$TARGET_DIR" "$FILES" fi if [ "${#FAILED_FILES[@]}" -gt 0 ]; then From d7d07adf1a4bdff160ca1298e8c0509bc582f6fd Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 12:01:32 +0200 Subject: [PATCH 30/54] define actions for installing Slither and solc-select --- .github/actions/setup-slither/action.yaml | 10 ++++++ .github/actions/setup-solc-select/action.yaml | 32 +++++++++++++++++++ .github/workflows/solidity-foundry.yml | 15 +++------ 3 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 .github/actions/setup-slither/action.yaml create mode 100644 .github/actions/setup-solc-select/action.yaml diff --git a/.github/actions/setup-slither/action.yaml b/.github/actions/setup-slither/action.yaml new file mode 100644 index 00000000000..d8e7bef9e5e --- /dev/null +++ b/.github/actions/setup-slither/action.yaml @@ -0,0 +1,10 @@ +name: Setup Slither +description: Installs Slither for contract analysis. Requires Python 3.6 or higher. +runs: + using: composite + steps: + - name: Install Slither + shell: bash + run: | + python -m pip install --upgrade pip + pip install slither-analyzer diff --git a/.github/actions/setup-solc-select/action.yaml b/.github/actions/setup-solc-select/action.yaml new file mode 100644 index 00000000000..7903b7acd3c --- /dev/null +++ b/.github/actions/setup-solc-select/action.yaml @@ -0,0 +1,32 @@ +name: Setup Solc Select +description: Installs Solc Select, required versions and selects the version to use. Requires Python 3.6 or higher. +inputs: + to_install: + description: Comma-separated list of solc versions to install + required: true + to_use: + description: Solc version to use + required: true + +runs: + using: composite + steps: + - name: Install solc-select and solc + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install solc-select + sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select + + IFS=',' read -ra versions <<< "${{ inputs.to_install }}" + for version in "${versions[@]}"; do + solc-select install $version + if [ $? -ne 0 ]; then + echo "Failed to install Solc $version" + exit 1 + fi + done + + solc-select install ${{ inputs.to_use }} + solc-select use ${{ inputs.to_use }} diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 266673cc69e..96ab4c46e28 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -239,18 +239,13 @@ jobs: python-version: '3.8' - name: Install solc-select and solc - run: | - sudo apt-get update - sudo apt-get install -y python3-pip - pip3 install solc-select - sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select - solc-select install 0.8.19 - solc-select use 0.8.19 + uses: ./.github/actions/setup-solc-select + with: + to_install: '0.8.19' + to_use: '0.8.19' - name: Install Slither - run: | - python -m pip install --upgrade pip - pip install slither-analyzer + uses: ./.github/actions/setup-slither - name: Run Slither shell: bash From 678f310bbeb5bdaffd5dd454c827781f1201b1ab Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 12:10:32 +0200 Subject: [PATCH 31/54] run all tests also if package.json changes; run them on all non_src changes --- .github/workflows/solidity-foundry.yml | 36 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 96ab4c46e28..f43263f4661 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -62,6 +62,7 @@ jobs: - '.github/workflows/solidity-foundry.yml' - 'contracts/foundry.toml' - 'contracts/gas-snapshots/*.gas-snapshot' + - 'contracts/package.json' sol: - modified|added: 'contracts/src/v0.8/**/*.sol' not_test_sol: @@ -108,7 +109,8 @@ jobs: steps: - name: Checkout the repo if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: submodules: recursive @@ -118,19 +120,22 @@ jobs: # stay in sync. - name: Setup NodeJS if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} uses: ./.github/actions/setup-nodejs - name: Install Foundry if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: version: ${{ env.FOUNDRY_VERSION }} - name: Run Forge build if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} run: | forge --version forge build @@ -141,7 +146,8 @@ jobs: - name: Run Forge tests if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared') }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} run: | forge test -vvv id: test @@ -151,8 +157,9 @@ jobs: - name: Run Forge snapshot if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) && - matrix.product.setup.run-gas-snapshot }} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true') + && matrix.product.setup.run-gas-snapshot }} run: | forge snapshot --nmt "test_?Fuzz_\w{1,}?" --check gas-snapshots/${{ matrix.product.name }}.gas-snapshot id: snapshot @@ -163,13 +170,15 @@ jobs: # required for code coverage report generation - name: Setup LCOV if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true') && matrix.product.setup.run-coverage }} uses: hrishikesh-kadam/setup-lcov@f5da1b26b0dcf5d893077a3c4f29cf78079c841d # v1.0.0 - name: Run coverage for ${{ matrix.product.name }} if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true') && matrix.product.setup.run-coverage }} working-directory: contracts run: forge coverage --report lcov @@ -178,7 +187,8 @@ jobs: - name: Prune lcov report if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true') && matrix.product.setup.run-coverage }} run: | sudo apt-get install lcov @@ -186,7 +196,8 @@ jobs: - name: Report code coverage for ${{ matrix.product.name }} if: ${{ (contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')) + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true') && matrix.product.setup.run-coverage }} uses: zgosalvez/github-actions-report-lcov@a546f89a65a0cdcd82a92ae8d65e74d450ff3fbc # v4.1.4 with: @@ -198,7 +209,8 @@ jobs: - name: Collect Metrics if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) - || contains(fromJson(needs.changes.outputs.all_changes), 'shared')}} + || contains(fromJson(needs.changes.outputs.all_changes), 'shared') + || needs.changes.outputs.non_src_changes == 'true' }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: From 7b6e1ac67f55b18118aad546f347eb9996439278 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 12:23:15 +0200 Subject: [PATCH 32/54] add action for validating whether all Slither reports and UML diagrams were generated --- .../validate-solidity-artifacts/action.yaml | 97 +++++++++++++++++++ .github/workflows/solidity-foundry.yml | 10 +- 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 .github/actions/validate-solidity-artifacts/action.yaml diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml new file mode 100644 index 00000000000..3a65eddb578 --- /dev/null +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -0,0 +1,97 @@ +name: Validate Solidity Artifacts +description: Checks whether Slither reports and UML diagrams were generated for all necessary files. If not, a warning is printed in job summary, but the job is not marked as failed. +inputs: + slither_reports_path: + description: Path to the Slither reports directory (without trailing slash) + required: true + uml_diagrams_path: + description: Path to the UML diagrams directory (without trailing slash) + required: true + validate_slither_reports: + description: Whether Slither reports should be validated + required: true + validate_uml_diagrams: + description: Whether UML diagrams should be validated + required: true + sol_files: + description: Comma-separated list of Solidity files to check + required: true + # remove this when AurorNZ/paths-filter is fixed + base_ref: + description: Base reference for comparison (skip if your list doesn't contain deleted files) + required: false + +runs: + using: composite + steps: + - name: Validate UML diagrams + if: ${{ inputs.validate_uml_diagrams == 'true' }} + shell: bash + run: | + echo "Validating UML diagrams" + IFS=',' read -r -a modified_files <<< "${{ inputs.sol_files }}" + missing_svgs=() + for file in "${modified_files[@]}"; do + if [ "${{ inputs.base_ref }}" != "" ]; then + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + fi + + svg_file="$(basename "${file%.sol}").svg" + if [ ! -f "${{ inputs.uml_diagrams_path }}/$svg_file" ]; then + echo "Error: UML diagram for $file not found" + missing_svgs+=("$file") + fi + done + + if [ ${#missing_svgs[@]} -gt 0 ]; then + echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_svgs[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY + else + echo "All UML diagrams generated successfully" + fi + + - name: Validate Slither reports + if: ${{ inputs.validate_slither_reports == 'true' }} + shell: bash + run: | + echo "Validating Slither reports" + IFS=',' read -r -a modified_files <<< "${{ inputs.sol_files }}" + missing_reports=() + for file in "${modified_files[@]}"; do + if [ "${{ inputs.base_ref }}" != "" ]; then + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + fi + + report_file="$(basename "${file%.sol}")-slither-report.md" + if [ ! -f "${{ inputs.slither_reports_path }}/$report_file" ]; then + echo "Error: Slither report for $file not found" + missing_reports+=("$file") + fi + done + + if [ ${#missing_reports[@]} -gt 0 ]; then + echo "Error: Missing Slither reports for files: ${missing_reports[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_reports[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY + else + echo "All Slither reports generated successfully" + fi diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index f43263f4661..63319d187b1 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -287,7 +287,15 @@ jobs: if [ -e "$file" ]; then cat "$file" >> $GITHUB_STEP_SUMMARY fi - done + done + + - name: Validate if all Slither run for all contracts + uses: ./.github/actions/validate-solidity-artifacts + with: + validate_slither_reports: 'true' + slither_reports_path: 'contracts/slither-reports' + sol_files: ${{ needs.changes.outputs.not_test_sol_modified_files }} + base_ref: ${{ github.base_ref }} - name: Upload Slither report uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 From ff9b2d9b33ab9e112f0e023dc91ab653e89d4849 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 13:50:11 +0200 Subject: [PATCH 33/54] fetch origin in validation action --- .github/actions/validate-solidity-artifacts/action.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index 3a65eddb578..732fce0034a 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -24,6 +24,10 @@ inputs: runs: using: composite steps: + - name: Fetch base ref + if: ${{ inputs.base_ref != '' }} + shell: bash + run: git fetch origin ${{ inputs.base_ref }} - name: Validate UML diagrams if: ${{ inputs.validate_uml_diagrams == 'true' }} shell: bash From e2912cc3ed1913396e437f688300c6431412184a Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 13:52:17 +0200 Subject: [PATCH 34/54] compare with HEAD in validate action --- .github/actions/validate-solidity-artifacts/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index 732fce0034a..98502309af6 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -38,7 +38,7 @@ runs: for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status ${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue @@ -74,7 +74,7 @@ runs: for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status ${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue From 77c7869be8b26805d7f873e47182cba9f50a4758 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 13:53:34 +0200 Subject: [PATCH 35/54] compare with origin in validation action --- .github/actions/validate-solidity-artifacts/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index 98502309af6..756d355cc90 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -38,7 +38,7 @@ runs: for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue @@ -74,7 +74,7 @@ runs: for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue From 6ce8e85853e6056c92830aac262d2968b1200771 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 14:30:01 +0200 Subject: [PATCH 36/54] handle both csv and shell arrays in the validation action --- .../validate-solidity-artifacts/action.yaml | 178 +++++++++++------- 1 file changed, 109 insertions(+), 69 deletions(-) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index 756d355cc90..fc5f4d5bbe4 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -14,7 +14,7 @@ inputs: description: Whether UML diagrams should be validated required: true sol_files: - description: Comma-separated list of Solidity files to check + description: Comma-separated (CSV) or space-separated (shell) list of Solidity files to check required: true # remove this when AurorNZ/paths-filter is fixed base_ref: @@ -24,78 +24,118 @@ inputs: runs: using: composite steps: - - name: Fetch base ref - if: ${{ inputs.base_ref != '' }} - shell: bash - run: git fetch origin ${{ inputs.base_ref }} - - name: Validate UML diagrams - if: ${{ inputs.validate_uml_diagrams == 'true' }} - shell: bash - run: | - echo "Validating UML diagrams" - IFS=',' read -r -a modified_files <<< "${{ inputs.sol_files }}" - missing_svgs=() - for file in "${modified_files[@]}"; do - if [ "${{ inputs.base_ref }}" != "" ]; then - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue + - name: Fetch base ref + if: ${{ inputs.base_ref != '' }} + shell: bash + run: git fetch origin ${{ inputs.base_ref }} + - name: Transform input array + id: transform_input_array + shell: bash + run: | + is_csv_format() { + local input="$1" + if [[ "$input" =~ "," ]]; then + return 0 + else + return 1 fi + } + + is_space_separated_string() { + local input="$1" + if [[ "$input" =~ ^[^[:space:]]+([[:space:]][^[:space:]]+)*$ ]]; then + return 0 + else + return 1 + fi + } + + array="${{ inputs.sol_files }}" + + if is_csv_format "$array"; then + echo "::debug::CSV format detected, nothing to do" + echo "sol_files=$array" >> $GITHUB_OUTPUT + exit 0 fi - - svg_file="$(basename "${file%.sol}").svg" - if [ ! -f "${{ inputs.uml_diagrams_path }}/$svg_file" ]; then - echo "Error: UML diagram for $file not found" - missing_svgs+=("$file") + + if is_space_separated_string "$array"; then + echo "::debug::Space-separated format detected, converting to CSV" + csv_array="${array// /,}" + echo "sol_files=$csv_array" >> $GITHUB_OUTPUT + exit 0 fi - done - - if [ ${#missing_svgs[@]} -gt 0 ]; then - echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_svgs[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY - else - echo "All UML diagrams generated successfully" - fi + + echo "::error::Invalid input format for sol_files. Please provide a comma-separated (CSV) or space-separated (shell) list of Solidity files" + exit 1 - - name: Validate Slither reports - if: ${{ inputs.validate_slither_reports == 'true' }} - shell: bash - run: | - echo "Validating Slither reports" - IFS=',' read -r -a modified_files <<< "${{ inputs.sol_files }}" - missing_reports=() - for file in "${modified_files[@]}"; do - if [ "${{ inputs.base_ref }}" != "" ]; then - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue + - name: Validate UML diagrams + if: ${{ inputs.validate_uml_diagrams == 'true' }} + shell: bash + run: | + echo "Validating UML diagrams" + IFS=',' read -r -a modified_files <<< "${{ steps.transform_input_array.outputs.sol_files }}" + missing_svgs=() + for file in "${modified_files[@]}"; do + if [ "${{ inputs.base_ref }}" != "" ]; then + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + fi + + svg_file="$(basename "${file%.sol}").svg" + if [ ! -f "${{ inputs.uml_diagrams_path }}/$svg_file" ]; then + echo "Error: UML diagram for $file not found" + missing_svgs+=("$file") fi + done + + if [ ${#missing_svgs[@]} -gt 0 ]; then + echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_svgs[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY + else + echo "All UML diagrams generated successfully" fi - - report_file="$(basename "${file%.sol}")-slither-report.md" - if [ ! -f "${{ inputs.slither_reports_path }}/$report_file" ]; then - echo "Error: Slither report for $file not found" - missing_reports+=("$file") + + - name: Validate Slither reports + if: ${{ inputs.validate_slither_reports == 'true' }} + shell: bash + run: | + echo "Validating Slither reports" + IFS=',' read -r -a modified_files <<< "${{ steps.transform_input_array.outputs.sol_files }}" + missing_reports=() + for file in "${modified_files[@]}"; do + if [ "${{ inputs.base_ref }}" != "" ]; then + # TODO remove this check when AurorNZ/paths-filter is fixed + status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + echo "File $file was deleted, skipping validation" + continue + fi + fi + + report_file="$(basename "${file%.sol}")-slither-report.md" + if [ ! -f "${{ inputs.slither_reports_path }}/$report_file" ]; then + echo "Error: Slither report for $file not found" + missing_reports+=("$file") + fi + done + + if [ ${#missing_reports[@]} -gt 0 ]; then + echo "Error: Missing Slither reports for files: ${missing_reports[@]}" + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY + for file in "${missing_reports[@]}"; do + echo " $file" >> $GITHUB_STEP_SUMMARY + done + echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY + else + echo "All Slither reports generated successfully" fi - done - - if [ ${#missing_reports[@]} -gt 0 ]; then - echo "Error: Missing Slither reports for files: ${missing_reports[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_reports[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY - else - echo "All Slither reports generated successfully" - fi From c2b8b48641329e9de9ae327c2379c91ac1d809df Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 15:06:26 +0200 Subject: [PATCH 37/54] update artifact pipeline with new actions --- .../workflows/solidity-foundry-artifacts.yml | 75 ++++--------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index bf5ce988520..8a55c8f0b9e 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -344,18 +344,13 @@ jobs: python-version: '3.8' - name: Install solc-select and solc - run: | - sudo apt-get update - sudo apt-get install -y python3-pip - pip3 install solc-select - sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select - solc-select install 0.8.19 - solc-select use 0.8.19 + uses: ./.github/actions/setup-solc-select + with: + to_install: '0.8.19' + to_use: '0.8.19' - name: Install Slither - run: | - python -m pip install --upgrade pip - pip install slither-analyzer + uses: ./.github/actions/setup-slither - name: Generate UML shell: bash @@ -386,57 +381,15 @@ jobs: contracts/slither-reports retention-days: 7 - - name: Validate UMLs and Slither reports - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} - run: | - echo "Validating UMLs and Slither reports" - IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" - missing_svgs=() - missing_reports=() - for file in "${modified_files[@]}"; do - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status ${{ inputs.base_ref }} ${{ github.sha }} -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue - fi - - svg_file="$(basename "${file%.sol}").svg" - if [ ! -f "contracts/uml-diagrams/$svg_file" ]; then - echo "Error: UML diagram for $file not found" - missing_svgs+=("$file") - fi - - report_file="$(basename "${file%.sol}")-slither-report.md" - if [ ! -f "contracts/slither-reports/$report_file" ]; then - echo "Error: Slither report for $file not found" - missing_reports+=("$file") - fi - done - - if [ ${#missing_svgs[@]} -gt 0 ]; then - echo "Error: Missing UML diagrams for files: ${missing_svgs[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing UML diagrams for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_svgs[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally or using a different tool" >> $GITHUB_STEP_SUMMARY - else - echo "All UML diagrams generated successfully" - fi - - if [ ${#missing_reports[@]} -gt 0 ]; then - echo "Error: Missing Slither reports for files: ${missing_reports[@]}" - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Missing Slither reports for files:" >> $GITHUB_STEP_SUMMARY - for file in "${missing_reports[@]}"; do - echo " $file" >> $GITHUB_STEP_SUMMARY - done - echo "## Action required: Please try to generate artifacts for them locally" >> $GITHUB_STEP_SUMMARY - else - echo "All Slither reports generated successfully" - fi + - name: Validate if all Slither run for all contracts + uses: ./.github/actions/validate-solidity-artifacts + with: + validate_slither_reports: 'true' + validate_uml_diagrams: 'true' + slither_reports_path: 'contracts/slither-reports' + uml_diagrams_path: 'contracts/uml-diagrams' + sol_files: ${{ needs.changes.outputs.not_test_sol_modified_files }} + base_ref: ${{ inputs.base_ref }} gather-all-artifacts: name: Gather all artifacts From e05a9b4a34fbda4cc2e5711120db49e889ed74d3 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 18:10:35 +0200 Subject: [PATCH 38/54] fix workflow after tests --- .../workflows/solidity-foundry-artifacts.yml | 62 +++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index 8a55c8f0b9e..c534926638c 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -50,6 +50,8 @@ jobs: ignored: &ignored - '!contracts/src/v0.8/**/test/**' - '!contracts/src/v0.8/**/tests/**' + - '!contracts/src/v0.8/**/mock/**' + - '!contracts/src/v0.8/**/mocks/**' - '!contracts/src/v0.8/**/*.t.sol' - '!contracts/src/v0.8/*.t.sol' - '!contracts/src/v0.8/**/testhelpers/**' @@ -64,7 +66,7 @@ jobs: - 'contracts/src/v0.8/**/*.sol' - *ignored product: &product - - 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' + - 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' - *ignored included: - *product @@ -81,13 +83,49 @@ jobs: changeset: - modified|added: 'contracts/.changeset/!(README)*.md' + # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it + - name: Check if changes are limited only to deleted files + if: ${{ steps.changes.outputs.sol == 'true' }} + shell: bash + run: | + echo "::debug::All modified contracts:" + echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' + + type=$(git cat-file -t "${{ inputs.base_ref }}") + if echo "$type" | grep -q commit; then + # we do not know to which branch the base ref belongs, so we fetch all branches + git fetch --all --prune + ref_to_use="${{ inputs.base_ref }}" + elif echo "$type" | grep -q tag; then + git fetch origin ${{ inputs.base_ref }} + ref_to_use="origin/${{ inputs.base_ref }}" + else + # must be branch + echo "::debug:: Base reference type: $type" + git fetch origin ${{ inputs.base_ref }} + ref_to_use="origin/${{ inputs.base_ref }}" + fi + + IFS=',' + for file in ${{ steps.changes.outputs.sol_files }}; do + status=$(git diff --name-status "$ref_to_use" HEAD -- "$file" | awk '{ print $1 }') + if [ "$status" != "D" ]; then + exit 0 + fi + done + + echo "::error:: All contract modifications are limited to deleted files. No artifacts will be generated." + echo "# Solidity Review Artifact NOT Generated" >> $GITHUB_STEP_SUMMARY + echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY + echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Only deleted Solidity files found for ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + exit 1 + - name: Check for changes outside of artifact scope if: ${{ steps.changes.outputs.sol == 'true' }} shell: bash run: | echo "::debug::All modified contracts:" - echo ${{ steps.changes.outputs.all_sol_files }} | tr ',' '\n' - echo "::debug::Filtered modified contracts:" echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' excluded_paths_pattern="!/^contracts\/src\/v0\.8\/shared/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" echo "::debug::Excluded paths: $excluded_paths_pattern" @@ -131,14 +169,14 @@ jobs: shell: bash run: | echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found No modified contracts for product ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found no modified contracts for product ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY echo "## Action required: Please check if you chose correct product and base reference" >> $GITHUB_STEP_SUMMARY echo "## Product: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY echo "## Base Ref: ${{ inputs.base_ref }}" >> $GITHUB_STEP_SUMMARY gather-basic-info: name: Gather basic info - if: ${{ steps.changes.outputs.product == 'true' || steps.changes.outputs.shared == 'true' }} + if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} runs-on: ubuntu-22.04 needs: [ changes ] steps: @@ -151,10 +189,10 @@ jobs: - name: Copy modified changesets if: ${{ needs.changes.outputs.changeset_changes == 'true' }} run: | - mkdir -p contracts/changesets/changesets + mkdir -p contracts/changesets for changeset in "${{ needs.changes.outputs.changeset_files }}"; do echo "::debug:: Copying $changeset" - cp $changeset contracts/changesets/changesets + cp $changeset contracts/changesets done - name: Generate basic info and modified contracts list @@ -223,7 +261,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ inputs.FOUNDRY_VERSION }} # required for code coverage report generation - name: Setup LCOV @@ -332,7 +370,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ inputs.FOUNDRY_VERSION }} - name: Install Sol2uml run: | @@ -388,7 +426,7 @@ jobs: validate_uml_diagrams: 'true' slither_reports_path: 'contracts/slither-reports' uml_diagrams_path: 'contracts/uml-diagrams' - sol_files: ${{ needs.changes.outputs.not_test_sol_modified_files }} + sol_files: ${{ needs.changes.outputs.files }} base_ref: ${{ inputs.base_ref }} gather-all-artifacts: @@ -439,5 +477,7 @@ jobs: echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY echo "## Reason: No modified Solidity files found for ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY - echo "No modified Solidity files found between ${{ inputs.base_ref }} and ${{ github.sha }} commits or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY + echo "* no modified Solidity files found between ${{ inputs.base_ref }} and ${{ github.sha }} commits" >> $GITHUB_STEP_SUMMARY + echo "* or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY + echo "* or they were limited to test files" >> $GITHUB_STEP_SUMMARY exit 1 From d845abeea8ab5837e38daa727c4d0c9ae4b6d48f Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Aug 2024 18:12:16 +0200 Subject: [PATCH 39/54] fix how validation actions works with commits --- .../validate-solidity-artifacts/action.yaml | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index fc5f4d5bbe4..b56191f9aee 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -26,8 +26,23 @@ runs: steps: - name: Fetch base ref if: ${{ inputs.base_ref != '' }} + id: fetch_base_ref shell: bash - run: git fetch origin ${{ inputs.base_ref }} + run: | + type=$(git cat-file -t "${{ inputs.base_ref }}") + if echo "$type" | grep -q commit; then + # we do not know to which branch the base ref belongs, so we fetch all branches + git fetch --all --prune + echo "ref_to_use=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT + elif echo "$type" | grep -q tag; then + git fetch origin ${{ inputs.base_ref }} + echo "ref_to_use=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT + else + # must be branch + echo "::debug:: Base reference type: $type" + git fetch origin ${{ inputs.base_ref }} + echo "ref_to_use=origin/${{ inputs.base_ref }}" >> $GITHUB_OUTPUT + fi - name: Transform input array id: transform_input_array shell: bash @@ -72,13 +87,13 @@ runs: if: ${{ inputs.validate_uml_diagrams == 'true' }} shell: bash run: | - echo "Validating UML diagrams" + echo "Validating UML diagrams" IFS=',' read -r -a modified_files <<< "${{ steps.transform_input_array.outputs.sol_files }}" - missing_svgs=() + missing_svgs=() for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status "${{ steps.fetch_base_ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue @@ -114,7 +129,7 @@ runs: for file in "${modified_files[@]}"; do if [ "${{ inputs.base_ref }}" != "" ]; then # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status origin/${{ inputs.base_ref }} HEAD -- "$file" | awk '{ print $1 }') + status=$(git diff --name-status "${{ steps.fetch_base_ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') if [ "$status" == "D" ]; then echo "File $file was deleted, skipping validation" continue From 03f48a909b7c749b215a171211ef4008dcc68301 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 13:20:50 +0200 Subject: [PATCH 40/54] treat shared as any other product --- .../workflows/solidity-foundry-artifacts.yml | 112 +++++++----------- 1 file changed, 45 insertions(+), 67 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index c534926638c..24c151d61f1 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -15,6 +15,7 @@ on: - "liquiditymanager" - "llo-feeds" - "operatorforwarder" + - "shared" - "transmission" - "vrf" base_ref: @@ -32,9 +33,8 @@ jobs: runs-on: ubuntu-latest outputs: changes: ${{ steps.changes.outputs.sol }} - product_changes: ${{ steps.changes.outputs.product }} - shared_changes: ${{ steps.changes.outputs.shared }} - files: ${{ steps.changes.outputs.included_files }} + product_changes: ${{ steps.changes-transform.outputs.product_changes }} + files: ${{ steps.changes.changes-transform.product_files }} changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} steps: @@ -57,20 +57,15 @@ jobs: - '!contracts/src/v0.8/**/testhelpers/**' - '!contracts/src/v0.8/testhelpers/**' - '!contracts/src/v0.8/vendor/**' - shared: &shared - - 'contracts/src/v0.8/shared/**/*.sol' + other_shared: - 'contracts/src/v0.8/interfaces/**/*.sol' - 'contracts/src/v0.8/*.sol' - - *ignored + - *ignored sol: - 'contracts/src/v0.8/**/*.sol' - *ignored product: &product - 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' - - *ignored - included: - - *product - - *shared - *ignored - name: Find modified changesets @@ -84,7 +79,7 @@ jobs: - modified|added: 'contracts/.changeset/!(README)*.md' # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it - - name: Check if changes are limited only to deleted files + - name: Check if changes are not limited only to deleted files if: ${{ steps.changes.outputs.sol == 'true' }} shell: bash run: | @@ -127,7 +122,11 @@ jobs: run: | echo "::debug::All modified contracts:" echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' - excluded_paths_pattern="!/^contracts\/src\/v0\.8\/shared/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + if [ "${{ inputs.product }}" = "shared" ]; then + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/interfaces/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + else + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + fi echo "::debug::Excluded paths: $excluded_paths_pattern" unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") set -e @@ -150,13 +149,13 @@ jobs: set +e set +o pipefail - echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}, shared" + echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}" echo "Files:" echo "$unexpected_files" echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}, shared" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY echo "### Files:" >> $GITHUB_STEP_SUMMARY echo "$unexpected_files" >> $GITHUB_STEP_SUMMARY echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY @@ -164,19 +163,38 @@ jobs: echo "No unexpected files found." fi - - name: Check for changes only in shared - if: ${{ steps.changes.outputs.product == 'false' && steps.changes.outputs.shared == 'true' }} + # Manual transformation needed, because shared contracts have a different folder structure + - name: Transform modified files + id: changes-transform shell: bash run: | - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found no modified contracts for product ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY - echo "## Action required: Please check if you chose correct product and base reference" >> $GITHUB_STEP_SUMMARY - echo "## Product: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY - echo "## Base Ref: ${{ inputs.base_ref }}" >> $GITHUB_STEP_SUMMARY + if [ "${{ inputs.product }}" = "shared" ]; then + echo "::debug:: Product is shared, transforming changes" + if [[ "${{ steps.changes.outputs.product }} == 'true' && "${{ steps.changes.outputs.other_shared }}" == 'true' ]]; then + echo "::debug:: Changes were found in 'shared' folder and in 'interfaces' and root folders" + echo "product_changes=true" >> $GITHUB_OUTPUT + echo "product_files=${{ steps.changes.outputs.product_files }},${{ steps.changes.outputs.other_shared_files }}" >> $GITHUB_OUTPUT + elif [[ "${{ steps.changes.outputs.product }} == 'false' && "${{ steps.changes.outputs.other_shared }}" == 'true' ]]; then + echo "::debug:: Only contracts in' interfaces' and root folders were modified" + echo "product_changes=true" >> $GITHUB_OUTPUT + echo "product_files=${{ steps.changes.outputs.other_shared_files }}" >> $GITHUB_OUTPUT + elif [[ "${{ steps.changes.outputs.product }} == 'true' && "${{ steps.changes.outputs.other_shared }}" == 'false' ]]; then + echo "::debug:: Only contracts in 'shared' folder were modified" + echo "product_changes=true" >> $GITHUB_OUTPUT + echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT" + else + echo "::debug:: No contracts were modified" + echo "product_changes=false" >> $GITHUB_OUTPUT + echo "product_files=" >> $GITHUB_OUTPUT + fi + else + echo "product_changes=${{ steps.changes.outputs.product }}" >> $GITHUB_OUTPUT + echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT" + fi gather-basic-info: name: Gather basic info - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + if: ${{ needs.changes.outputs.product_changes == 'true' }} runs-on: ubuntu-22.04 needs: [ changes ] steps: @@ -231,7 +249,7 @@ jobs: # some of the artifacts can only be generated on product level, and we cannot scope them to single contracts # some product-level modifications might also require shared contracts changes, so if these happened we need to generate artifacts for shared contracts as well coverage-and-book: - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + if: ${{ needs.changes.outputs.product_changes == 'true' }} name: Generate Docs and Code Coverage reports runs-on: ubuntu-22.04 needs: [changes] @@ -276,15 +294,6 @@ jobs: env: FOUNDRY_PROFILE: ${{ inputs.product }} - - name: Run Forge build for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - run: | - forge --version - forge build - working-directory: contracts - env: - FOUNDRY_PROFILE: "shared" - - name: Run coverage for product contracts if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} working-directory: contracts @@ -292,23 +301,11 @@ jobs: env: FOUNDRY_PROFILE: ${{ inputs.product }} - - name: Run coverage for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - working-directory: contracts - run: forge coverage --report lcov --report-file code-coverage/lcov-shared.info - env: - FOUNDRY_PROFILE: "shared" - - name: Generate Code Coverage HTML report for product contracts if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} shell: bash run: cd contracts && genhtml code-coverage/lcov-${{ inputs.product }}.info --branch-coverage --output-directory code-coverage/${{ inputs.product }} - - name: Generate Code Coverage HTML report for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - shell: bash - run: cd contracts && genhtml code-coverage/lcov-shared.info --branch-coverage --output-directory code-coverage/shared - - name: Run Forge doc for product contracts if: ${{ needs.changes.outputs.product_changes == 'true' }} run: forge doc --build -o docs/${{ inputs.product }} @@ -316,13 +313,6 @@ jobs: env: FOUNDRY_PROFILE: ${{ inputs.product }} - - name: Run Forge doc for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - run: forge doc --build -o docs/shared - working-directory: contracts - env: - FOUNDRY_PROFILE: "shared" - - name: Upload Artifacts for product contracts if: ${{ needs.changes.outputs.product_changes == 'true' }} uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 @@ -336,22 +326,9 @@ jobs: contracts/code-coverage/${{ inputs.product }} retention-days: 7 - - name: Upload Artifacts for shared contracts - if: ${{ needs.changes.outputs.shared_changes == 'true' }} - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - timeout-minutes: 2 - continue-on-error: true - with: - name: tmp-shared-artifacts - path: | - contracts/docs/shared - contracts/code-coverage/lcov-shared.info - contracts/code-coverage/shared - retention-days: 7 - # Generates UML diagrams and Slither reports for modified contracts uml-static-analysis: - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + if: ${{ needs.changes.outputs.product_changes == 'true' }} name: Generate UML and Slither reports for modified contracts runs-on: ubuntu-22.04 needs: [changes] @@ -431,7 +408,7 @@ jobs: gather-all-artifacts: name: Gather all artifacts - if: ${{ needs.changes.outputs.product_changes == 'true' || needs.changes.outputs.shared_changes == 'true' }} + if: ${{ needs.changes.outputs.product_changes == 'true' }} runs-on: ubuntu-latest needs: [coverage-and-book, uml-static-analysis, gather-basic-info, changes] steps: @@ -446,6 +423,7 @@ jobs: with: name: review-artifacts-${{ inputs.product }}-${{ github.sha }} path: review_artifacts + retention-days: 60 - name: Remove temporary artifacts uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1 # v5.0 @@ -466,7 +444,7 @@ jobs: echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY notify-no-changes: - if: ${{ needs.changes.outputs.product_changes == 'false' && needs.changes.outputs.shared_changes == 'false' }} + if: ${{ needs.changes.outputs.product_changes == 'false' }} needs: [changes] runs-on: ubuntu-latest steps: From 7b472a83ca2af7a2b4ceb6cde63c3fb40131d50b Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 14:31:57 +0200 Subject: [PATCH 41/54] small fixes --- .../workflows/solidity-foundry-artifacts.yml | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index 24c151d61f1..635a3df2cf7 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -34,7 +34,7 @@ jobs: outputs: changes: ${{ steps.changes.outputs.sol }} product_changes: ${{ steps.changes-transform.outputs.product_changes }} - files: ${{ steps.changes.changes-transform.product_files }} + files: ${{ steps.changes-transform.outputs.product_files }} changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} steps: @@ -79,31 +79,37 @@ jobs: - modified|added: 'contracts/.changeset/!(README)*.md' # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it - - name: Check if changes are not limited only to deleted files - if: ${{ steps.changes.outputs.sol == 'true' }} + - name: Get reference to compare to + id: get-base-ref shell: bash run: | - echo "::debug::All modified contracts:" - echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' - type=$(git cat-file -t "${{ inputs.base_ref }}") if echo "$type" | grep -q commit; then - # we do not know to which branch the base ref belongs, so we fetch all branches - git fetch --all --prune - ref_to_use="${{ inputs.base_ref }}" + # we do not know to which branch the base ref belongs, so we fetch all branches + git fetch --all --prune + ref_to_use="${{ inputs.base_ref }}" elif echo "$type" | grep -q tag; then - git fetch origin ${{ inputs.base_ref }} - ref_to_use="origin/${{ inputs.base_ref }}" + git fetch origin ${{ inputs.base_ref }} + ref_to_use="origin/${{ inputs.base_ref }}" else - # must be branch - echo "::debug:: Base reference type: $type" - git fetch origin ${{ inputs.base_ref }} - ref_to_use="origin/${{ inputs.base_ref }}" + # must be branch + echo "::debug:: Base reference type: $type" + git fetch origin ${{ inputs.base_ref }} + ref_to_use="origin/${{ inputs.base_ref }}" fi + echo "ref_to_use=$ref_to_use" >> $GITHUB_OUTPUT + + - name: Check if changes are not limited only to deleted files + if: ${{ steps.changes.outputs.sol == 'true' }} + shell: bash + run: | + echo "::debug::All modified contracts:" + echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' IFS=',' - for file in ${{ steps.changes.outputs.sol_files }}; do - status=$(git diff --name-status "$ref_to_use" HEAD -- "$file" | awk '{ print $1 }') + for file in ${{ steps.changes.outputs.sol_files }}; do + # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it + status=$(git diff --name-status "${{ steps.get-base-ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') if [ "$status" != "D" ]; then exit 0 fi @@ -114,6 +120,7 @@ jobs: echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY echo "## Reason: Only deleted Solidity files found for ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + exit 1 - name: Check for changes outside of artifact scope @@ -125,10 +132,11 @@ jobs: if [ "${{ inputs.product }}" = "shared" ]; then excluded_paths_pattern="!/^contracts\/src\/v0\.8\/interfaces/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" else - excluded_paths_pattern="!/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/${{ inputs.product }}/" fi echo "::debug::Excluded paths: $excluded_paths_pattern" unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") + missing_files="" set -e set -o pipefail if [[ -n "$unexpected_files" ]]; then @@ -138,7 +146,14 @@ jobs: echo "Files: ${files[@]}" for file in "${files[@]}"; do - product=$(echo "$file" | awk -F'src/[^/]*/' '{print $2}' | cut -d'/' -f1) + # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it + status=$(git diff --name-status "${{ steps.get-base-ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') + if [ "$status" == "D" ]; then + continue + fi + missing_files+="$file" + + product=$(echo "$file" | awk -F'src/v0.8/' '{if ($2 ~ /\//) print substr($2, 1, index($2, "/")-1); else print "shared"}') if [[ ! " ${products[@]} " =~ " ${product} " ]]; then products+=("$product") productsStr+="$product, " @@ -149,15 +164,17 @@ jobs: set +e set +o pipefail + missing_files=$(echo $missing_files | tr ',' '\n') + echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}" echo "Files:" - echo "$unexpected_files" + echo "$missing_files" echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" echo "# Warning!" >> $GITHUB_STEP_SUMMARY echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY echo "### Files:" >> $GITHUB_STEP_SUMMARY - echo "$unexpected_files" >> $GITHUB_STEP_SUMMARY + echo "$missing_files" >> $GITHUB_STEP_SUMMARY echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY else echo "No unexpected files found." @@ -170,18 +187,18 @@ jobs: run: | if [ "${{ inputs.product }}" = "shared" ]; then echo "::debug:: Product is shared, transforming changes" - if [[ "${{ steps.changes.outputs.product }} == 'true' && "${{ steps.changes.outputs.other_shared }}" == 'true' ]]; then + if [[ "${{ steps.changes.outputs.product }}" == "true" && "${{ steps.changes.outputs.other_shared }}" == "true" ]]; then echo "::debug:: Changes were found in 'shared' folder and in 'interfaces' and root folders" echo "product_changes=true" >> $GITHUB_OUTPUT echo "product_files=${{ steps.changes.outputs.product_files }},${{ steps.changes.outputs.other_shared_files }}" >> $GITHUB_OUTPUT - elif [[ "${{ steps.changes.outputs.product }} == 'false' && "${{ steps.changes.outputs.other_shared }}" == 'true' ]]; then + elif [[ "${{ steps.changes.outputs.product }}" == "false" && "${{ steps.changes.outputs.other_shared }}" == "true" ]]; then echo "::debug:: Only contracts in' interfaces' and root folders were modified" echo "product_changes=true" >> $GITHUB_OUTPUT echo "product_files=${{ steps.changes.outputs.other_shared_files }}" >> $GITHUB_OUTPUT - elif [[ "${{ steps.changes.outputs.product }} == 'true' && "${{ steps.changes.outputs.other_shared }}" == 'false' ]]; then + elif [[ "${{ steps.changes.outputs.product }}" == "true" && "${{ steps.changes.outputs.other_shared }}" == "false" ]]; then echo "::debug:: Only contracts in 'shared' folder were modified" echo "product_changes=true" >> $GITHUB_OUTPUT - echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT" + echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT else echo "::debug:: No contracts were modified" echo "product_changes=false" >> $GITHUB_OUTPUT @@ -189,7 +206,7 @@ jobs: fi else echo "product_changes=${{ steps.changes.outputs.product }}" >> $GITHUB_OUTPUT - echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT" + echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT fi gather-basic-info: @@ -459,3 +476,4 @@ jobs: echo "* or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY echo "* or they were limited to test files" >> $GITHUB_STEP_SUMMARY exit 1 + From 024e9e3bfdf0e5140144dd51f29a6c8243684789 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 16:27:31 +0200 Subject: [PATCH 42/54] apply CR changes --- .github/actions/setup-slither/action.yaml | 4 ++-- .github/actions/setup-solc-select/action.yaml | 1 - .github/workflows/solidity-foundry.yml | 5 ----- contracts/scripts/ci/generate_slither_report.sh | 2 +- contracts/scripts/ci/modify_remappings.sh | 4 +++- contracts/scripts/ci/select_solc_version.sh | 2 +- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-slither/action.yaml b/.github/actions/setup-slither/action.yaml index d8e7bef9e5e..b8bef38575d 100644 --- a/.github/actions/setup-slither/action.yaml +++ b/.github/actions/setup-slither/action.yaml @@ -1,5 +1,5 @@ name: Setup Slither -description: Installs Slither for contract analysis. Requires Python 3.6 or higher. +description: Installs Slither 0.10.3 for contract analysis. Requires Python 3.6 or higher. runs: using: composite steps: @@ -7,4 +7,4 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install slither-analyzer + pip install slither-analyzer==0.10.3 diff --git a/.github/actions/setup-solc-select/action.yaml b/.github/actions/setup-solc-select/action.yaml index 7903b7acd3c..780fb6d3117 100644 --- a/.github/actions/setup-solc-select/action.yaml +++ b/.github/actions/setup-solc-select/action.yaml @@ -15,7 +15,6 @@ runs: shell: bash run: | sudo apt-get update - sudo apt-get install -y python3-pip pip3 install solc-select sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 63319d187b1..b4d9ec9066f 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -235,11 +235,6 @@ jobs: - name: Setup NodeJS uses: ./.github/actions/setup-nodejs - - name: Install semver - shell: bash - run: | - npm install -g semver - - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh index 50fc991e894..e1c526604d1 100755 --- a/contracts/scripts/ci/generate_slither_report.sh +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function check_chainlink_dir() { local param_dir="chainlink" diff --git a/contracts/scripts/ci/modify_remappings.sh b/contracts/scripts/ci/modify_remappings.sh index 72394a2fe89..a6736f966a7 100755 --- a/contracts/scripts/ci/modify_remappings.sh +++ b/contracts/scripts/ci/modify_remappings.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash + +set -euo pipefail if [ "$#" -ne 2 ]; then >&2 echo "Usage: $0 " diff --git a/contracts/scripts/ci/select_solc_version.sh b/contracts/scripts/ci/select_solc_version.sh index 316b261616a..057d2a29979 100755 --- a/contracts/scripts/ci/select_solc_version.sh +++ b/contracts/scripts/ci/select_solc_version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function check_chainlink_dir() { local param_dir="chainlink" From a222fe77568fccc255e0fdea252ca77bd0ebdd69 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 17:12:54 +0200 Subject: [PATCH 43/54] remove special handling for deleted files --- .../validate-solidity-artifacts/action.yaml | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/.github/actions/validate-solidity-artifacts/action.yaml b/.github/actions/validate-solidity-artifacts/action.yaml index b56191f9aee..5357a87f96b 100644 --- a/.github/actions/validate-solidity-artifacts/action.yaml +++ b/.github/actions/validate-solidity-artifacts/action.yaml @@ -16,33 +16,10 @@ inputs: sol_files: description: Comma-separated (CSV) or space-separated (shell) list of Solidity files to check required: true - # remove this when AurorNZ/paths-filter is fixed - base_ref: - description: Base reference for comparison (skip if your list doesn't contain deleted files) - required: false runs: using: composite steps: - - name: Fetch base ref - if: ${{ inputs.base_ref != '' }} - id: fetch_base_ref - shell: bash - run: | - type=$(git cat-file -t "${{ inputs.base_ref }}") - if echo "$type" | grep -q commit; then - # we do not know to which branch the base ref belongs, so we fetch all branches - git fetch --all --prune - echo "ref_to_use=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT - elif echo "$type" | grep -q tag; then - git fetch origin ${{ inputs.base_ref }} - echo "ref_to_use=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT - else - # must be branch - echo "::debug:: Base reference type: $type" - git fetch origin ${{ inputs.base_ref }} - echo "ref_to_use=origin/${{ inputs.base_ref }}" >> $GITHUB_OUTPUT - fi - name: Transform input array id: transform_input_array shell: bash @@ -91,15 +68,6 @@ runs: IFS=',' read -r -a modified_files <<< "${{ steps.transform_input_array.outputs.sol_files }}" missing_svgs=() for file in "${modified_files[@]}"; do - if [ "${{ inputs.base_ref }}" != "" ]; then - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status "${{ steps.fetch_base_ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue - fi - fi - svg_file="$(basename "${file%.sol}").svg" if [ ! -f "${{ inputs.uml_diagrams_path }}/$svg_file" ]; then echo "Error: UML diagram for $file not found" @@ -127,15 +95,6 @@ runs: IFS=',' read -r -a modified_files <<< "${{ steps.transform_input_array.outputs.sol_files }}" missing_reports=() for file in "${modified_files[@]}"; do - if [ "${{ inputs.base_ref }}" != "" ]; then - # TODO remove this check when AurorNZ/paths-filter is fixed - status=$(git diff --name-status "${{ steps.fetch_base_ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - echo "File $file was deleted, skipping validation" - continue - fi - fi - report_file="$(basename "${file%.sol}")-slither-report.md" if [ ! -f "${{ inputs.slither_reports_path }}/$report_file" ]; then echo "Error: Slither report for $file not found" From aa0af665887de4cdd4ebf5c77c8ef01d5daa4f7a Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 17:14:06 +0200 Subject: [PATCH 44/54] remove apt-get update --- .github/actions/setup-solc-select/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/setup-solc-select/action.yaml b/.github/actions/setup-solc-select/action.yaml index 780fb6d3117..b74ffae018d 100644 --- a/.github/actions/setup-solc-select/action.yaml +++ b/.github/actions/setup-solc-select/action.yaml @@ -14,7 +14,6 @@ runs: - name: Install solc-select and solc shell: bash run: | - sudo apt-get update pip3 install solc-select sudo ln -s /usr/local/bin/solc-select /usr/bin/solc-select From 8e22de6b7d3907fb855b6b899b44cd993386b408 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 17:20:15 +0200 Subject: [PATCH 45/54] use only dorny/paths --- .../workflows/solidity-foundry-artifacts.yml | 71 ++----------------- 1 file changed, 7 insertions(+), 64 deletions(-) diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index 635a3df2cf7..e4c553c9964 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -41,11 +41,12 @@ jobs: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Find modified contracts - uses: AurorNZ/paths-filter@3b1f3abc3371cca888d8eb03dfa70bc8a9867629 # v4.0.0 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes with: list-files: 'csv' base: ${{ inputs.base_ref }} + predicate-quantifier: every filters: | ignored: &ignored - '!contracts/src/v0.8/**/test/**' @@ -58,70 +59,17 @@ jobs: - '!contracts/src/v0.8/testhelpers/**' - '!contracts/src/v0.8/vendor/**' other_shared: - - 'contracts/src/v0.8/interfaces/**/*.sol' - - 'contracts/src/v0.8/*.sol' + - modified|added: 'contracts/src/v0.8/interfaces/**/*.sol' + - modified|added: 'contracts/src/v0.8/*.sol' - *ignored sol: - - 'contracts/src/v0.8/**/*.sol' + - modified|added: 'contracts/src/v0.8/**/*.sol' - *ignored product: &product - - 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' + - modified|added: 'contracts/src/v0.8/${{ inputs.product }}/**/*.sol' - *ignored - - - name: Find modified changesets - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: changes-dorny - with: - list-files: 'shell' - base: ${{ inputs.base_ref }} - filters: | changeset: - - modified|added: 'contracts/.changeset/!(README)*.md' - - # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it - - name: Get reference to compare to - id: get-base-ref - shell: bash - run: | - type=$(git cat-file -t "${{ inputs.base_ref }}") - if echo "$type" | grep -q commit; then - # we do not know to which branch the base ref belongs, so we fetch all branches - git fetch --all --prune - ref_to_use="${{ inputs.base_ref }}" - elif echo "$type" | grep -q tag; then - git fetch origin ${{ inputs.base_ref }} - ref_to_use="origin/${{ inputs.base_ref }}" - else - # must be branch - echo "::debug:: Base reference type: $type" - git fetch origin ${{ inputs.base_ref }} - ref_to_use="origin/${{ inputs.base_ref }}" - fi - echo "ref_to_use=$ref_to_use" >> $GITHUB_OUTPUT - - - name: Check if changes are not limited only to deleted files - if: ${{ steps.changes.outputs.sol == 'true' }} - shell: bash - run: | - echo "::debug::All modified contracts:" - echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' - - IFS=',' - for file in ${{ steps.changes.outputs.sol_files }}; do - # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it - status=$(git diff --name-status "${{ steps.get-base-ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" != "D" ]; then - exit 0 - fi - done - - echo "::error:: All contract modifications are limited to deleted files. No artifacts will be generated." - echo "# Solidity Review Artifact NOT Generated" >> $GITHUB_STEP_SUMMARY - echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY - echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Only deleted Solidity files found for ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY - - exit 1 + - modified|added: 'contracts/.changeset/!(README)*.md' - name: Check for changes outside of artifact scope if: ${{ steps.changes.outputs.sol == 'true' }} @@ -146,11 +94,6 @@ jobs: echo "Files: ${files[@]}" for file in "${files[@]}"; do - # TODO remove when https://github.com/AurorNZ/paths-filter/issues/12 is fixed and we can use modifiers with it - status=$(git diff --name-status "${{ steps.get-base-ref.outputs.ref_to_use }}" HEAD -- "$file" | awk '{ print $1 }') - if [ "$status" == "D" ]; then - continue - fi missing_files+="$file" product=$(echo "$file" | awk -F'src/v0.8/' '{if ($2 ~ /\//) print substr($2, 1, index($2, "/")-1); else print "shared"}') From afdef19491d6364f933b280a7de2b9f23ccbd237 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 17:20:45 +0200 Subject: [PATCH 46/54] remove unused input --- .github/workflows/solidity-foundry.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index b4d9ec9066f..08039456bc0 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -290,7 +290,6 @@ jobs: validate_slither_reports: 'true' slither_reports_path: 'contracts/slither-reports' sol_files: ${{ needs.changes.outputs.not_test_sol_modified_files }} - base_ref: ${{ github.base_ref }} - name: Upload Slither report uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 From 7b7b9ddac6de536bc84b3b45bd6104948bbf2650 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 17:41:16 +0200 Subject: [PATCH 47/54] CR changes: use dorny/paths with quantifier, move scope validation to an action, remove whitespaces --- .../validate-artifact-scope/action.yaml | 104 +++++++++++++++++ .../workflows/solidity-foundry-artifacts.yml | 110 ++++++------------ contracts/scripts/ci/generate_uml.sh | 4 +- 3 files changed, 143 insertions(+), 75 deletions(-) create mode 100644 .github/actions/validate-artifact-scope/action.yaml diff --git a/.github/actions/validate-artifact-scope/action.yaml b/.github/actions/validate-artifact-scope/action.yaml new file mode 100644 index 00000000000..51ec9bfa143 --- /dev/null +++ b/.github/actions/validate-artifact-scope/action.yaml @@ -0,0 +1,104 @@ +name: Validate Artifact Scope +description: Checks there are any modified Solidity files outside of the specified scope. If so, it prints a warning message, but does not fail the workflow. +inputs: + product: + description: The product for which the artifacts are being generated + required: true + sol_files: + description: Comma-separated (CSV) or space-separated (shell) list of Solidity files to check + required: true + +runs: + using: composite + steps: + - name: Transform input array + id: transform_input_array + shell: bash + run: | + is_csv_format() { + local input="$1" + if [[ "$input" =~ "," ]]; then + return 0 + else + return 1 + fi + } + + is_space_separated_string() { + local input="$1" + if [[ "$input" =~ ^[^[:space:]]+([[:space:]][^[:space:]]+)*$ ]]; then + return 0 + else + return 1 + fi + } + + array="${{ inputs.sol_files }}" + + if is_csv_format "$array"; then + echo "::debug::CSV format detected, nothing to do" + echo "sol_files=$array" >> $GITHUB_OUTPUT + exit 0 + fi + + if is_space_separated_string "$array"; then + echo "::debug::Space-separated format detected, converting to CSV" + csv_array="${array// /,}" + echo "sol_files=$csv_array" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "::error::Invalid input format for sol_files. Please provide a comma-separated (CSV) or space-separated (shell) list of Solidity files" + exit 1 + + - name: Check for changes outside of artifact scope + if: ${{ steps.changes.outputs.sol == 'true' }} + shell: bash + run: | + echo "::debug::All modified contracts:" + echo "${{ steps.transform_input_array.outputs.sol_files }}" | tr ',' '\n' + if [ "${{ inputs.product }}" = "shared" ]; then + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/interfaces/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" + else + excluded_paths_pattern="!/^contracts\/src\/v0\.8\/${{ inputs.product }}/" + fi + echo "::debug::Excluded paths: $excluded_paths_pattern" + unexpected_files=$(echo "${{ steps.transform_input_array.outputs.sol_files }}" | tr ',' '\n' | awk "$excluded_paths_pattern") + missing_files="" + set -e + set -o pipefail + if [[ -n "$unexpected_files" ]]; then + products=() + productsStr="" + IFS=$'\n' read -r -d '' -a files <<< "$unexpected_files" || true + echo "Files: ${files[@]}" + + for file in "${files[@]}"; do + missing_files+="$file" + + product=$(echo "$file" | awk -F'src/v0.8/' '{if ($2 ~ /\//) print substr($2, 1, index($2, "/")-1); else print "shared"}') + if [[ ! " ${products[@]} " =~ " ${product} " ]]; then + products+=("$product") + productsStr+="$product, " + fi + done + productsStr=${productsStr%, } + + set +e + set +o pipefail + + missing_files=$(echo $missing_files | tr ',' '\n') + + echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}" + echo "Files:" + echo "$missing_files" + echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" + + echo "# Warning!" >> $GITHUB_STEP_SUMMARY + echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY + echo "### Files:" >> $GITHUB_STEP_SUMMARY + echo "$missing_files" >> $GITHUB_STEP_SUMMARY + echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY + else + echo "No unexpected files found." + fi diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index e4c553c9964..7de8631e6e3 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -24,8 +24,8 @@ on: env: FOUNDRY_PROFILE: ci - # Has to match the `make foundry` version. - FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 + # Empty, because it's extracted from contracts/GNUMakefile + FOUNDRY_VERSION: "" jobs: changes: @@ -34,7 +34,7 @@ jobs: outputs: changes: ${{ steps.changes.outputs.sol }} product_changes: ${{ steps.changes-transform.outputs.product_changes }} - files: ${{ steps.changes-transform.outputs.product_files }} + product_files: ${{ steps.changes-transform.outputs.product_files }} changeset_changes: ${{ steps.changes-dorny.outputs.changeset }} changeset_files: ${{ steps.changes-dorny.outputs.changeset_files }} steps: @@ -72,56 +72,10 @@ jobs: - modified|added: 'contracts/.changeset/!(README)*.md' - name: Check for changes outside of artifact scope - if: ${{ steps.changes.outputs.sol == 'true' }} - shell: bash - run: | - echo "::debug::All modified contracts:" - echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' - if [ "${{ inputs.product }}" = "shared" ]; then - excluded_paths_pattern="!/^contracts\/src\/v0\.8\/interfaces/ && !/^contracts\/src\/v0\.8\/${{ inputs.product }}/ && !/^contracts\/src\/v0\.8\/[^\/]+\.sol$/" - else - excluded_paths_pattern="!/^contracts\/src\/v0\.8\/${{ inputs.product }}/" - fi - echo "::debug::Excluded paths: $excluded_paths_pattern" - unexpected_files=$(echo ${{ steps.changes.outputs.sol_files }} | tr ',' '\n' | awk "$excluded_paths_pattern") - missing_files="" - set -e - set -o pipefail - if [[ -n "$unexpected_files" ]]; then - products=() - productsStr="" - IFS=$'\n' read -r -d '' -a files <<< "$unexpected_files" || true - echo "Files: ${files[@]}" - - for file in "${files[@]}"; do - missing_files+="$file" - - product=$(echo "$file" | awk -F'src/v0.8/' '{if ($2 ~ /\//) print substr($2, 1, index($2, "/")-1); else print "shared"}') - if [[ ! " ${products[@]} " =~ " ${product} " ]]; then - products+=("$product") - productsStr+="$product, " - fi - done - productsStr=${productsStr%, } - - set +e - set +o pipefail - - missing_files=$(echo $missing_files | tr ',' '\n') - - echo "Error: Found modified contracts outside of the expected scope: ${{ inputs.product }}" - echo "Files:" - echo "$missing_files" - echo "Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" - - echo "# Warning!" >> $GITHUB_STEP_SUMMARY - echo "## Reason: Found modified contracts outside of the expected scope: ${{ inputs.product }}" >> $GITHUB_STEP_SUMMARY - echo "### Files:" >> $GITHUB_STEP_SUMMARY - echo "$missing_files" >> $GITHUB_STEP_SUMMARY - echo "## Action required: If you want to generate artifacts for other products ($productsStr) run this workflow again with updated configuration" >> $GITHUB_STEP_SUMMARY - else - echo "No unexpected files found." - fi + uses: ./.github/actions/validate-artifact-scope + with: + sol_files: ${{ steps.changes.outputs.sol_files }} + product: ${{ inputs.product }} # Manual transformation needed, because shared contracts have a different folder structure - name: Transform modified files @@ -161,9 +115,21 @@ jobs: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: - submodules: recursive fetch-depth: 0 + - name: Extract Foundry version + id: extract-foundry-version + shell: bash + working-directory: contracts + run: | + foundry_version=$(grep -Eo "foundryup --version [^ ]+" GNUmakefile | awk '{print $3}') + if [ -z "$foundry_version" ]; then + echo "Error: Foundry version not found in GNUmakefile" + exit 1 + fi + echo "Foundry version found: $foundry_version" + echo "FOUNDRY_VERSION=$foundry_version" >> $GITHUB_OUTPUT + - name: Copy modified changesets if: ${{ needs.changes.outputs.changeset_changes == 'true' }} run: | @@ -179,7 +145,7 @@ jobs: echo "Commit SHA used to generate artifacts: ${{ github.sha }}" > contracts/commit_sha_base_ref.txt echo "Base reference SHA used to find modified contracts: ${{ inputs.base_ref }}" >> contracts/commit_sha_base_ref.txt - IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.files }}" + IFS=',' read -r -a modified_files <<< "${{ needs.changes.outputs.product_files }}" echo "# Modified contracts:" > contracts/modified_contracts.md for file in "${modified_files[@]}"; do # TODO remove this check when AurorNZ/paths-filter is fixed @@ -202,7 +168,7 @@ jobs: path: | contracts/modified_contracts.md contracts/modified_contracts.txt - contracts/commit_sha_base_ref.txt + contracts/commit_sha_base_ref.txt contracts/changesets retention-days: 7 @@ -225,8 +191,6 @@ jobs: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - submodules: recursive - name: Setup NodeJS uses: ./.github/actions/setup-nodejs @@ -239,7 +203,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ inputs.FOUNDRY_VERSION }} + version: ${{ env.FOUNDRY_VERSION }} # required for code coverage report generation - name: Setup LCOV @@ -257,18 +221,19 @@ jobs: - name: Run coverage for product contracts if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} working-directory: contracts - run: forge coverage --report lcov --report-file code-coverage/lcov-${{ inputs.product }}.info + run: forge coverage --report lcov --report-file code-coverage/lcov.info env: FOUNDRY_PROFILE: ${{ inputs.product }} - name: Generate Code Coverage HTML report for product contracts if: ${{ !contains(fromJson(steps.prepare-exclusion-list.outputs.coverage_exclusions), inputs.product) && needs.changes.outputs.product_changes == 'true' }} shell: bash - run: cd contracts && genhtml code-coverage/lcov-${{ inputs.product }}.info --branch-coverage --output-directory code-coverage/${{ inputs.product }} + working-directory: contracts + run: genhtml code-coverage/lcov.info --branch-coverage --output-directory code-coverage - name: Run Forge doc for product contracts if: ${{ needs.changes.outputs.product_changes == 'true' }} - run: forge doc --build -o docs/${{ inputs.product }} + run: forge doc --build -o docs working-directory: contracts env: FOUNDRY_PROFILE: ${{ inputs.product }} @@ -281,9 +246,9 @@ jobs: with: name: tmp-${{ inputs.product }}-artifacts path: | - contracts/docs/${{ inputs.product }} - contracts/code-coverage/lcov-${{ inputs.product }}.info - contracts/code-coverage/${{ inputs.product }} + contracts/docs + contracts/code-coverage/lcov-.info + contracts/code-coverage retention-days: 7 # Generates UML diagrams and Slither reports for modified contracts @@ -293,12 +258,11 @@ jobs: runs-on: ubuntu-22.04 needs: [changes] env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: - submodules: recursive fetch-depth: 0 - name: Setup NodeJS @@ -307,7 +271,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ inputs.FOUNDRY_VERSION }} + version: ${{ env.FOUNDRY_VERSION }} - name: Install Sol2uml run: | @@ -330,7 +294,7 @@ jobs: - name: Generate UML shell: bash run: | - contract_list="${{ needs.changes.outputs.files }}" + contract_list="${{ needs.changes.outputs.product_files }}" # modify remappings so that solc can find dependencies ./contracts/scripts/ci/modify_remappings.sh contracts contracts/remappings.txt @@ -340,7 +304,7 @@ jobs: - name: Generate Slither Markdown reports run: | - contract_list="${{ needs.changes.outputs.files }}" + contract_list="${{ needs.changes.outputs.product_files }}" echo "::debug::Processing contracts: $contract_list" ./contracts/scripts/ci/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}/" contracts/configs/slither/.slither.config-artifacts.json "." "$contract_list" "contracts/slither-reports" "--solc-remaps @=contracts/node_modules/@" @@ -363,7 +327,7 @@ jobs: validate_uml_diagrams: 'true' slither_reports_path: 'contracts/slither-reports' uml_diagrams_path: 'contracts/uml-diagrams' - sol_files: ${{ needs.changes.outputs.files }} + sol_files: ${{ needs.changes.outputs.product_files }} base_ref: ${{ inputs.base_ref }} gather-all-artifacts: @@ -392,7 +356,7 @@ jobs: - name: Print Artifact URL in job summary env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ARTIFACTS=$(gh api -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts) ARTIFACT_ID=$(echo "$ARTIFACTS" | jq '.artifacts[] | select(.name=="review-artifacts-${{ inputs.product }}-${{ github.sha }}") | .id') @@ -401,7 +365,7 @@ jobs: echo "# Solidity Review Artifact Generated" >> $GITHUB_STEP_SUMMARY echo "Base Ref used: **${{ inputs.base_ref }}**" >> $GITHUB_STEP_SUMMARY echo "Commit SHA used: **${{ github.sha }}**" >> $GITHUB_STEP_SUMMARY - echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY + echo "[Artifact URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID)" >> $GITHUB_STEP_SUMMARY notify-no-changes: if: ${{ needs.changes.outputs.product_changes == 'false' }} diff --git a/contracts/scripts/ci/generate_uml.sh b/contracts/scripts/ci/generate_uml.sh index fbab2724205..341995ab361 100755 --- a/contracts/scripts/ci/generate_uml.sh +++ b/contracts/scripts/ci/generate_uml.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function check_chainlink_dir() { local param_dir="chainlink" @@ -6,7 +6,7 @@ function check_chainlink_dir() { current_base=$(basename "$current_dir") - if [ "$current_base" != "$param_dir" ]; then + if [[ "$current_base" != "$param_dir" ]]; then >&2 echo "The script must be run from the root of $param_dir directory" exit 1 fi From 0928fb978387c28a244546fbbaee4e999e4831e4 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 7 Aug 2024 18:26:55 +0200 Subject: [PATCH 48/54] fix workflow --- .../validate-artifact-scope/action.yaml | 7 ++-- .../workflows/solidity-foundry-artifacts.yml | 36 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/actions/validate-artifact-scope/action.yaml b/.github/actions/validate-artifact-scope/action.yaml index 51ec9bfa143..7440efc63a3 100644 --- a/.github/actions/validate-artifact-scope/action.yaml +++ b/.github/actions/validate-artifact-scope/action.yaml @@ -2,8 +2,8 @@ name: Validate Artifact Scope description: Checks there are any modified Solidity files outside of the specified scope. If so, it prints a warning message, but does not fail the workflow. inputs: product: - description: The product for which the artifacts are being generated - required: true + description: The product for which the artifacts are being generated + required: true sol_files: description: Comma-separated (CSV) or space-separated (shell) list of Solidity files to check required: true @@ -52,7 +52,6 @@ runs: exit 1 - name: Check for changes outside of artifact scope - if: ${{ steps.changes.outputs.sol == 'true' }} shell: bash run: | echo "::debug::All modified contracts:" @@ -74,7 +73,7 @@ runs: echo "Files: ${files[@]}" for file in "${files[@]}"; do - missing_files+="$file" + missing_files+="$file," product=$(echo "$file" | awk -F'src/v0.8/' '{if ($2 ~ /\//) print substr($2, 1, index($2, "/")-1); else print "shared"}') if [[ ! " ${products[@]} " =~ " ${product} " ]]; then diff --git a/.github/workflows/solidity-foundry-artifacts.yml b/.github/workflows/solidity-foundry-artifacts.yml index 7de8631e6e3..a6da89d4712 100644 --- a/.github/workflows/solidity-foundry-artifacts.yml +++ b/.github/workflows/solidity-foundry-artifacts.yml @@ -24,8 +24,6 @@ on: env: FOUNDRY_PROFILE: ci - # Empty, because it's extracted from contracts/GNUMakefile - FOUNDRY_VERSION: "" jobs: changes: @@ -59,8 +57,7 @@ jobs: - '!contracts/src/v0.8/testhelpers/**' - '!contracts/src/v0.8/vendor/**' other_shared: - - modified|added: 'contracts/src/v0.8/interfaces/**/*.sol' - - modified|added: 'contracts/src/v0.8/*.sol' + - modified|added: 'contracts/src/v0.8/(interfaces/**/*.sol|*.sol)' - *ignored sol: - modified|added: 'contracts/src/v0.8/**/*.sol' @@ -71,12 +68,6 @@ jobs: changeset: - modified|added: 'contracts/.changeset/!(README)*.md' - - name: Check for changes outside of artifact scope - uses: ./.github/actions/validate-artifact-scope - with: - sol_files: ${{ steps.changes.outputs.sol_files }} - product: ${{ inputs.product }} - # Manual transformation needed, because shared contracts have a different folder structure - name: Transform modified files id: changes-transform @@ -106,11 +97,20 @@ jobs: echo "product_files=${{ steps.changes.outputs.product_files }}" >> $GITHUB_OUTPUT fi + - name: Check for changes outside of artifact scope + uses: ./.github/actions/validate-artifact-scope + if: ${{ steps.changes.outputs.sol == 'true' }} + with: + sol_files: ${{ steps.changes.outputs.sol_files }} + product: ${{ inputs.product }} + gather-basic-info: name: Gather basic info if: ${{ needs.changes.outputs.product_changes == 'true' }} runs-on: ubuntu-22.04 needs: [ changes ] + outputs: + foundry_version: ${{ steps.extract-foundry-version.outputs.foundry_version }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -128,13 +128,15 @@ jobs: exit 1 fi echo "Foundry version found: $foundry_version" - echo "FOUNDRY_VERSION=$foundry_version" >> $GITHUB_OUTPUT + echo "foundry_version=$foundry_version" >> $GITHUB_OUTPUT - name: Copy modified changesets if: ${{ needs.changes.outputs.changeset_changes == 'true' }} run: | mkdir -p contracts/changesets - for changeset in "${{ needs.changes.outputs.changeset_files }}"; do + files="${{ needs.changes.outputs.changeset_files }}" + IFS=",' + for changeset in $files; do echo "::debug:: Copying $changeset" cp $changeset contracts/changesets done @@ -178,7 +180,7 @@ jobs: if: ${{ needs.changes.outputs.product_changes == 'true' }} name: Generate Docs and Code Coverage reports runs-on: ubuntu-22.04 - needs: [changes] + needs: [changes, gather-basic-info] steps: - name: Prepare exclusion list id: prepare-exclusion-list @@ -203,7 +205,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ needs.gather-basic-info.outputs.foundry_version }} # required for code coverage report generation - name: Setup LCOV @@ -256,7 +258,7 @@ jobs: if: ${{ needs.changes.outputs.product_changes == 'true' }} name: Generate UML and Slither reports for modified contracts runs-on: ubuntu-22.04 - needs: [changes] + needs: [changes, gather-basic-info] env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -271,7 +273,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ needs.gather-basic-info.outputs.foundry_version }} - name: Install Sol2uml run: | @@ -328,7 +330,6 @@ jobs: slither_reports_path: 'contracts/slither-reports' uml_diagrams_path: 'contracts/uml-diagrams' sol_files: ${{ needs.changes.outputs.product_files }} - base_ref: ${{ inputs.base_ref }} gather-all-artifacts: name: Gather all artifacts @@ -383,4 +384,3 @@ jobs: echo "* or they are located outside of ./contracts/src/v0.8 folder" >> $GITHUB_STEP_SUMMARY echo "* or they were limited to test files" >> $GITHUB_STEP_SUMMARY exit 1 - From 7d46431a8768942c45febc282ad38e764057dd54 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 10:19:31 +0200 Subject: [PATCH 49/54] fail bash scripts on erors --- contracts/scripts/ci/generate_slither_report.sh | 14 +++++++------- contracts/scripts/ci/generate_uml.sh | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh index e1c526604d1..e51dd82828b 100755 --- a/contracts/scripts/ci/generate_slither_report.sh +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + function check_chainlink_dir() { local param_dir="chainlink" current_dir=$(pwd) @@ -31,16 +33,14 @@ run_slither() { local FILE=$1 local TARGET_DIR=$2 - # needed, because the action we use returns all modified files, also deleted ones and we must skip those if [ ! -f "$FILE" ]; then - echo "Warning: File not found: $FILE" - echo "Skipping..." - return + >&2 echo "::error:File not found: $FILE" + exit 1 fi source ./contracts/scripts/ci/select_solc_version.sh "$FILE" if [ $? -ne 0 ]; then - >&2 echo "Error: Failed to select Solc version for $FILE" + >&2 echo "::error::Failed to select Solc version for $FILE" exit 1 fi @@ -48,7 +48,7 @@ run_slither() { output=$(slither --config-file "$CONFIG_FILE" "$FILE" --checklist --markdown-root "$REPO_URL" --fail-none $SLITHER_EXTRA_PARAMS) if [ $? -ne 0 ]; then - >&2 echo "Slither failed for $FILE" + >&2 echo "::error::Slither failed for $FILE" exit 1 fi output=$(echo "$output" | sed '/\*\*THIS CHECKLIST IS NOT COMPLETE\*\*. Use `--show-ignored-findings` to show all the results./d' | sed '/Summary/d') @@ -73,7 +73,7 @@ process_files() { process_files "$SOURCE_DIR" "$TARGET_DIR" "${FILES[@]}" if [ $? -ne 0 ]; then - >&2 echo "Error: Failed to generate Slither reports" + >&2 echo "::error::Failed to generate Slither reports" exit 1 fi diff --git a/contracts/scripts/ci/generate_uml.sh b/contracts/scripts/ci/generate_uml.sh index 341995ab361..703628b35b0 100755 --- a/contracts/scripts/ci/generate_uml.sh +++ b/contracts/scripts/ci/generate_uml.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + function check_chainlink_dir() { local param_dir="chainlink" current_dir=$(pwd) @@ -29,6 +31,7 @@ flatten_and_generate_uml() { local FILE=$1 local TARGET_DIR=$2 + set +e FLATTENED_FILE="$TARGET_DIR/flattened_$(basename "$FILE")" echo "::debug::Flattening $FILE to $FLATTENED_FILE" forge flatten "$FILE" -o "$FLATTENED_FILE" --root contracts @@ -59,6 +62,7 @@ flatten_and_generate_uml() { fi rm "$FLATTENED_FILE" + set -e } process_all_files_in_directory() { From df771e3c2c408587f159a7ca0afde8af5e510491 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 10:45:51 +0200 Subject: [PATCH 50/54] add set -euo pipefail to bash scripts --- .../scripts/ci/generate_slither_report.sh | 17 ++++++----- contracts/scripts/ci/generate_uml.sh | 30 +++++++++---------- contracts/scripts/ci/modify_remappings.sh | 2 +- contracts/scripts/ci/select_solc_version.sh | 26 +++++++++------- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh index e51dd82828b..b79745bc7e3 100755 --- a/contracts/scripts/ci/generate_slither_report.sh +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -8,7 +8,7 @@ function check_chainlink_dir() { current_base=$(basename "$current_dir") - if [ "$current_base" != "$param_dir" ]; then + if [[ "$current_base" != "$param_dir" ]]; then >&2 echo "The script must be run from the root of $param_dir directory" exit 1 fi @@ -19,7 +19,7 @@ check_chainlink_dir if [ "$#" -lt 5 ]; then >&2 echo "Generates Markdown Slither reports and saves them to a target directory." >&2 echo "Usage: $0 [slither extra params]" -exit 1 + exit 1 fi REPO_URL=$1 @@ -33,16 +33,18 @@ run_slither() { local FILE=$1 local TARGET_DIR=$2 - if [ ! -f "$FILE" ]; then + if [[ ! -f "$FILE" ]]; then >&2 echo "::error:File not found: $FILE" - exit 1 + return 1 fi + set +e source ./contracts/scripts/ci/select_solc_version.sh "$FILE" - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then >&2 echo "::error::Failed to select Solc version for $FILE" - exit 1 + return 1 fi + set -e SLITHER_OUTPUT_FILE="$TARGET_DIR/$(basename "${FILE%.sol}")-slither-report.md" @@ -70,9 +72,10 @@ process_files() { done } +set +e process_files "$SOURCE_DIR" "$TARGET_DIR" "${FILES[@]}" -if [ $? -ne 0 ]; then +if [[ $? -ne 0 ]]; then >&2 echo "::error::Failed to generate Slither reports" exit 1 fi diff --git a/contracts/scripts/ci/generate_uml.sh b/contracts/scripts/ci/generate_uml.sh index 703628b35b0..65745c93bbe 100755 --- a/contracts/scripts/ci/generate_uml.sh +++ b/contracts/scripts/ci/generate_uml.sh @@ -35,8 +35,8 @@ flatten_and_generate_uml() { FLATTENED_FILE="$TARGET_DIR/flattened_$(basename "$FILE")" echo "::debug::Flattening $FILE to $FLATTENED_FILE" forge flatten "$FILE" -o "$FLATTENED_FILE" --root contracts - if [ $? -ne 0 ]; then - >&2 echo "Error: Failed to flatten $FILE" + if [[ $? -ne 0 ]]; then + >&2 echo "::error::Failed to flatten $FILE" FAILED_FILES+=("$FILE") return fi @@ -45,8 +45,8 @@ flatten_and_generate_uml() { OUTPUT_FILE_SVG="${OUTPUT_FILE%.sol}.svg" echo "::debug::Generating SVG UML for $FLATTENED_FILE to $OUTPUT_FILE_SVG" sol2uml "$FLATTENED_FILE" -o "$OUTPUT_FILE_SVG" - if [ $? -ne 0 ]; then - >&2 echo "Error: Failed to generate UML diagram in SVG format for $FILE" + if [[ $? -ne 0 ]]; then + >&2 echo "::error::Failed to generate UML diagram in SVG format for $FILE" FAILED_FILES+=("$FILE") rm "$FLATTENED_FILE" return @@ -54,8 +54,8 @@ flatten_and_generate_uml() { OUTPUT_FILE_DOT="${OUTPUT_FILE%.sol}.dot" echo "::debug::Generating DOT UML for $FLATTENED_FILE to $OUTPUT_FILE_DOT" sol2uml "$FLATTENED_FILE" -o "$OUTPUT_FILE_DOT" -f dot - if [ $? -ne 0 ]; then - >&2 echo "Error: Failed to generate UML diagram in DOT format for $FILE" + if [[ $? -ne 0 ]]; then + >&2 echo "::error::Failed to generate UML diagram in DOT format for $FILE" FAILED_FILES+=("$FILE") rm "$FLATTENED_FILE" return @@ -87,33 +87,31 @@ process_selected_files() { FILE=${FILE//\"/} MATCHES=($(find "$SOURCE_DIR" -type f -path "*/$FILE")) - if [ ${#MATCHES[@]} -gt 1 ]; then + if [[ ${#MATCHES[@]} -gt 1 ]]; then >&2 echo "Error: Multiple matches found for $FILE:" for MATCH in "${MATCHES[@]}"; do >&2 echo " $MATCH" done exit 1 - elif [ ${#MATCHES[@]} -eq 1 ]; then - >&2 echo "File found: ${MATCHES[0]}" + elif [[ ${#MATCHES[@]} -eq 1 ]]; then + >&2 echo "::debug::File found: ${MATCHES[0]}" flatten_and_generate_uml "${MATCHES[0]}" "$TARGET_DIR" else - # needed, because the action we use returns all modified files, also deleted ones and we must skip those - echo "File $FILE does not exist within the source directory $SOURCE_DIR." - echo "Skipping..." - continue + >&2 echo "::error::File $FILE does not exist within the source directory $SOURCE_DIR." + exit 1 fi done } # if FILES is empty, process all files in the directory, otherwise process only the selected files -if [ -z "$FILES" ]; then +if [[ -z "$FILES" ]]; then process_all_files_in_directory "$SOURCE_DIR" "$TARGET_DIR" else process_selected_files "$SOURCE_DIR" "$TARGET_DIR" "$FILES" fi -if [ "${#FAILED_FILES[@]}" -gt 0 ]; then - >&2 echo "Error: Failed to generate UML diagrams for ${#FAILED_FILES[@]} files:" +if [[ "${#FAILED_FILES[@]}" -gt 0 ]]; then + >&2 echo ":error::Failed to generate UML diagrams for ${#FAILED_FILES[@]} files:" for FILE in "${FAILED_FILES[@]}"; do >&2 echo " $FILE" echo "$FILE" >> "$TARGET_DIR/uml_generation_failures.txt" diff --git a/contracts/scripts/ci/modify_remappings.sh b/contracts/scripts/ci/modify_remappings.sh index a6736f966a7..e64ca369b0c 100755 --- a/contracts/scripts/ci/modify_remappings.sh +++ b/contracts/scripts/ci/modify_remappings.sh @@ -11,7 +11,7 @@ DIR_PREFIX=$1 REMAPPINGS_FILE=$2 if [ ! -f "$REMAPPINGS_FILE" ]; then - >&2 echo "Error: Remappings file '$REMAPPINGS_FILE' not found." + >&2 echo "::error:: Remappings file '$REMAPPINGS_FILE' not found." exit 1 fi diff --git a/contracts/scripts/ci/select_solc_version.sh b/contracts/scripts/ci/select_solc_version.sh index 057d2a29979..867f0126c34 100755 --- a/contracts/scripts/ci/select_solc_version.sh +++ b/contracts/scripts/ci/select_solc_version.sh @@ -1,13 +1,15 @@ #!/usr/bin/env bash +set -euo pipefail + function check_chainlink_dir() { local param_dir="chainlink" current_dir=$(pwd) current_base=$(basename "$current_dir") - if [ "$current_base" != "$param_dir" ]; then - echo "The script must be run from the root of $param_dir directory" + if [[ "$current_base" != "$param_dir" ]]; then + >&2 echo "::error::The script must be run from the root of $param_dir directory" exit 1 fi } @@ -16,7 +18,7 @@ check_chainlink_dir FILE="$1" -if [ "$#" -lt 1 ]; then +if [[ "$#" -lt 1 ]]; then echo "Detects the Solidity version of a file and selects the appropriate Solc version." echo "If the version is not installed, it will be installed and selected." echo "Will prefer to use the version from Foundry profile if it satisfies the version in the file." @@ -24,8 +26,8 @@ if [ "$#" -lt 1 ]; then exit 1 fi -if [ -z "$FILE" ]; then - echo "Error: File not provided." +if [[ -z "$FILE" ]]; then + >&2 echo "::error:: File not provided." exit 1 fi @@ -41,11 +43,11 @@ extract_pragma() { if [[ -f "$FILE" ]]; then SOLCVER="$(grep --no-filename '^pragma solidity' "$FILE" | cut -d' ' -f3)" else - echo "$FILE is not a file or it could not be found. Exiting." + >&2 echo ":error::$FILE is not a file or it could not be found. Exiting." return 1 fi SOLCVER="$(echo "$SOLCVER" | sed 's/[^0-9\.^]//g')" - >&2 echo "::debug::Detected Solidity version in pragma: $SOLCVER" + echo "::debug::Detected Solidity version in pragma: $SOLCVER" echo "$SOLCVER" } @@ -60,14 +62,16 @@ SOLC_IN_PROFILE=$(forge config --json --root contracts | jq ".solc") SOLC_IN_PROFILE=$(echo "$SOLC_IN_PROFILE" | tr -d "'\"") echo "::debug::Detected Solidity version in profile: $SOLC_IN_PROFILE" +set +e SOLCVER=$(extract_pragma "$FILE") -exit_code=$? -if [ $exit_code -ne 0 ]; then +if [[ $? -ne 0 ]]; then echo "Error: Failed to extract the Solidity version from $FILE." return 1 fi +set -e + SOLCVER=$(echo "$SOLCVER" | tr -d "'\"") if [[ "$SOLC_IN_PROFILE" != "null" && -n "$SOLCVER" ]]; then @@ -83,13 +87,13 @@ if [[ "$SOLC_IN_PROFILE" != "null" && -n "$SOLCVER" ]]; then SOLC_TO_USE="$SOLCVER" fi elif [[ "$SOLC_IN_PROFILE" != "null" && -z "$SOLCVER" ]]; then - >&2 echo "No version found in the Solidity file. Exiting" + >&2 echo "::error::No version found in the Solidity file. Exiting" return 1 elif [[ "$SOLC_IN_PROFILE" == "null" && -n "$SOLCVER" ]]; then echo "::debug::Using the version from the file: $SOLCVER" SOLC_TO_USE="$SOLCVER" else - >&2 echo "No version found in the profile or the Solidity file." + >&2 echo "::error::No version found in the profile or the Solidity file." return 1 fi From 1fb00d23c0607bc7224c3f5517df785ada296829 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 16:58:43 +0200 Subject: [PATCH 51/54] define action to detect foundry version --- .../action.yml | 26 +++++++++++++++++++ .github/workflows/solidity-foundry.yml | 16 +++++++----- 2 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 .github/actions/detect-solidity-foundry-version/action.yml diff --git a/.github/actions/detect-solidity-foundry-version/action.yml b/.github/actions/detect-solidity-foundry-version/action.yml new file mode 100644 index 00000000000..b37f1e25094 --- /dev/null +++ b/.github/actions/detect-solidity-foundry-version/action.yml @@ -0,0 +1,26 @@ +name: 'Detect Foundry version in GNUmakefile' +description: 'Detects Foundry version in GNUmakefile' +inputs: + working-directory: + description: 'The GNUmakefile directory' + required: false + default: 'contracts' +outputs: + foundry-version: + description: 'Foundry version found in GNUmakefile' + value: ${{ steps.extract-foundry-version.outputs.foundry-version }} +runs: + using: 'composite' + steps: + - name: Extract Foundry version + id: extract-foundry-version + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + foundry_version=$(grep -Eo "foundryup --version [^ ]+" GNUmakefile | awk '{print $3}') + if [ -z "$foundry_version" ]; then + echo "::error::Foundry version not found in GNUmakefile" + exit 1 + fi + echo "Foundry version found: $foundry_version" + echo "foundry-version=$foundry_version" >> $GITHUB_OUTPUT diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 08039456bc0..6d24db74cb7 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -3,8 +3,6 @@ on: [pull_request] env: FOUNDRY_PROFILE: ci - # Has to match the `make foundry` version in `contracts/GNUmakefile` - FOUNDRY_VERSION: nightly-de33b6af53005037b463318d2628b5cfcaf39916 # Making changes: # * use the top-level matrix to decide, which checks should run for each product. @@ -16,6 +14,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.define-matrix.outputs.matrix }} + foundry-version: ${{ steps.extract-foundry-version.outputs.foundry-version }} steps: - name: Define test matrix id: define-matrix @@ -40,6 +39,11 @@ jobs: matrix=$(cat matrix.json | jq -c .) echo "matrix=$matrix" >> $GITHUB_OUTPUT + - name: Extract Foundry version + uses: ./.github/actions/detect-solidity-foundry-version + with: + working-directory: contracts + changes: name: Detect changes runs-on: ubuntu-latest @@ -130,7 +134,7 @@ jobs: || needs.changes.outputs.non_src_changes == 'true' }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ needs.define-matrix.outputs.foundry-version }} - name: Run Forge build if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) @@ -222,7 +226,7 @@ jobs: continue-on-error: true analyze: - needs: [ changes ] + needs: [ changes, define-matrix ] name: Run static analysis if: needs.changes.outputs.not_test_sol_modified == 'true' runs-on: ubuntu-22.04 @@ -238,7 +242,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ needs.define-matrix.outputs.foundry-version }} - name: Set up Python uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f #v5.1.1 @@ -336,7 +340,7 @@ jobs: if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0 with: - version: ${{ env.FOUNDRY_VERSION }} + version: ${{ needs.define-matrix.outputs.foundry-version }} - name: Run Forge fmt if: ${{ contains(fromJson(needs.changes.outputs.all_changes), matrix.product.name) && matrix.product.setup.run-forge-fmt }} From a18c5ad79a72156f2eac6263546237d585006248 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 17:02:26 +0200 Subject: [PATCH 52/54] fix select solc version script, better slither report output --- contracts/scripts/ci/generate_slither_report.sh | 4 ++++ contracts/scripts/ci/select_solc_version.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/contracts/scripts/ci/generate_slither_report.sh b/contracts/scripts/ci/generate_slither_report.sh index b79745bc7e3..7fe31d40efc 100755 --- a/contracts/scripts/ci/generate_slither_report.sh +++ b/contracts/scripts/ci/generate_slither_report.sh @@ -57,6 +57,10 @@ run_slither() { echo "# Summary for $FILE" > "$SLITHER_OUTPUT_FILE" echo "$output" >> "$SLITHER_OUTPUT_FILE" + + if [[ -z "$output" ]]; then + echo "No issues found." >> "$SLITHER_OUTPUT_FILE" + fi } process_files() { diff --git a/contracts/scripts/ci/select_solc_version.sh b/contracts/scripts/ci/select_solc_version.sh index 867f0126c34..3f7d7864ab7 100755 --- a/contracts/scripts/ci/select_solc_version.sh +++ b/contracts/scripts/ci/select_solc_version.sh @@ -47,7 +47,7 @@ extract_pragma() { return 1 fi SOLCVER="$(echo "$SOLCVER" | sed 's/[^0-9\.^]//g')" - echo "::debug::Detected Solidity version in pragma: $SOLCVER" + >&2 echo "::debug::Detected Solidity version in pragma: $SOLCVER" echo "$SOLCVER" } From 32addb7fdc8bd86f4d3c303bfe9c3284ef7c7bba Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 17:04:54 +0200 Subject: [PATCH 53/54] checkout repo --- .github/workflows/solidity-foundry.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 6d24db74cb7..8c410ca0ff7 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -39,6 +39,9 @@ jobs: matrix=$(cat matrix.json | jq -c .) echo "matrix=$matrix" >> $GITHUB_OUTPUT + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Extract Foundry version uses: ./.github/actions/detect-solidity-foundry-version with: From 8c9d3c4f1a0f31cb97074e0ed86b3efc871e28f2 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 8 Aug 2024 17:20:18 +0200 Subject: [PATCH 54/54] add id --- .github/workflows/solidity-foundry.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml index 8c410ca0ff7..906cb76ffe5 100644 --- a/.github/workflows/solidity-foundry.yml +++ b/.github/workflows/solidity-foundry.yml @@ -43,6 +43,7 @@ jobs: uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Extract Foundry version + id: extract-foundry-version uses: ./.github/actions/detect-solidity-foundry-version with: working-directory: contracts