diff --git a/.github/workflows/nightly_wan.yml b/.github/workflows/nightly_wan.yml index 78c1ff756f..0ee6bc1ad3 100644 --- a/.github/workflows/nightly_wan.yml +++ b/.github/workflows/nightly_wan.yml @@ -1,13 +1,16 @@ name: Nightly -- Full WAN Network Tests on: -# todo: this is totally broken atm. Fix and re-enable. -# schedule: -# - cron: "0 0 * * *" + schedule: + - cron: "0 0 * * *" +# enable as below for testing purpose. + # pull_request: + # branches: ["*"] workflow_dispatch: env: CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI. + NETWORK_NAME: DEV-01 WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs jobs: e2e: @@ -29,93 +32,95 @@ jobs: run: cargo build --release --bin safe timeout-minutes: 30 - - name: Start a WAN network - uses: maidsafe/sn-testnet-action@main + - name: setup testnet-deploy + uses: maidsafe/sn-testnet-control-action/init-testnet-deploy@main with: - action: create - re-attempts: 3 - rust-log: debug ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-2 do-token: ${{ secrets.SN_TESTNET_DO_PAT }} ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - security-group-id: sg-0d47df5b3f0d01e2a - subnet-id: subnet-018f2ab26755df7f9 + + - name: launch ${{ env.NETWORK_NAME }} + uses: maidsafe/sn-testnet-control-action/launch-network@main + with: + ansible-forks: ${{ env.ANSIBLE_FORKS }} + environment-type: development + node-vm-count: 10 node-count: 20 - vm-count: 1 + uploader-vm-count: 0 + bootstrap-node-vm-count: 0 + log-format: json + network-name: ${{ env.NETWORK_NAME }} provider: digital-ocean - testnet-name: NightlyE2E - # if we were to run on a PR, use the below - # safe-network-user: ${{ github.actor }}" - # safe-network-branch: ${{ github.event.pull_request.head.ref }} - # Specify custom branch to prevent the deployer from fetching the latest release. - # The latest release contains the `network-contacts` feature turned on. - safe-network-user: maidsafe safe-network-branch: main + safe-network-user: maidsafe - name: Check env variables shell: bash run: | - echo "Peer is $SAFE_PEERS" - echo "Deployment inventory is $SN_INVENTORY" + echo "Peer is $SAFE_PEERS" + echo "Deployment inventory is $SN_INVENTORY" + - name: start faucet + uses: maidsafe/sn-testnet-control-action/start-faucet@main + with: + network-name: ${{ env.NETWORK_NAME }} + - name: Obtain the funds from the faucet run: | + set -e + # read the inventory file - inventory_path=/home/runner/.local/share/safe/testnet-deploy/NightlyE2E-inventory.json + inventory_path=/home/runner/.local/share/safe/testnet-deploy/${{ env.NETWORK_NAME }}-inventory.json echo "Inventory Path: $inventory_path" faucet_address=$(jq -r '.faucet_address' $inventory_path) cargo run --bin safe --release -- wallet get-faucet ${faucet_address} env: SN_LOG: "all" timeout-minutes: 2 - + - name: Start a client to carry out chunk actions - run: cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" --retry-strategy quick + run: | + set -e + cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" --retry-strategy quick env: SN_LOG: "all" timeout-minutes: 2 - + - name: Start a client to create a register - run: cargo run --bin safe --release -- --log-output-dest=data-dir register create -n baobao + run: | + set -e + cargo run --bin safe --release -- --log-output-dest=data-dir register create -n baobao env: SN_LOG: "all" timeout-minutes: 2 - + - name: Start a client to get a register - run: cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao + run: | + set -e + cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao env: SN_LOG: "all" timeout-minutes: 2 - + - name: Start a client to edit a register - run: cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood + run: | + set -e + cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood env: SN_LOG: "all" timeout-minutes: 2 - - - name: Fetch network logs - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: logs - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyE2E - safe-network-user: maidsafe - safe-network-branch: main - + + # - name: Fetch network logs + # uses: maidsafe/sn-testnet-control-action/fetch-logs@main + # with: + # re-attempts: 3 + # rust-log: debug + # provider: digital-ocean + # network-name: ${{ env.NETWORK_NAME }} + - name: Upload local logs if: always() uses: actions/upload-artifact@v4 @@ -126,554 +131,518 @@ jobs: ~/.local/share/safe/*/*/*.log* ~/.local/share/safe/client/logs/*/*.log* - - name: Stop the WAN network - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: destroy - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyE2E - safe-network-user: maidsafe - safe-network-branch: main - - # - name: post notification to slack on failure - # if: ${{ failure() }} - # uses: bryannice/gitactions-slack-notification@2.0.0 - # env: - # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - # SLACK_TITLE: "Nightly E2E Test Run Failed" - - spend_test: - name: Spend tests against network - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 - continue-on-error: true - - - name: Build testing executable - run: cargo test --release -p sn_node --features=local-discovery --test sequential_transfers --test storage_payments --test double_spend --test spend_simulation --no-run - timeout-minutes: 40 - - - name: Start a WAN network - uses: maidsafe/sn-testnet-action@main - with: - action: create - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - security-group-id: sg-0d47df5b3f0d01e2a - subnet-id: subnet-018f2ab26755df7f9 - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlySpendTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Check env variables - shell: bash - run: | - echo "Peer is $SAFE_PEERS" - echo "Deployment inventory is $SN_INVENTORY" - - - name: execute the sequential transfers test - run: cargo test --release -p sn_node --test sequential_transfers -- --nocapture --test-threads=1 - env: - SN_LOG: "all" - timeout-minutes: 45 - - - name: execute the storage payment tests - run: cargo test --release -p sn_node --test storage_payments -- --nocapture --test-threads=1 - env: - SN_LOG: "all" - timeout-minutes: 45 - - - name: execute the double spend tests - run: cargo test --release -p sn_node --test double_spend -- --nocapture --test-threads=1 - timeout-minutes: 45 - - - - name: execute the spend simulation tests - run: cargo test --release -p sn_node --test spend_simulation -- --nocapture --test-threads=1 - timeout-minutes: 45 - - - name: Small wait to allow reward receipt - run: sleep 30 - timeout-minutes: 1 - - - name: Fetch network logs - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: logs - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlySpendTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Upload local logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: local_logs_NightlySpendTest - path: | - ~/.local/share/safe/node/*/logs/*.log* - ~/.local/share/safe/*/*/*.log* - ~/.local/share/safe/client/logs/*/*.log* - - - name: Stop the WAN network - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: destroy - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlySpendTest - safe-network-user: maidsafe - safe-network-branch: main - - # - name: post notification to slack on failure - # if: ${{ failure() }} - # uses: bryannice/gitactions-slack-notification@2.0.0 - # env: - # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - # SLACK_TITLE: "Nightly Spend Test Run Failed" - - churn: - name: Network churning tests - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-latest - wan_logs_path: /home/runner/sn-testnet-deploy/logs - local_safe_path: /home/runner/.local/share/safe - # - os: windows-latest - # node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node - # safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe - # - os: macos-latest - # node_data_path: /Users/runner/Library/Application Support/safe/node - # safe_path: /Users/runner/Library/Application Support/safe - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: install ripgrep - run: sudo apt-get -y install ripgrep - - - uses: Swatinem/rust-cache@v2 - continue-on-error: true - - - name: Build churn tests - run: cargo test --release -p sn_node --test data_with_churn --no-run - timeout-minutes: 30 - - - name: Start a WAN network - uses: maidsafe/sn-testnet-action@main - with: - action: create - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - security-group-id: sg-0d47df5b3f0d01e2a - subnet-id: subnet-018f2ab26755df7f9 - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyChurnTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Check env variables - shell: bash - run: | - echo "Peer is $SAFE_PEERS" - echo "Deployment inventory is $SN_INVENTORY" - - - name: Chunks data integrity during nodes churn - run: cargo test --release -p sn_node --test data_with_churn -- --nocapture - env: - # TEST_DURATION_MINS: 60 - # TEST_CHURN_CYCLES: 6 - # SN_LOG: "all" - # todo: lower time for testing - TEST_DURATION_MINS: 10 - TEST_CHURN_CYCLES: 2 - SN_LOG: "all" - timeout-minutes: 90 - - - name: Fetch network logs - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: logs - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyChurnTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Upload local logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: local_logs_NightlyChurnTest - path: | - ~/.local/share/safe/node/*/logs/*.log* - ~/.local/share/safe/*/*/*.log* - ~/.local/share/safe/client/logs/*/*.log* - - - name: Stop the WAN network + - name: destroy network if: always() - uses: maidsafe/sn-testnet-action@main + uses: maidsafe/sn-testnet-control-action/destroy-network@main with: - action: destroy - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 + network-name: ${{ env.NETWORK_NAME }} provider: digital-ocean - testnet-name: NightlyChurnTest - safe-network-user: maidsafe - safe-network-branch: main - - # TODO: re-enable the below scripts once we have proper way to restart nodes. - # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same - # log location and the logs are over written. Hence the scripts might give false outputs. - - # - name: Verify restart of nodes using rg - # shell: bash - # timeout-minutes: 1 - # # get the counts, then the specific line, and then the digit count only - # # then check we have an expected level of restarts - # # TODO: make this use an env var, or relate to testnet size - # run : | - # restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \ - # rg "(\d+) matches" | rg "\d+" -o) - # echo "Restart $restart_count nodes" - # peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \ - # rg "(\d+) matches" | rg "\d+" -o) - # echo "PeerRemovedFromRoutingTable $peer_removed times" - # if [ $peer_removed -lt $restart_count ]; then - # echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" - # exit 1 - # fi - # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) - # echo "Node dir count is $node_count" - # # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here - - # # if [ $restart_count -lt $node_count ]; then - # # echo "Restart count of: $restart_count is less than the node count of: $node_count" - # # exit 1 - # # fi - - # - name: Verify data replication using rg - # shell: bash - # timeout-minutes: 1 - # # get the counts, then the specific line, and then the digit count only - # # then check we have an expected level of replication - # # TODO: make this use an env var, or relate to testnet size - # run : | - # fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.wan_logs_path }}" -c --stats | \ - # rg "(\d+) matches" | rg "\d+" -o) - # echo "Carried out $fetching_attempt_count fetching attempts" - # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) - # if [ $fetching_attempt_count -lt $node_count ]; then - # echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count" - # exit 1 - # fi - - # Only error out after uploading the logs - - name: Don't log raw data - if: always() && matrix.os != 'windows-latest' # causes error - shell: bash - timeout-minutes: 10 - run: | - if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }' - then - echo "We are logging an extremely large data" - exit 1 - fi - # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log - #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log - if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }' - then - echo "We are logging an extremely large data" - exit 1 - fi - - # sanity check - if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }' - then - echo "Sanity check pass for local safe path" - fi - # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log - #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log - if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }' - then - echo "Sanity check pass for wan logs path" - fi - - # - name: post notification to slack on failure - # if: ${{ failure() }} - # uses: bryannice/gitactions-slack-notification@2.0.0 - # env: - # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - # SLACK_TITLE: "Nightly Churn Test Run Failed" - - verify_data_location_routing_table: - name: Verify data location and Routing Table - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-latest - wan_logs_path: /home/runner/sn-testnet-deploy/logs - local_safe_path: /home/runner/.local/share/safe - # - os: windows-latest - # node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node - # safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe - # - os: macos-latest - # node_data_path: /Users/runner/Library/Application Support/safe/node - # safe_path: /Users/runner/Library/Application Support/safe - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - name: install ripgrep - run: sudo apt-get -y install ripgrep - - - uses: Swatinem/rust-cache@v2 - continue-on-error: true - - - name: Build data location and routing table tests - run: cargo test --release -p sn_node --test verify_data_location --test verify_routing_table --no-run - timeout-minutes: 30 - - - name: Start a WAN network - uses: maidsafe/sn-testnet-action@main - with: - action: create - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - security-group-id: sg-0d47df5b3f0d01e2a - subnet-id: subnet-018f2ab26755df7f9 - node-count: 20 - vm-count: 1 - testnet-name: NightlyDataLocationTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Check env variables - shell: bash - run: | - echo "Peer is $SAFE_PEERS" - echo "Deployment inventory is $SN_INVENTORY" - - - name: Verify the Routing table of the nodes - run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture - timeout-minutes: 5 - - - name: Verify the location of the data on the network - run: cargo test --release -p sn_node --test verify_data_location -- --nocapture + - name: post notification to slack on failure + if: ${{ failure() }} + uses: bryannice/gitactions-slack-notification@2.0.0 env: - SN_LOG: "all" - timeout-minutes: 90 - - - name: Verify the routing tables of the nodes - run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture - timeout-minutes: 5 - - - name: Fetch network logs - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: logs - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyDataLocationTest - safe-network-user: maidsafe - safe-network-branch: main - - - name: Upload local logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: local_logs_NightlyDataLocationTest - path: | - ~/.local/share/safe/node/*/logs/*.log* - ~/.local/share/safe/*/*/*.log* - ~/.local/share/safe/client/logs/*/*.log* - - - name: Stop the WAN network - if: always() - uses: maidsafe/sn-testnet-action@main - with: - action: destroy - re-attempts: 3 - rust-log: debug - ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - node-count: 20 - vm-count: 1 - provider: digital-ocean - testnet-name: NightlyDataLocationTest - safe-network-user: maidsafe - safe-network-branch: main - - # TODO: re-enable the below scripts once we have proper way to restart nodes. - # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same - # log location and the logs are over written. Hence the scripts might give false outputs. - - # - name: Verify restart of nodes using rg - # shell: bash - # timeout-minutes: 1 - # # get the counts, then the specific line, and then the digit count only - # # then check we have an expected level of restarts - # # TODO: make this use an env var, or relate to testnet size - # run : | - # restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \ - # rg "(\d+) matches" | rg "\d+" -o) - # echo "Restart $restart_count nodes" - # peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \ - # rg "(\d+) matches" | rg "\d+" -o) - # echo "PeerRemovedFromRoutingTable $peer_removed times" - # if [ $peer_removed -lt $restart_count ]; then - # echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" - # exit 1 - # fi - # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) - # echo "Node dir count is $node_count" - # # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here - - # # if [ $restart_count -lt $node_count ]; then - # # echo "Restart count of: $restart_count is less than the node count of: $node_count" - # # exit 1 - # # fi - - # Only error out after uploading the logs - - name: Don't log raw data - if: always() && matrix.os != 'windows-latest' # causes error - shell: bash - timeout-minutes: 10 - run: | - if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }' - then - echo "We are logging an extremely large data" - exit 1 - fi - # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log - #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log - if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }' - then - echo "We are logging an extremely large data" - exit 1 - fi - - # sanity check - if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }' - then - echo "Sanity check pass for local safe path" - fi - # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log - #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log - if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }' - then - echo echo "Sanity check pass for wan logs path" - fi - - # - name: post notification to slack on failure - # if: ${{ failure() }} - # uses: bryannice/gitactions-slack-notification@2.0.0 - # env: - # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - # SLACK_TITLE: "Nightly Data Location Test Run Failed" + SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + SLACK_TITLE: "Nightly E2E Test Run Failed" + + # spend_test: + # name: Spend tests against network + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: [ubuntu-latest] + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + + # - uses: Swatinem/rust-cache@v2 + # continue-on-error: true + + # - name: Build testing executable + # run: cargo test --release -p sn_node --features=local-discovery --test sequential_transfers --test storage_payments --test double_spend --test spend_simulation --no-run + # timeout-minutes: 40 + + # - name: setup testnet-deploy + # uses: maidsafe/sn-testnet-control-action/init-testnet-deploy@main + # with: + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + + # - name: launch ${{ env.NETWORK_NAME }} + # uses: maidsafe/sn-testnet-control-action/launch-network@main + # with: + # ansible-forks: ${{ env.ANSIBLE_FORKS }} + # beta-encryption-key: ${{ env.DEFAULT_PAYMENT_FORWARD_SK }} + # environment-type: development + # faucet-version: ${{ env.FAUCET_VERSION }} + # log-format: json + # network-name: ${{ env.NETWORK_NAME }} + # network-contacts-file-name: ${{ env.NETWORK_CONTACTS_FILE_NAME }} + # provider: digital-ocean + # safe-network-branch: main + # safe-network-user: maidsafe + + # - name: Check env variables + # shell: bash + # run: | + # echo "Peer is $SAFE_PEERS" + # echo "Deployment inventory is $SN_INVENTORY" + + # - name: execute the sequential transfers test + # run: cargo test --release -p sn_node --test sequential_transfers -- --nocapture --test-threads=1 + # env: + # SN_LOG: "all" + # timeout-minutes: 45 + + # - name: execute the storage payment tests + # run: cargo test --release -p sn_node --test storage_payments -- --nocapture --test-threads=1 + # env: + # SN_LOG: "all" + # timeout-minutes: 45 + + # - name: execute the double spend tests + # run: cargo test --release -p sn_node --test double_spend -- --nocapture --test-threads=1 + # timeout-minutes: 45 + + + # - name: execute the spend simulation tests + # run: cargo test --release -p sn_node --test spend_simulation -- --nocapture --test-threads=1 + # timeout-minutes: 45 + + # - name: Small wait to allow reward receipt + # run: sleep 30 + # timeout-minutes: 1 + + # - name: Fetch network logs + # uses: ermineJose/sn-testnet-control-action/fetch-logs@feat-add_fetch-logs-action + # with: + # re-attempts: 3 + # rust-log: debug + # provider: digital-ocean + # testnet-name: ${{ env.NETWORK_NAME }} + + # - name: Upload local logs + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: local_logs_NightlySpendTest + # path: | + # ~/.local/share/safe/node/*/logs/*.log* + # ~/.local/share/safe/*/*/*.log* + # ~/.local/share/safe/client/logs/*/*.log* + + # - name: destroy network + # uses: maidsafe/sn-testnet-control-action/destroy-network@main + # with: + # network-name: ${{ env.NETWORK_NAME }} + # provider: digital-ocean + + # - name: post notification to slack on failure + # if: ${{ failure() }} + # uses: bryannice/gitactions-slack-notification@2.0.0 + # env: + # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + # SLACK_TITLE: "Nightly Spend Test Run Failed" + + # churn: + # name: Network churning tests + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # include: + # - os: ubuntu-latest + # wan_logs_path: /home/runner/sn-testnet-deploy/logs + # local_safe_path: /home/runner/.local/share/safe + # # - os: windows-latest + # # node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node + # # safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe + # # - os: macos-latest + # # node_data_path: /Users/runner/Library/Application Support/safe/node + # # safe_path: /Users/runner/Library/Application Support/safe + # steps: + # - uses: actions/checkout@v4 + # + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + # + # - name: install ripgrep + # run: sudo apt-get -y install ripgrep + # + # - uses: Swatinem/rust-cache@v2 + # continue-on-error: true + # + # - name: Build churn tests + # run: cargo test --release -p sn_node --test data_with_churn --no-run + # timeout-minutes: 30 + # + # - name: Start a WAN network + # uses: maidsafe/sn-testnet-action@main + # with: + # action: create + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # security-group-id: sg-0d47df5b3f0d01e2a + # subnet-id: subnet-018f2ab26755df7f9 + # node-count: 20 + # vm-count: 1 + # provider: digital-ocean + # testnet-name: NightlyChurnTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # - name: Check env variables + # shell: bash + # run: | + # echo "Peer is $SAFE_PEERS" + # echo "Deployment inventory is $SN_INVENTORY" + # + # - name: Chunks data integrity during nodes churn + # run: cargo test --release -p sn_node --test data_with_churn -- --nocapture + # env: + # # TEST_DURATION_MINS: 60 + # # TEST_CHURN_CYCLES: 6 + # # SN_LOG: "all" + # # todo: lower time for testing + # TEST_DURATION_MINS: 10 + # TEST_CHURN_CYCLES: 2 + # SN_LOG: "all" + # timeout-minutes: 90 + # + # - name: Fetch network logs + # if: always() + # uses: maidsafe/sn-testnet-action@main + # with: + # action: logs + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # node-count: 20 + # vm-count: 1 + # provider: digital-ocean + # testnet-name: NightlyChurnTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # - name: Upload local logs + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: local_logs_NightlyChurnTest + # path: | + # ~/.local/share/safe/node/*/logs/*.log* + # ~/.local/share/safe/*/*/*.log* + # ~/.local/share/safe/client/logs/*/*.log* + # + # - name: Stop the WAN network + # if: always() + # uses: maidsafe/sn-testnet-action@main + # with: + # action: destroy + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # node-count: 20 + # vm-count: 1 + # provider: digital-ocean + # testnet-name: NightlyChurnTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # # TODO: re-enable the below scripts once we have proper way to restart nodes. + # # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same + # # log location and the logs are over written. Hence the scripts might give false outputs. + # + # # - name: Verify restart of nodes using rg + # # shell: bash + # # timeout-minutes: 1 + # # # get the counts, then the specific line, and then the digit count only + # # # then check we have an expected level of restarts + # # # TODO: make this use an env var, or relate to testnet size + # # run : | + # # restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \ + # # rg "(\d+) matches" | rg "\d+" -o) + # # echo "Restart $restart_count nodes" + # # peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \ + # # rg "(\d+) matches" | rg "\d+" -o) + # # echo "PeerRemovedFromRoutingTable $peer_removed times" + # # if [ $peer_removed -lt $restart_count ]; then + # # echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" + # # exit 1 + # # fi + # # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) + # # echo "Node dir count is $node_count" + # # # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here + # + # # # if [ $restart_count -lt $node_count ]; then + # # # echo "Restart count of: $restart_count is less than the node count of: $node_count" + # # # exit 1 + # # # fi + # + # # - name: Verify data replication using rg + # # shell: bash + # # timeout-minutes: 1 + # # # get the counts, then the specific line, and then the digit count only + # # # then check we have an expected level of replication + # # # TODO: make this use an env var, or relate to testnet size + # # run : | + # # fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.wan_logs_path }}" -c --stats | \ + # # rg "(\d+) matches" | rg "\d+" -o) + # # echo "Carried out $fetching_attempt_count fetching attempts" + # # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) + # # if [ $fetching_attempt_count -lt $node_count ]; then + # # echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count" + # # exit 1 + # # fi + # + # # Only error out after uploading the logs + # - name: Don't log raw data + # if: always() && matrix.os != 'windows-latest' # causes error + # shell: bash + # timeout-minutes: 10 + # run: | + # if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }' + # then + # echo "We are logging an extremely large data" + # exit 1 + # fi + # # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log + # #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log + # if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }' + # then + # echo "We are logging an extremely large data" + # exit 1 + # fi + # + # # sanity check + # if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }' + # then + # echo "Sanity check pass for local safe path" + # fi + # # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log + # #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log + # if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }' + # then + # echo "Sanity check pass for wan logs path" + # fi + # + # # - name: post notification to slack on failure + # # if: ${{ failure() }} + # # uses: bryannice/gitactions-slack-notification@2.0.0 + # # env: + # # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + # # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + # # SLACK_TITLE: "Nightly Churn Test Run Failed" + # + # verify_data_location_routing_table: + # name: Verify data location and Routing Table + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # include: + # - os: ubuntu-latest + # wan_logs_path: /home/runner/sn-testnet-deploy/logs + # local_safe_path: /home/runner/.local/share/safe + # # - os: windows-latest + # # node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node + # # safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe + # # - os: macos-latest + # # node_data_path: /Users/runner/Library/Application Support/safe/node + # # safe_path: /Users/runner/Library/Application Support/safe + # steps: + # - uses: actions/checkout@v4 + # + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + # + # - name: install ripgrep + # run: sudo apt-get -y install ripgrep + # + # - uses: Swatinem/rust-cache@v2 + # continue-on-error: true + # + # - name: Build data location and routing table tests + # run: cargo test --release -p sn_node --test verify_data_location --test verify_routing_table --no-run + # timeout-minutes: 30 + # + # - name: Start a WAN network + # uses: maidsafe/sn-testnet-action@main + # with: + # action: create + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # security-group-id: sg-0d47df5b3f0d01e2a + # subnet-id: subnet-018f2ab26755df7f9 + # node-count: 20 + # vm-count: 1 + # testnet-name: NightlyDataLocationTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # - name: Check env variables + # shell: bash + # run: | + # echo "Peer is $SAFE_PEERS" + # echo "Deployment inventory is $SN_INVENTORY" + # + # - name: Verify the Routing table of the nodes + # run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture + # timeout-minutes: 5 + # + # - name: Verify the location of the data on the network + # run: cargo test --release -p sn_node --test verify_data_location -- --nocapture + # env: + # SN_LOG: "all" + # timeout-minutes: 90 + # + # - name: Verify the routing tables of the nodes + # run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture + # timeout-minutes: 5 + # + # - name: Fetch network logs + # if: always() + # uses: maidsafe/sn-testnet-action@main + # with: + # action: logs + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # node-count: 20 + # vm-count: 1 + # provider: digital-ocean + # testnet-name: NightlyDataLocationTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # - name: Upload local logs + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: local_logs_NightlyDataLocationTest + # path: | + # ~/.local/share/safe/node/*/logs/*.log* + # ~/.local/share/safe/*/*/*.log* + # ~/.local/share/safe/client/logs/*/*.log* + # + # - name: Stop the WAN network + # if: always() + # uses: maidsafe/sn-testnet-action@main + # with: + # action: destroy + # re-attempts: 3 + # rust-log: debug + # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} + # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} + # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} + # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} + # node-count: 20 + # vm-count: 1 + # provider: digital-ocean + # testnet-name: NightlyDataLocationTest + # safe-network-user: maidsafe + # safe-network-branch: main + # + # # TODO: re-enable the below scripts once we have proper way to restart nodes. + # # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same + # # log location and the logs are over written. Hence the scripts might give false outputs. + # + # # - name: Verify restart of nodes using rg + # # shell: bash + # # timeout-minutes: 1 + # # # get the counts, then the specific line, and then the digit count only + # # # then check we have an expected level of restarts + # # # TODO: make this use an env var, or relate to testnet size + # # run : | + # # restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \ + # # rg "(\d+) matches" | rg "\d+" -o) + # # echo "Restart $restart_count nodes" + # # peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \ + # # rg "(\d+) matches" | rg "\d+" -o) + # # echo "PeerRemovedFromRoutingTable $peer_removed times" + # # if [ $peer_removed -lt $restart_count ]; then + # # echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" + # # exit 1 + # # fi + # # node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l) + # # echo "Node dir count is $node_count" + # # # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here + # + # # # if [ $restart_count -lt $node_count ]; then + # # # echo "Restart count of: $restart_count is less than the node count of: $node_count" + # # # exit 1 + # # # fi + # + # # Only error out after uploading the logs + # - name: Don't log raw data + # if: always() && matrix.os != 'windows-latest' # causes error + # shell: bash + # timeout-minutes: 10 + # run: | + # if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }' + # then + # echo "We are logging an extremely large data" + # exit 1 + # fi + # # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log + # #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log + # if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }' + # then + # echo "We are logging an extremely large data" + # exit 1 + # fi + # + # # sanity check + # if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }' + # then + # echo "Sanity check pass for local safe path" + # fi + # # node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log + # #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log + # if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }' + # then + # echo echo "Sanity check pass for wan logs path" + # fi + # + # # - name: post notification to slack on failure + # # if: ${{ failure() }} + # # uses: bryannice/gitactions-slack-notification@2.0.0 + # # env: + # # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + # # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + # # SLACK_TITLE: "Nightly Data Location Test Run Failed"