Data further refactor #10912
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Check before merge | |
on: | |
# tests must run for a PR to be valid and pass merge queue muster | |
# on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors | |
# the merge run checks should show on master and enable this clear test/passing history | |
merge_group: | |
branches: [main, alpha*, beta*, rc*] | |
pull_request: | |
branches: ["*"] | |
env: | |
CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI. | |
WINSW_URL: https://github.com/winsw/winsw/releases/download/v3.0.0-alpha.11/WinSW-x64.exe | |
GENESIS_PK: 9377ab39708a59d02d09bfd3c9bc7548faab9e0c2a2700b9ac7d5c14f0842f0b4bb0df411b6abd3f1a92b9aa1ebf5c3d | |
GENESIS_SK: 5ec88891c1098a0fede5b98b07f8abc931d7247b7aa310d21ab430cc957f9f02 | |
jobs: | |
build: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.x' | |
- name: Display Python version | |
run: python --version | |
cargo-udeps: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: Unused dependency check | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@nightly # Needs nightly to distinguish between deps of different versions | |
with: | |
# we need rustfmt here while we have a build step | |
components: rustfmt | |
- name: Install cargo-udeps | |
run: cargo install cargo-udeps --locked | |
- name: Run cargo-udeps | |
run: cargo +nightly udeps --all-targets | |
# ignore the error cause by the latest nightly changes. | |
# should be fixed by https://github.com/dalek-cryptography/curve25519-dalek/pull/619 | |
continue-on-error: true | |
lint: | |
runs-on: ubuntu-latest | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- uses: wagoid/commitlint-github-action@0184f5a228ee06430bb9e67d65f73a1a6767496a | |
checks: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: various checks | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@stable | |
with: | |
components: rustfmt, clippy | |
- uses: Swatinem/rust-cache@v2 | |
- name: Check formatting | |
run: cargo fmt --all -- --check | |
- shell: bash | |
run: cargo clippy --all-targets --all-features -- -Dwarnings | |
- name: Check documentation | |
# Deny certain `rustdoc` lints that are unwanted with `RUSTDOCFLAGS`. See | |
# https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default. | |
# | |
# We exclude autonomi-cli because it is not published and conflicts with the `autonomi` crate name, | |
# resulting in an error when building docs. | |
run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps --workspace --exclude=autonomi-cli | |
- name: Check local is not a default feature | |
shell: bash | |
run: if [[ ! $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].features.default[]? | select(. == "local")') ]]; then echo "local is not a default feature in any package."; else echo "local is a default feature in at least one package." && exit 1; fi | |
- name: Clean out the target directory | |
run: cargo clean | |
# In a cargo workspace, feature unification can occur, allowing a crate to be built successfully even if it | |
# doesn't explicitly specify a feature it uses, provided another crate in the workspace enables that feature. | |
# To detect such cases, we must build each crate using `--package` flag, building all packages at once does not work. | |
- name: Check the whole workspace can build | |
shell: bash | |
run: | | |
for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name'); do | |
cargo build -p "$package" --all-targets --all-features | |
done | |
echo "All packages built successfully. Cleaning up..." | |
cargo clean | |
unit: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: Unit Tests | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
os: [ubuntu-latest, windows-latest, macos-latest] | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Check we're on the right commit | |
run: git log -1 --oneline | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@stable | |
- uses: Swatinem/rust-cache@v2 | |
- name: Run autonomi tests | |
timeout-minutes: 25 | |
run: cargo test --release --package autonomi --features full,local --lib | |
- name: Run autonomi doc tests | |
timeout-minutes: 25 | |
run: cargo test --release --package autonomi --features full,local --doc | |
- name: Run bootstrap tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-bootstrap | |
- name: Run node tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-node --lib | |
- name: Run launchpad tests | |
timeout-minutes: 25 | |
run: cargo test --release --package node-launchpad | |
# The `can_store_after_restart` can be executed with other package tests together and passing | |
# on local machine. However keeps failing (when executed together) on CI machines. | |
# This is most likely due to the setup and cocurrency issues of the tests. | |
# As the `record_store` is used in a single thread style, get the test passing executed | |
# and passing standalone is enough. | |
- name: Run network tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-networking --features="open-metrics" -- --skip can_store_after_restart | |
- name: Run network tests (can_store_after_restart) | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-networking --features="open-metrics" can_store_after_restart | |
- name: Run protocol tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-protocol | |
- name: Run logging tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-logging | |
- name: Run register tests | |
timeout-minutes: 25 | |
run: cargo test --release --package ant-registers | |
env: | |
# this will speed up PR merge flows, while giving us a modicum | |
# of proptesting | |
# we do many more runs on the nightly run | |
PROPTEST_CASES: 50 | |
e2e: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: E2E tests | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
include: | |
- os: ubuntu-latest | |
ant_path: /home/runner/.local/share/autonomi | |
- os: windows-latest | |
ant_path: C:\\Users\\runneradmin\\AppData\\Roaming\\autonomi | |
- os: macos-latest | |
ant_path: /Users/runner/Library/Application\ Support/autonomi | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@stable | |
- uses: Swatinem/rust-cache@v2 | |
- name: Build binaries | |
run: cargo build --release --features local --bin antnode --bin ant | |
timeout-minutes: 30 | |
- name: Start a local network | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: start | |
enable-evm-testnet: true | |
node-path: target/release/antnode | |
platform: ${{ matrix.os }} | |
build: true | |
- name: Check if ANT_PEERS and EVM_NETWORK are set | |
shell: bash | |
run: | | |
if [[ -z "$ANT_PEERS" ]]; then | |
echo "The ANT_PEERS variable has not been set" | |
exit 1 | |
elif [[ -z "$EVM_NETWORK" ]]; then | |
echo "The EVM_NETWORK variable has not been set" | |
exit 1 | |
else | |
echo "ANT_PEERS has been set to $ANT_PEERS" | |
echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
fi | |
# only these unit tests require a network, the rest are run above in unit test section | |
- name: Run autonomi --tests | |
run: cargo test --package autonomi --tests -- --nocapture | |
env: | |
ANT_LOG: "v" | |
# only set the target dir for windows to bypass the linker issue. | |
# happens if we build the node manager via testnet action | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 15 | |
# FIXME: do this in a generic way for localtestnets | |
- name: export default secret key | |
if: matrix.os != 'windows-latest' | |
run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" >> $GITHUB_ENV | |
shell: bash | |
- name: Set secret key for Windows | |
if: matrix.os == 'windows-latest' | |
run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
shell: pwsh | |
- name: Get file cost | |
run: ./target/release/ant --log-output-dest=data-dir file cost "./resources" | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: File upload | |
run: ./target/release/ant --log-output-dest=data-dir file upload "./resources" > ./upload_output 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: parse address (unix) | |
if: matrix.os != 'windows-latest' | |
run: | | |
UPLOAD_ADDRESS=$(rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output) | |
echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" >> $GITHUB_ENV | |
shell: bash | |
- name: parse address (win) | |
if: matrix.os == 'windows-latest' | |
run: | | |
$UPLOAD_ADDRESS = rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output | |
echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
shell: pwsh | |
- name: File Download | |
run: ./target/release/ant --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: Generate register signing key | |
run: ./target/release/ant --log-output-dest=data-dir register generate-key | |
- name: Create register (writeable by owner) | |
run: ./target/release/ant --log-output-dest=data-dir register create baobao 123 > ./register_create_output 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 10 | |
- name: parse register address (unix) | |
if: matrix.os != 'windows-latest' | |
run: | | |
REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_create_output) | |
echo "REGISTER_ADDRESS=$REGISTER_ADDRESS" >> $GITHUB_ENV | |
shell: bash | |
- name: parse register address (win) | |
if: matrix.os == 'windows-latest' | |
run: | | |
$REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_create_output | |
echo "REGISTER_ADDRESS=$REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
shell: pwsh | |
- name: Get register | |
run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: Edit register | |
run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.REGISTER_ADDRESS }} 456 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 10 | |
- name: Get register (after edit) | |
run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: Create Public Register (writeable by anyone) | |
run: ./target/release/ant --log-output-dest=data-dir register create bao 111 --public > ./register_public_create_output 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: parse public register address (unix) | |
if: matrix.os != 'windows-latest' | |
run: | | |
PUBLIC_REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output) | |
echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" >> $GITHUB_ENV | |
shell: bash | |
- name: parse public register address (win) | |
if: matrix.os == 'windows-latest' | |
run: | | |
$PUBLIC_REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output | |
echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
shell: pwsh | |
- name: Get Public Register (current key is the owner) | |
run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: Edit Public Register (current key is the owner) | |
run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 222 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 10 | |
- name: Delete current register signing key | |
shell: bash | |
run: rm -rf ${{ matrix.ant_path }}/client | |
- name: Generate new register signing key | |
run: ./target/release/ant --log-output-dest data-dir register generate-key | |
- name: Get Public Register (new signing key is not the owner) | |
run: ./target/release/ant --log-output-dest data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: Edit Public Register (new signing key is not the owner) | |
run: ./target/release/ant --log-output-dest data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 333 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 10 | |
- name: Get Public Register (new signing key is not the owner) | |
run: ./target/release/ant --log-output-dest data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: create local user file | |
run: echo random > random.txt | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: file upload | |
run: ./target/release/ant --log-output-dest data-dir file upload random.txt | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: create a local register | |
run: ./target/release/ant --log-output-dest data-dir register create sample_new_register 1234 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: Estimate cost to create a vault | |
run: ./target/release/ant --log-output-dest data-dir vault cost | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: create a vault with existing user data as above | |
run: ./target/release/ant --log-output-dest data-dir vault create | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: add more files - linux/macos | |
if: matrix.os != 'windows-latest' | |
run: | | |
set -e | |
for i in {1..50}; do | |
dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none | |
./target/release/ant --log-output-dest data-dir file upload random_file_$i.bin --public | |
./target/release/ant --log-output-dest data-dir file upload random_file_$i.bin | |
./target/release/ant --log-output-dest data-dir register create $i random_file_$i.bin | |
done | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 25 | |
- name: add more files - windows | |
if: matrix.os == 'windows-latest' | |
shell: pwsh | |
run: | | |
$ErrorActionPreference = "Stop" | |
for ($i = 1; $i -le 50; $i++) { | |
$fileName = "random_file_$i.bin" | |
$byteArray = [byte[]]@(0xFF) * (1MB) # Create a 1 MB array filled with 0xFF | |
[System.IO.File]::WriteAllBytes($fileName, $byteArray) | |
# Run autonomi commands | |
./target/release/ant --log-output-dest data-dir file upload "random_file_$i.bin" --public | |
./target/release/ant --log-output-dest data-dir file upload "random_file_$i.bin" | |
./target/release/ant --log-output-dest data-dir register create $i "random_file_$i.bin" | |
} | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 25 | |
- name: sync the vault | |
run: ./target/release/ant --log-output-dest data-dir vault sync | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: load the vault from network | |
run: ./target/release/ant --log-output-dest data-dir vault load | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: vault sync validation | |
if: matrix.os != 'windows-latest' | |
shell: bash | |
run: | | |
set -e | |
NUM_OF_PUBLIC_FILES="" | |
NUM_OF_PRIVATE_FILES="" | |
NUM_OF_PUBLIC_FILES_IN_VAULT="" | |
NUM_OF_PRIVATE_FILES_IN_VAULT="" | |
./target/release/ant --log-output-dest data-dir file list 2>&1 > file_list.txt | |
NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` | |
NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` | |
./target/release/ant --log-output-dest data-dir vault load 2>&1 > vault_data.txt | |
NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` | |
NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` | |
echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" | |
echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" | |
rm -rf file_list.txt vault_data.txt | |
python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: local data and vault in network dont match, Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT | |
python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: local data and vault in network dont match, Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT | |
echo "vault synced successfully!" | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: Set up variables - vault sync - windows | |
if: matrix.os == 'windows-latest' | |
shell: pwsh | |
run: | | |
$ErrorActionPreference = "Stop" | |
./target/release/ant --log-output-dest data-dir file list > file_list.txt 2>&1 | |
./target/release/ant --log-output-dest data-dir vault load > vault_data.txt 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: Vault sync validation | |
if: matrix.os == 'windows-latest' | |
shell: python | |
run: | | |
import re | |
def find_number_before_word(file_name, search_word): | |
""" | |
Reads a file and finds the number immediately preceding a specified word in a line. | |
:param file_name: Name of the file to read. | |
:param search_word: Word to search for in the file. | |
:return: The number before the word as an integer, or None if not found. | |
""" | |
try: | |
with open(file_name, 'r') as file: | |
for line in file: | |
if search_word in line: | |
match = re.search(r'(\d+)\s+' + re.escape(search_word), line) | |
if match: | |
return int(match.group(1)) # Convert to integer | |
return None # Return None if no match is found | |
except FileNotFoundError: | |
print(f"Error: File '{file_name}' not found.") | |
return None | |
NUM_OF_PUBLIC_FILES = find_number_before_word("file_list.txt", "public") | |
print("NUM_OF_PUBLIC_FILES:", NUM_OF_PUBLIC_FILES) | |
NUM_OF_PRIVATE_FILES = find_number_before_word("file_list.txt", "private") | |
print("NUM_OF_PRIVATE_FILES:", NUM_OF_PRIVATE_FILES) | |
NUM_OF_PUBLIC_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "public") | |
print("NUM_OF_PUBLIC_FILES_IN_VAULT:", NUM_OF_PUBLIC_FILES_IN_VAULT) | |
NUM_OF_PRIVATE_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "private") | |
print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT) | |
# Assertions | |
assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: local data and vault in network dont match, Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" | |
assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: local data and vault in network dont match, Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" | |
print("Vault synced successfully!") | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: load an existing vault from the network | |
run: ./target/release/ant --log-output-dest=data-dir vault load | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 2 | |
- name: Time profiling for Different files | |
if: matrix.os != 'windows-latest' | |
run: | | |
set -e | |
# 1 MB | |
python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" | |
# 10 MB | |
python3 -c "with open('random_10MB.bin', 'wb') as f: f.write(bytearray([0xff] * 10 * 1024 * 1024))" | |
# 100 MB | |
python3 -c "with open('random_100MB.bin', 'wb') as f: f.write(bytearray([0xff] * 100 * 1024 * 1024))" | |
# 1 GB | |
python3 -c "with open('random_1GB.bin', 'wb') as f: f.write(bytearray([0xff] * 1000 * 1024 * 1024))" | |
./target/release/ant --log-output-dest=data-dir file list | |
time ./target/release/ant --log-output-dest=data-dir file upload random_1MB.bin | |
time ./target/release/ant --log-output-dest=data-dir file upload random_10MB.bin | |
time ./target/release/ant --log-output-dest=data-dir file upload random_100MB.bin | |
time ./target/release/ant --log-output-dest=data-dir file upload random_1GB.bin | |
./target/release/ant --log-output-dest=data-dir vault sync | |
rm -rf random*.bin | |
rm -rf ${{ matrix.ant_path }}/autonomi | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: Stop the local network and upload logs | |
if: always() | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: stop | |
log_file_prefix: safe_test_logs_e2e | |
platform: ${{ matrix.os }} | |
churn: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: Network churning tests | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
include: | |
- os: ubuntu-latest | |
node_data_path: /home/runner/.local/share/autonomi/node | |
ant_path: /home/runner/.local/share/autonomi | |
- os: windows-latest | |
node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\autonomi\\node | |
ant_path: C:\\Users\\runneradmin\\AppData\\Roaming\\autonomi | |
- os: macos-latest | |
node_data_path: /Users/runner/Library/Application Support/autonomi/node | |
ant_path: /Users/runner/Library/Application Support/autonomi | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: dtolnay/rust-toolchain@stable | |
- uses: Swatinem/rust-cache@v2 | |
- name: Build binaries | |
run: cargo build --release --features local --bin antnode | |
timeout-minutes: 30 | |
- name: Build churn tests | |
run: cargo test --release -p ant-node --features=local --test data_with_churn --no-run | |
env: | |
# only set the target dir for windows to bypass the linker issue. | |
# happens if we build the node manager via testnet action | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 30 | |
- name: Start a local network | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: start | |
enable-evm-testnet: true | |
node-path: target/release/antnode | |
platform: ${{ matrix.os }} | |
build: true | |
- name: Check if ANT_PEERS and EVM_NETWORK are set | |
shell: bash | |
run: | | |
if [[ -z "$ANT_PEERS" ]]; then | |
echo "The ANT_PEERS variable has not been set" | |
exit 1 | |
elif [[ -z "$EVM_NETWORK" ]]; then | |
echo "The EVM_NETWORK variable has not been set" | |
exit 1 | |
else | |
echo "ANT_PEERS has been set to $ANT_PEERS" | |
echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
fi | |
- name: Chunks data integrity during nodes churn | |
run: cargo test --release -p ant-node --features=local --test data_with_churn -- --nocapture | |
env: | |
TEST_DURATION_MINS: 5 | |
TEST_TOTAL_CHURN_CYCLES: 15 | |
ANT_LOG: "all" | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 30 | |
# Sleep for a while to allow restarted nodes can be detected by others | |
- name: Sleep a while | |
run: sleep 300 | |
- name: Stop the local network and upload logs | |
if: always() | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: stop | |
log_file_prefix: safe_test_logs_churn | |
platform: ${{ matrix.os }} | |
- name: Get total node count | |
shell: bash | |
timeout-minutes: 1 | |
run: | | |
node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
echo "Node dir count is $node_count" | |
- name: Get restart of nodes using rg | |
shell: bash | |
timeout-minutes: 1 | |
# get the counts, then the specific line, and then the digit count only | |
# then check we have an expected level of restarts | |
# TODO: make this use an env var, or relate to testnet size | |
run: | | |
restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) | |
echo "Restarted $restart_count nodes" | |
# `PeerRemovedFromRoutingTable` now only happens when a peer reported as `BadNode`. | |
# Otherwise kad will remove a `dropped out node` directly from RT. | |
# So, the detection of the removal explicity will now have much less chance, | |
# due to the removal of connection_issue tracking. | |
- name: Get peers removed from nodes using rg | |
shell: bash | |
timeout-minutes: 1 | |
run: | | |
peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to extract peer removal count"; exit 0; } | |
if [ -z "$peer_removed" ]; then | |
echo "No peer removal count found" | |
exit 1 | |
fi | |
echo "PeerRemovedFromRoutingTable $peer_removed times" | |
# TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here | |
# if [ $restart_count -lt $node_count ]; then | |
# echo "Restart count of: $restart_count is less than the node count of: $node_count" | |
# exit 1 | |
# fi | |
- name: Verify data replication using rg | |
shell: bash | |
timeout-minutes: 1 | |
# get the counts, then the specific line, and then the digit count only | |
# then check we have an expected level of replication | |
# TODO: make this use an env var, or relate to testnet size | |
run: | | |
fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.node_data_path }}" -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) | |
echo "Carried out $fetching_attempt_count fetching attempts" | |
node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
if [ $fetching_attempt_count -lt $node_count ]; then | |
echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count" | |
exit 1 | |
fi | |
# Only error out after uploading the logs | |
- name: Don't log raw data | |
if: matrix.os != 'windows-latest' # causes error | |
shell: bash | |
timeout-minutes: 10 | |
run: | | |
if ! rg '^' "${{ matrix.ant_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }' | |
then | |
echo "We are logging an extremely large data" | |
exit 1 | |
fi | |
verify_data_location_routing_table: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: Verify data location and Routing Table | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
include: | |
- os: ubuntu-latest | |
node_data_path: /home/runner/.local/share/autonomi/node | |
ant_path: /home/runner/.local/share/autonomi | |
- os: windows-latest | |
node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\autonomi\\node | |
ant_path: C:\\Users\\runneradmin\\AppData\\Roaming\\autonomi | |
- os: macos-latest | |
node_data_path: /Users/runner/Library/Application Support/autonomi/node | |
ant_path: /Users/runner/Library/Application Support/autonomi | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: dtolnay/rust-toolchain@stable | |
- uses: Swatinem/rust-cache@v2 | |
- name: Build binaries | |
run: cargo build --release --features local --bin antnode | |
timeout-minutes: 30 | |
- name: Build data location and routing table tests | |
run: cargo test --release -p ant-node --features=local --test verify_data_location --test verify_routing_table --no-run | |
env: | |
# only set the target dir for windows to bypass the linker issue. | |
# happens if we build the node manager via testnet action | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 30 | |
- name: Start a local network | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: start | |
enable-evm-testnet: true | |
node-path: target/release/antnode | |
platform: ${{ matrix.os }} | |
build: true | |
- name: Check if ANT_PEERS and EVM_NETWORK are set | |
shell: bash | |
run: | | |
if [[ -z "$ANT_PEERS" ]]; then | |
echo "The ANT_PEERS variable has not been set" | |
exit 1 | |
elif [[ -z "$EVM_NETWORK" ]]; then | |
echo "The EVM_NETWORK variable has not been set" | |
exit 1 | |
else | |
echo "ANT_PEERS has been set to $ANT_PEERS" | |
echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
fi | |
- name: Verify the routing tables of the nodes | |
run: cargo test --release -p ant-node --features "local" --test verify_routing_table -- --nocapture | |
env: | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 5 | |
- name: Verify the location of the data on the network | |
run: cargo test --release -p ant-node --features "local" --test verify_data_location -- --nocapture | |
env: | |
CHURN_COUNT: 6 | |
ANT_LOG: "all" | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 25 | |
- name: Verify the routing tables of the nodes | |
run: cargo test --release -p ant-node --features "local" --test verify_routing_table -- --nocapture | |
env: | |
CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
timeout-minutes: 5 | |
# Sleep for a while to allow restarted nodes can be detected by others | |
- name: Sleep a while | |
run: sleep 300 | |
- name: Stop the local network and upload logs | |
if: always() | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: stop | |
log_file_prefix: safe_test_logs_data_location | |
platform: ${{ matrix.os }} | |
- name: Verify restart of nodes using rg | |
shell: bash | |
timeout-minutes: 1 | |
# get the counts, then the specific line, and then the digit count only | |
# then check we have an expected level of restarts | |
# | |
# `PeerRemovedFromRoutingTable` now only happens when a peer reported as `BadNode`. | |
# Otherwise kad will remove a `dropped out node` directly from RT. | |
# So, the detection of the removal explicity will now have much less chance, | |
# due to the removal of connection_issue tracking. | |
# | |
# With the further reduction of replication frequency, | |
# it now becomes harder to detect a `dropped out node` as a `failed to replicate` node. | |
# Hence now remove the assertion check and replace with a print out only. | |
run: | | |
node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
echo "Node dir count is $node_count" | |
restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) | |
echo "Restart $restart_count nodes" | |
if ! rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | |
then | |
echo "No peer removal count found" | |
exit 0 | |
fi | |
peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) | |
echo "PeerRemovedFromRoutingTable $peer_removed times" | |
# Only error out after uploading the logs | |
- name: Don't log raw data | |
if: matrix.os != 'windows-latest' # causes error | |
shell: bash | |
timeout-minutes: 10 | |
run: | | |
if ! rg '^' "${{ matrix.ant_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }' | |
then | |
echo "We are logging an extremely large data" | |
exit 1 | |
fi | |
large_file_upload_test: | |
if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
name: Large file upload | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@stable | |
- uses: Swatinem/rust-cache@v2 | |
- name: install ripgrep | |
shell: bash | |
run: sudo apt-get install -y ripgrep | |
- name: Check the available space | |
run: | | |
df | |
echo "Home dir:" | |
du -sh /home/runner/ | |
echo "Home subdirs:" | |
du -sh /home/runner/*/ | |
echo "PWD:" | |
du -sh . | |
echo "PWD subdirs:" | |
du -sh */ | |
- name: Download material (135MB) | |
shell: bash | |
run: | | |
mkdir test_data_1 | |
cd test_data_1 | |
wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safe-qiWithListeners-x86_64.tar.gz | |
wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safenode-qiWithListeners-x86_64.tar.gz | |
ls -l | |
cd .. | |
tar -cvzf test_data_1.tar.gz test_data_1 | |
ls -l | |
- name: Build binaries | |
run: cargo build --release --features local --bin antnode --bin ant | |
timeout-minutes: 30 | |
- name: Start a local network | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: start | |
enable-evm-testnet: true | |
node-path: target/release/antnode | |
platform: ubuntu-latest | |
build: true | |
- name: Check if ANT_PEERS and EVM_NETWORK are set | |
shell: bash | |
run: | | |
if [[ -z "$ANT_PEERS" ]]; then | |
echo "The ANT_PEERS variable has not been set" | |
exit 1 | |
elif [[ -z "$EVM_NETWORK" ]]; then | |
echo "The EVM_NETWORK variable has not been set" | |
exit 1 | |
else | |
echo "ANT_PEERS has been set to $ANT_PEERS" | |
echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
fi | |
- name: Check the available space post download | |
run: | | |
df | |
echo "Home dir:" | |
du -sh /home/runner/ | |
echo "Home subdirs:" | |
du -sh /home/runner/*/ | |
echo "PWD:" | |
du -sh . | |
echo "PWD subdirs:" | |
du -sh */ | |
- name: export default secret key | |
run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" >> $GITHUB_ENV | |
shell: bash | |
- name: File upload | |
run: ./target/release/ant --log-output-dest data-dir file upload "./test_data_1.tar.gz" > ./upload_output 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 15 | |
- name: showing the upload terminal output | |
run: cat upload_output | |
shell: bash | |
if: always() | |
- name: parse address | |
run: | | |
UPLOAD_ADDRESS=$(rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output) | |
echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" >> $GITHUB_ENV | |
shell: bash | |
- name: File Download | |
run: ./target/release/ant --log-output-dest data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources > ./download_output 2>&1 | |
env: | |
ANT_LOG: "v" | |
timeout-minutes: 5 | |
- name: showing the download terminal output | |
run: | | |
cat download_output | |
ls -l | |
cd downloaded_resources | |
ls -l | |
shell: bash | |
if: always() | |
- name: Confirming connection errors | |
shell: bash | |
timeout-minutes: 1 | |
env: | |
NODE_DATA_PATH: /home/runner/.local/share/autonomi/node | |
run: | | |
incoming_connection_errors=$(rg "IncomingConnectionError" $NODE_DATA_PATH -c --stats | \ | |
rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to find IncomingConnectionError error"; exit 0; } | |
if [ -z "$incoming_connection_errors" ]; then | |
echo "Doesn't find any IncomingConnectionError error !" | |
else | |
echo "Found $incoming_connection_errors IncomingConnectionError errors." | |
fi | |
if ! rg "UnexpectedEof" $NODE_DATA_PATH -c --stats; then | |
echo "Doesn't find any UnexpectedEof error !" | |
else | |
echo "Found errors." | |
exit 1 | |
fi | |
- name: Stop the local network and upload logs | |
if: always() | |
uses: maidsafe/ant-local-testnet-action@main | |
with: | |
action: stop | |
platform: ubuntu-latest | |
log_file_prefix: safe_test_logs_large_file_upload_no_ws | |
build: true |