From a5038bec01ffdba0b65e444894dbed67bedf8a82 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Tue, 22 Aug 2023 13:46:35 -0700 Subject: [PATCH] Revert "Revert "swich to gcp"" This reverts commit 72b46b676e12b31b6dac398479398720d8157a21. --- .github/workflows/module-verify.yaml | 12 ++++++------ .github/workflows/replay-verify.yaml | 18 +++++++++--------- .../workflows/workflow-run-module-verify.yaml | 10 +++++----- .../workflows/workflow-run-replay-verify.yaml | 10 +++++----- .../docs/nodes/full-node/aptos-db-restore.md | 6 +++--- testsuite/module_verify_run_local.py | 4 ++-- testsuite/replay_verify.py | 10 +++++----- testsuite/replay_verify_run_local.py | 4 ++-- 8 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/module-verify.yaml b/.github/workflows/module-verify.yaml index 17b99be413f9d..a5fb333c9b189 100644 --- a/.github/workflows/module-verify.yaml +++ b/.github/workflows/module-verify.yaml @@ -35,9 +35,9 @@ jobs: secrets: inherit with: GIT_SHA: ${{ inputs.GIT_SHA }} - BUCKET: aptos-testnet-backup-2223d95b + BUCKET: aptos-testnet-backup-b7b1ad7a SUB_DIR: e1 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: high-perf-docker-with-local-ssd TIMEOUT_MINUTES: 20 @@ -48,9 +48,9 @@ jobs: secrets: inherit with: GIT_SHA: ${{ inputs.GIT_SHA }} - BUCKET: aptos-mainnet-backup-backup-831a69a8 + BUCKET: aptos-mainnet-backup-backup-e098483d SUB_DIR: e1 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: high-perf-docker-with-local-ssd TIMEOUT_MINUTES: 20 @@ -61,9 +61,9 @@ jobs: secrets: inherit with: GIT_SHA: ${{ github.event.pull_request.head.sha }} - BUCKET: aptos-testnet-backup-2223d95b + BUCKET: aptos-testnet-backup-b7b1ad7a SUB_DIR: e1 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: "high-perf-docker-with-local-ssd" TIMEOUT_MINUTES: 20 diff --git a/.github/workflows/replay-verify.yaml b/.github/workflows/replay-verify.yaml index 2bbc6e9d6e0ee..898cfb419f413 100644 --- a/.github/workflows/replay-verify.yaml +++ b/.github/workflows/replay-verify.yaml @@ -51,16 +51,16 @@ jobs: github.event_name == 'push' || github.event_name == 'workflow_dispatch' && (inputs.CHAIN_NAME == 'testnet' || inputs.CHAIN_NAME == 'all') needs: determine-test-metadata - uses: aptos-labs/aptos-core/.github/workflows/workflow-run-replay-verify.yaml@main + uses: ./.github/workflows/workflow-run-replay-verify.yaml secrets: inherit with: GIT_SHA: ${{ inputs.GIT_SHA }} # replay-verify config - BUCKET: aptos-testnet-backup-2223d95b + BUCKET: aptos-testnet-backup-b7b1ad7a SUB_DIR: e1 HISTORY_START: 250000000 # TODO: We need an exhaustive list of txns_to_skip before we can set this to 0. TXNS_TO_SKIP: 46874937 151020059 409163615 409163669 409163708 409163774 409163845 409163955 409164059 409164191 414625832 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: "high-perf-docker-with-local-ssd" TIMEOUT_MINUTES: 840 @@ -71,16 +71,16 @@ jobs: github.event_name == 'push' || github.event_name == 'workflow_dispatch' && (inputs.CHAIN_NAME == 'testnet' || inputs.CHAIN_NAME == 'all' ) needs: determine-test-metadata - uses: aptos-labs/aptos-core/.github/workflows/workflow-run-replay-verify.yaml@main + uses: ./.github/workflows/workflow-run-replay-verify.yaml secrets: inherit with: GIT_SHA: ${{ inputs.GIT_SHA }} # replay-verify config - BUCKET: aptos-mainnet-backup-backup-831a69a8 + BUCKET: aptos-mainnet-backup-backup-e098483d SUB_DIR: e1 HISTORY_START: 0 TXNS_TO_SKIP: 12253479 12277499 148358668 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: "high-perf-docker-with-local-ssd" TIMEOUT_MINUTES: 240 @@ -88,16 +88,16 @@ jobs: test-replay: if: ${{ github.event_name == 'pull_request' }} needs: determine-test-metadata - uses: aptos-labs/aptos-core/.github/workflows/workflow-run-replay-verify.yaml@main + uses: ./.github/workflows/workflow-run-replay-verify.yaml secrets: inherit with: GIT_SHA: ${{ github.event.pull_request.head.sha }} # replay-verify config - BUCKET: aptos-testnet-backup-2223d95b + BUCKET: aptos-testnet-backup-b7b1ad7a SUB_DIR: e1 HISTORY_START: 250000000 # TODO: We need an exhaustive list of txns_to_skip before we can set this to 0. TXNS_TO_SKIP: 46874937 151020059 409163615 409163669 409163708 409163774 409163845 409163955 409164059 409164191 414625832 - BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml + BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml # workflow config RUNS_ON: "high-perf-docker-with-local-ssd" TIMEOUT_MINUTES: 20 diff --git a/.github/workflows/workflow-run-module-verify.yaml b/.github/workflows/workflow-run-module-verify.yaml index 6856fb7afe942..e49058213600b 100644 --- a/.github/workflows/workflow-run-module-verify.yaml +++ b/.github/workflows/workflow-run-module-verify.yaml @@ -46,11 +46,11 @@ jobs: with: GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} - - name: Install AWS CLI - shell: bash - run: | - scripts/dev_setup.sh -b -i awscli - echo "${HOME}/bin/" >> $GITHUB_PATH # default INSTALL_DIR to path + - name: Install GCloud SDK + uses: "google-github-actions/setup-gcloud@62d4898025f6041e16b1068643bfc5a696863587" # pin@v1 + with: + version: ">= 418.0.0" + install_components: "kubectl,gke-gcloud-auth-plugin" - name: Build CLI binaries in release mode shell: bash diff --git a/.github/workflows/workflow-run-replay-verify.yaml b/.github/workflows/workflow-run-replay-verify.yaml index c7ae14703c1b8..4bc5d535e86a8 100644 --- a/.github/workflows/workflow-run-replay-verify.yaml +++ b/.github/workflows/workflow-run-replay-verify.yaml @@ -94,11 +94,11 @@ jobs: with: GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} - - name: Install AWS CLI - shell: bash - run: | - scripts/dev_setup.sh -b -i awscli - echo "${HOME}/bin/" >> $GITHUB_PATH # default INSTALL_DIR to path + - name: Install GCloud SDK + uses: "google-github-actions/setup-gcloud@62d4898025f6041e16b1068643bfc5a696863587" # pin@v1 + with: + version: ">= 418.0.0" + install_components: "kubectl,gke-gcloud-auth-plugin" - name: Build CLI binaries in release mode shell: bash diff --git a/developer-docs-site/docs/nodes/full-node/aptos-db-restore.md b/developer-docs-site/docs/nodes/full-node/aptos-db-restore.md index 75d0ae9a47390..a33919f6d9371 100644 --- a/developer-docs-site/docs/nodes/full-node/aptos-db-restore.md +++ b/developer-docs-site/docs/nodes/full-node/aptos-db-restore.md @@ -54,7 +54,7 @@ Example command: ```bash aptos node bootstrap-db \ --target-version 500000000 \ - --command-adapter-config /path/to/s3-public.yaml \ + --command-adapter-config /path/to/gcs.yaml \ --target-db-dir /path/to/local/db ``` @@ -74,7 +74,7 @@ Example command: aptos node bootstrap-db \ --ledger-history-start-version 150000000 \ --target-version 155000000 - --command-adapter-config /path/to/s3-public.yaml \ + --command-adapter-config /path/to/gcs.yaml \ --target-db-dir /path/to/local/db ``` @@ -100,7 +100,7 @@ Example command: aptos node bootstrap-db \ --ledger-history-start-version 0 \ --target-version use_the_largest_version_in_backup \ ---command-adapter-config /path/to/s3-public.yaml \ +--command-adapter-config /path/to/gcs.yaml \ --target-db-dir /path/to/local/db ``` diff --git a/testsuite/module_verify_run_local.py b/testsuite/module_verify_run_local.py index 3d534b4fa97b5..49f26466a8e3b 100755 --- a/testsuite/module_verify_run_local.py +++ b/testsuite/module_verify_run_local.py @@ -17,9 +17,9 @@ def local_setup(): # Take these from the expected replay verify run envs = { - "BUCKET": "aptos-testnet-backup-2223d95b", + "BUCKET": "aptos-testnet-backup-b7b1ad7a", "SUB_DIR": "e1", - "BACKUP_CONFIG_TEMPLATE_PATH": "terraform/helm/fullnode/files/backup/s3-public.yaml", + "BACKUP_CONFIG_TEMPLATE_PATH": "terraform/helm/fullnode/files/backup/gcs.yaml", } # build backup tools diff --git a/testsuite/replay_verify.py b/testsuite/replay_verify.py index cda63d314918a..d5e041012717c 100755 --- a/testsuite/replay_verify.py +++ b/testsuite/replay_verify.py @@ -73,16 +73,16 @@ def replay_verify_partition( backup_config_template_path, ], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) if process.stdout is None: raise Exception(f"[partition {n}] stdout is None") for line in iter(process.stdout.readline, b""): print(f"[partition {n}] {line}", flush=True) - # set the returncode - process.communicate() + stdout_output, stderr_output = process.communicate() - return (n, process.returncode) + return (n, process.returncode, stdout_output, stderr_output) def main(): @@ -157,10 +157,10 @@ def main(): print("[main process] finished") err = False - for partition_num, return_code in all_partitions: + for partition_num, return_code, stdout, stderr in all_partitions: if return_code != 0: print("======== ERROR ========") - print(f"ERROR: partition {partition_num} failed (exit {return_code})") + print(f"ERROR: partition {partition_num} failed (exit {return_code}, {stderr})") err = True if err: diff --git a/testsuite/replay_verify_run_local.py b/testsuite/replay_verify_run_local.py index 6a337bd4e1fab..c3498a1d7b0c2 100755 --- a/testsuite/replay_verify_run_local.py +++ b/testsuite/replay_verify_run_local.py @@ -18,11 +18,11 @@ def local_setup(): # Take these from the expected replay verify run envs = { "TIMEOUT_MINUTES": "5", - "BUCKET": "aptos-testnet-backup-2223d95b", + "BUCKET": "aptos-testnet-backup-b7b1ad7a", "SUB_DIR": "e1", "HISTORY_START": "350000000", "TXNS_TO_SKIP": "46874937 151020059", - "BACKUP_CONFIG_TEMPLATE_PATH": "terraform/helm/fullnode/files/backup/s3-public.yaml", + "BACKUP_CONFIG_TEMPLATE_PATH": "terraform/helm/fullnode/files/backup/gcs.yaml", } # build backup tools