diff --git a/.cicd/platforms.json b/.cicd/platforms.json deleted file mode 100644 index fccc8dbc00..0000000000 --- a/.cicd/platforms.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "ubuntu20": { - "dockerfile": ".cicd/platforms/ubuntu20.Dockerfile" - }, - "ubuntu22": { - "dockerfile": ".cicd/platforms/ubuntu22.Dockerfile" - } -} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6739f7eec2..1a03de2215 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -31,23 +31,23 @@ defaults: shell: bash jobs: - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: packages: write contents: read + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: .cicd/platforms build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml - needs: [platforms] + needs: [platform-cache] with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} - permissions: - packages: write - contents: read + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: ${{needs.platform-cache.outputs.platform-list}} v: name: Discover Versions @@ -79,14 +79,13 @@ jobs: dev-package: name: Build leap-dev package - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ubuntu-latest - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: @@ -115,15 +114,14 @@ jobs: tests: name: Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ["self-hosted", "enf-x86-hightier"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 @@ -143,8 +141,7 @@ jobs: np-tests: name: NP Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: @@ -159,7 +156,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: nonparallelizable_tests @@ -173,8 +170,7 @@ jobs: lr-tests: name: LR Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: @@ -189,7 +185,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: long_running_tests @@ -203,15 +199,14 @@ jobs: libtester-tests: name: libtester tests - needs: [platforms, build-base, v, dev-package] - if: always() && needs.platforms.result == 'success' && needs.v.result == 'success' && needs.dev-package.result == 'success' + needs: [platform-cache, build-base, v, dev-package] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] test: [build-tree, make-dev-install, deb-install] runs-on: ["self-hosted", "enf-x86-midtier"] - container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platforms.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} + container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} env: DEBIAN_FRONTEND: noninteractive TZ: Etc/UTC @@ -296,8 +291,6 @@ jobs: all-passing: name: All Required Tests Passed needs: [dev-package, tests, np-tests, libtester-tests] - if: always() runs-on: ubuntu-latest steps: - - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success' - run: false + - run: true diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 51dd5d2167..5e6639f968 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -3,12 +3,12 @@ name: "Build leap" on: workflow_call: inputs: - p: - description: "Discovered Build Platforms" + platforms: + description: "Platforms definitions" type: string required: true - platform-matrix: - description: "Platform Matrix" + platform-list: + description: "Array of platforms" type: string required: true @@ -26,9 +26,9 @@ jobs: strategy: fail-fast: false matrix: - platform: ${{fromJSON(inputs.platform-matrix)}} + platform: ${{fromJSON(inputs.platform-list)}} runs-on: ["self-hosted", "enf-x86-beefy"] - container: ${{fromJSON(inputs.p)[matrix.platform].image}} + container: ${{fromJSON(inputs.platforms)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/performance_harness_run.yaml b/.github/workflows/performance_harness_run.yaml index ea49f4375b..ce037555a7 100644 --- a/.github/workflows/performance_harness_run.yaml +++ b/.github/workflows/performance_harness_run.yaml @@ -58,21 +58,19 @@ jobs: echo leap-prerelease=${{inputs.override-leap-prerelease}} >> $GITHUB_OUTPUT fi - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml - with: - override-build-matrix: ${{github.event.inputs.platform-choice}} + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: packages: write contents: read + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: .cicd/platforms reuse-build: name: Reuse leap build needs: [v] - if: | - always() && - needs.v.result == 'success' runs-on: ubuntu-latest outputs: build-artifact: ${{steps.downloadBuild.outputs.downloaded-file}} @@ -97,23 +95,20 @@ jobs: build-base: name: Run Build Workflow - needs: [platforms, reuse-build] - if: always() && needs.platforms.result == 'success' && needs.reuse-build.outputs.build-artifact == '' + needs: [platform-cache, reuse-build] + if: needs.reuse-build.outputs.build-artifact == '' uses: ./.github/workflows/build_base.yaml with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} - permissions: - packages: write - contents: read + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: '["${{github.event.inputs.platform-choice}}"]' tests: name: Tests - needs: [v, platforms, reuse-build, build-base] - if: always() && needs.platforms.result == 'success' && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success') + needs: [v, platform-cache, reuse-build, build-base] + if: always() && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success') runs-on: ["Leap-Perf-Ubuntu-22-16x64x600"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[github.event.inputs.platform-choice].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[github.event.inputs.platform-choice].image}} steps: - name: Download builddir uses: actions/download-artifact@v3 diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index a5cdd04b10..4633108821 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -12,36 +12,38 @@ defaults: shell: bash jobs: - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: - packages: write contents: read + packages: write + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: .cicd/platforms build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml - needs: [platforms] + needs: [platform-cache] with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: ${{needs.platform-cache.outputs.platform-list}} permissions: packages: write contents: read tests: name: Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: ${{fromJSON(needs.platforms.outputs.platform-matrix)}} + platform: ${{fromJSON(needs.platform-cache.outputs.platform-list)}} release: [3.1, 3.2, 4.0] runs-on: ["self-hosted", "enf-x86-lowtier"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/platforms.yaml b/.github/workflows/platforms.yaml deleted file mode 100644 index e44ce0ffda..0000000000 --- a/.github/workflows/platforms.yaml +++ /dev/null @@ -1,88 +0,0 @@ -name: "Platforms" - -on: - workflow_call: - inputs: - override-build-matrix: - description: 'Override build matrix' - type: string - required: false - outputs: - p: - description: "Discovered Build Platforms" - value: ${{ jobs.d.outputs.p }} - platform-matrix: - description: "Overridden Platform Matrix" - value: ${{ jobs.pm.outputs.platform-matrix }} - -permissions: - packages: read - contents: read - -defaults: - run: - shell: bash - -jobs: - d: - name: Discover Platforms - runs-on: ubuntu-latest - outputs: - missing-platforms: ${{steps.discover.outputs.missing-platforms}} - p: ${{steps.discover.outputs.platforms}} - steps: - - name: Discover Platforms - id: discover - uses: AntelopeIO/discover-platforms-action@v1 - with: - platform-file: .cicd/platforms.json - password: ${{secrets.GITHUB_TOKEN}} - package-name: builders - - build-platforms: - name: Build Platforms - needs: d - if: needs.d.outputs.missing-platforms != '[]' - strategy: - fail-fast: false - matrix: - platform: ${{fromJSON(needs.d.outputs.missing-platforms)}} - runs-on: ["self-hosted", "enf-x86-beefy"] - permissions: - packages: write - contents: read - steps: - - name: Login to Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{github.repository_owner}} - password: ${{secrets.GITHUB_TOKEN}} - - name: Build and push - uses: docker/build-push-action@v3 - with: - push: true - tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}} - file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}} - - pm: - name: Platform Matrix - needs: [d] - if: always() && needs.d.result == 'success' - runs-on: ubuntu-latest - outputs: - platform-matrix: ${{steps.pm-results.outputs.platform-matrix}} - steps: - - name: Parse Platform Matrix - id: parse-pm - uses: actions/github-script@v6 - with: - script: return Object.keys(${{needs.d.outputs.p}}) - - name: Check | Override result - id: pm-results - run: | - echo 'platform-matrix=${{steps.parse-pm.outputs.result}}' >> $GITHUB_OUTPUT - - if [[ "${{inputs.override-build-matrix}}" != "" ]]; then - echo 'platform-matrix=["${{inputs.override-build-matrix}}"]' >> $GITHUB_OUTPUT - fi diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 082aeb02f6..b97008a9df 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -326,7 +326,7 @@ namespace eosio { namespace chain { block_log_data log_data; block_log_index log_index; - block_log_bundle(std::filesystem::path block_file, std::filesystem::path index_file) + block_log_bundle(std::filesystem::path block_file, std::filesystem::path index_file, bool validate_indx) : block_file_name(std::move(block_file)), index_file_name(std::move(index_file)) { log_data.open(block_file_name); @@ -335,6 +335,15 @@ namespace eosio { namespace chain { EOS_ASSERT(!log_data.get_preamble().is_currently_pruned(), block_log_unsupported_version, "Block log is currently in pruned format, it must be vacuumed before doing this operation"); + if (validate_indx) + validate_index(); + } + + explicit block_log_bundle(const std::filesystem::path& block_dir, bool validate_index=true) + : block_log_bundle(block_dir / "blocks.log", block_dir / "blocks.index", validate_index) {} + + // throws if not valid + void validate_index() { uint32_t log_num_blocks = log_data.num_blocks(); uint32_t index_num_blocks = log_index.num_blocks(); @@ -345,9 +354,6 @@ namespace eosio { namespace chain { ("block_file_name", block_file_name)("log_num_blocks", log_num_blocks)( "index_num_blocks", index_num_blocks)("index_file_name", index_file_name)); } - - explicit block_log_bundle(const std::filesystem::path& block_dir) - : block_log_bundle(block_dir / "blocks.log", block_dir / "blocks.index") {} }; /// Used to traverse the block position (i.e. the last 8 bytes in each block log entry) of the blocks.log file @@ -1572,12 +1578,14 @@ namespace eosio { namespace chain { // static void block_log::smoke_test(const std::filesystem::path& block_dir, uint32_t interval) { - block_log_bundle log_bundle(block_dir); + block_log_bundle log_bundle(block_dir, false); ilog("block log version= ${version}",("version", log_bundle.log_data.version())); ilog("first block= ${first}",("first", log_bundle.log_data.first_block_num())); ilog("last block= ${last}",("last", log_bundle.log_data.last_block_num())); + log_bundle.validate_index(); + ilog("blocks.log and blocks.index agree on number of blocks"); if (interval == 0) { diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 13166ce0bf..807660bd6f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3025,7 +3025,8 @@ namespace eosio { return true; } if (my_impl->sync_master->syncing_from_peer()) { - peer_wlog(this, "syncing, dropping trx"); + peer_dlog(this, "syncing, dropping trx"); + pending_message_buffer.advance_read_ptr( message_length ); return true; } diff --git a/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp b/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp index 0413bd57c1..1ca104cf67 100644 --- a/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp +++ b/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace eosio { @@ -13,7 +15,7 @@ namespace eosio { prometheus_plugin(); ~prometheus_plugin() override; - APPBASE_PLUGIN_REQUIRES((http_plugin)(chain_plugin)) + APPBASE_PLUGIN_REQUIRES((http_plugin)(chain_plugin)(producer_plugin)(net_plugin)) void set_program_options(options_description&, options_description& cfg) override; diff --git a/plugins/prometheus_plugin/prometheus_plugin.cpp b/plugins/prometheus_plugin/prometheus_plugin.cpp index 2f14e5eba3..ff929d282f 100644 --- a/plugins/prometheus_plugin/prometheus_plugin.cpp +++ b/plugins/prometheus_plugin/prometheus_plugin.cpp @@ -3,9 +3,7 @@ #include #include #include -#include #include -#include #include @@ -36,9 +34,6 @@ namespace eosio { prometheus_plugin::~prometheus_plugin() = default; void prometheus_plugin::set_program_options(options_description&, options_description& cfg) { - cfg.add_options() - ("prometheus-exporter-address", bpo::value()->default_value("127.0.0.1:9101"), - "The local IP and port to listen for incoming prometheus metrics http request."); } struct prometheus_api_handle { diff --git a/tests/trx_generator/http_client_async.hpp b/tests/trx_generator/http_client_async.hpp index 15e525a0aa..7aad239e5c 100644 --- a/tests/trx_generator/http_client_async.hpp +++ b/tests/trx_generator/http_client_async.hpp @@ -73,8 +73,9 @@ class session : public std::enable_shared_from_this { req_.set(http::field::host, host); req_.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); req_.set(http::field::content_type, content_type); - req_.set(http::field::connection, "close"); req_.body() = std::move(request_body); + // current implementation does not reuse socket, disable keep_alive + req_.set(http::field::connection, "close"); req_.keep_alive(false); req_.prepare_payload();