From 4f2f8419dba67b8eba877d89facfb621febbad25 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 29 Sep 2023 08:49:52 -0400 Subject: [PATCH 01/12] Update appbase to tip (branch `main`) --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index b81a6dc7aa..88078d723b 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit b81a6dc7aa758817fd61ea55b08bf2d0c099eafc +Subproject commit 88078d723b2d135e2f6ba86a7895410e2aa62f3f From 5de8cf538bbb07c7a162e7d57c8cbbdfe8188bb2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 29 Sep 2023 15:37:56 -0500 Subject: [PATCH 02/12] On a eof shutdown both and close for good measure. --- .../include/eosio/http_plugin/beast_http_session.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp index 6496d51ac5..4ad2d928f0 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp @@ -503,7 +503,8 @@ class beast_http_session : public detail::abstract_conn, try { // Send a shutdown signal beast::error_code ec; - socket_.shutdown(Socket::shutdown_send, ec); + socket_.shutdown(Socket::shutdown_both, ec); + socket_.close(ec); // At this point the connection is closed gracefully } catch(...) { handle_exception(); From 25bb1572f20e20b527be57ac44c559dd513c297d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 29 Sep 2023 15:38:50 -0500 Subject: [PATCH 03/12] Set keep_alive false and indicate close on reply. Also add better error handling for ec --- tests/trx_generator/http_client_async.hpp | 8 +++----- tests/trx_generator/trx_provider.cpp | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/trx_generator/http_client_async.hpp b/tests/trx_generator/http_client_async.hpp index d55dff5e46..15e525a0aa 100644 --- a/tests/trx_generator/http_client_async.hpp +++ b/tests/trx_generator/http_client_async.hpp @@ -73,7 +73,9 @@ class session : public std::enable_shared_from_this { req_.set(http::field::host, host); req_.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); req_.set(http::field::content_type, content_type); + req_.set(http::field::connection, "close"); req_.body() = std::move(request_body); + req_.keep_alive(false); req_.prepare_payload(); // Look up the domain name @@ -129,16 +131,12 @@ class session : public std::enable_shared_from_this { void on_read(beast::error_code ec, std::size_t bytes_transferred) { boost::ignore_unused(bytes_transferred); - if (ec) { - response_callback_(ec, {}); - return fail(ec, "read"); - } - // Write the response message to the callback response_callback_(ec, res_); // Gracefully close the socket stream_.socket().shutdown(tcp::socket::shutdown_both, ec); + stream_.close(); // not_connected happens sometimes so don't bother reporting it. if (ec && ec != beast::errc::not_connected) diff --git a/tests/trx_generator/trx_provider.cpp b/tests/trx_generator/trx_provider.cpp index df2541146e..190c6b4632 100644 --- a/tests/trx_generator/trx_provider.cpp +++ b/tests/trx_generator/trx_provider.cpp @@ -133,6 +133,10 @@ namespace eosio::testing { [this, trx_id = trx.id()](boost::beast::error_code ec, boost::beast::http::response response) { trx_acknowledged(trx_id, fc::time_point::now()); + if (ec) { + elog("http error: ${c}: ${m}", ("c", ec.value())("m", ec.message())); + throw std::runtime_error(ec.message()); + } if (this->needs_response_trace_info() && response.result() == boost::beast::http::status::ok) { try { From 5158ccd65d712c03075ad2642b86226e42db01a1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 29 Sep 2023 15:54:36 -0500 Subject: [PATCH 04/12] GH-1681 Remove prometheus-exporter-address option as it is ignored. Misc cleanup. --- .../include/eosio/prometheus_plugin/prometheus_plugin.hpp | 4 +++- plugins/prometheus_plugin/prometheus_plugin.cpp | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp b/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp index 0413bd57c1..1ca104cf67 100644 --- a/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp +++ b/plugins/prometheus_plugin/include/eosio/prometheus_plugin/prometheus_plugin.hpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace eosio { @@ -13,7 +15,7 @@ namespace eosio { prometheus_plugin(); ~prometheus_plugin() override; - APPBASE_PLUGIN_REQUIRES((http_plugin)(chain_plugin)) + APPBASE_PLUGIN_REQUIRES((http_plugin)(chain_plugin)(producer_plugin)(net_plugin)) void set_program_options(options_description&, options_description& cfg) override; diff --git a/plugins/prometheus_plugin/prometheus_plugin.cpp b/plugins/prometheus_plugin/prometheus_plugin.cpp index 2f14e5eba3..ff929d282f 100644 --- a/plugins/prometheus_plugin/prometheus_plugin.cpp +++ b/plugins/prometheus_plugin/prometheus_plugin.cpp @@ -3,9 +3,7 @@ #include #include #include -#include #include -#include #include @@ -36,9 +34,6 @@ namespace eosio { prometheus_plugin::~prometheus_plugin() = default; void prometheus_plugin::set_program_options(options_description&, options_description& cfg) { - cfg.add_options() - ("prometheus-exporter-address", bpo::value()->default_value("127.0.0.1:9101"), - "The local IP and port to listen for incoming prometheus metrics http request."); } struct prometheus_api_handle { From 17110a17d87a40c54186d0930c6efa51079aee0b Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 2 Oct 2023 10:28:37 -0500 Subject: [PATCH 05/12] GH-1662 Add comment --- tests/trx_generator/http_client_async.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/trx_generator/http_client_async.hpp b/tests/trx_generator/http_client_async.hpp index 15e525a0aa..7aad239e5c 100644 --- a/tests/trx_generator/http_client_async.hpp +++ b/tests/trx_generator/http_client_async.hpp @@ -73,8 +73,9 @@ class session : public std::enable_shared_from_this { req_.set(http::field::host, host); req_.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); req_.set(http::field::content_type, content_type); - req_.set(http::field::connection, "close"); req_.body() = std::move(request_body); + // current implementation does not reuse socket, disable keep_alive + req_.set(http::field::connection, "close"); req_.keep_alive(false); req_.prepare_payload(); From efc48d638cff58b6ac3d4c49b4f5694ad2022018 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:53:29 -0400 Subject: [PATCH 06/12] migrate to new platform-cache-workflow --- .cicd/platforms.json | 8 -- .github/workflows/build.yaml | 47 +++++----- .github/workflows/build_base.yaml | 12 +-- .../workflows/performance_harness_run.yaml | 31 +++---- .../workflows/ph_backward_compatibility.yaml | 26 +++--- .github/workflows/platforms.yaml | 88 ------------------- 6 files changed, 54 insertions(+), 158 deletions(-) delete mode 100644 .cicd/platforms.json delete mode 100644 .github/workflows/platforms.yaml diff --git a/.cicd/platforms.json b/.cicd/platforms.json deleted file mode 100644 index fccc8dbc00..0000000000 --- a/.cicd/platforms.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "ubuntu20": { - "dockerfile": ".cicd/platforms/ubuntu20.Dockerfile" - }, - "ubuntu22": { - "dockerfile": ".cicd/platforms/ubuntu22.Dockerfile" - } -} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6739f7eec2..1a03de2215 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -31,23 +31,23 @@ defaults: shell: bash jobs: - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: packages: write contents: read + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: .cicd/platforms build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml - needs: [platforms] + needs: [platform-cache] with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} - permissions: - packages: write - contents: read + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: ${{needs.platform-cache.outputs.platform-list}} v: name: Discover Versions @@ -79,14 +79,13 @@ jobs: dev-package: name: Build leap-dev package - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ubuntu-latest - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: @@ -115,15 +114,14 @@ jobs: tests: name: Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ["self-hosted", "enf-x86-hightier"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 @@ -143,8 +141,7 @@ jobs: np-tests: name: NP Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: @@ -159,7 +156,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: nonparallelizable_tests @@ -173,8 +170,7 @@ jobs: lr-tests: name: LR Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: @@ -189,7 +185,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: long_running_tests @@ -203,15 +199,14 @@ jobs: libtester-tests: name: libtester tests - needs: [platforms, build-base, v, dev-package] - if: always() && needs.platforms.result == 'success' && needs.v.result == 'success' && needs.dev-package.result == 'success' + needs: [platform-cache, build-base, v, dev-package] strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] test: [build-tree, make-dev-install, deb-install] runs-on: ["self-hosted", "enf-x86-midtier"] - container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platforms.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} + container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} env: DEBIAN_FRONTEND: noninteractive TZ: Etc/UTC @@ -296,8 +291,6 @@ jobs: all-passing: name: All Required Tests Passed needs: [dev-package, tests, np-tests, libtester-tests] - if: always() runs-on: ubuntu-latest steps: - - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success' - run: false + - run: true diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 51dd5d2167..5e6639f968 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -3,12 +3,12 @@ name: "Build leap" on: workflow_call: inputs: - p: - description: "Discovered Build Platforms" + platforms: + description: "Platforms definitions" type: string required: true - platform-matrix: - description: "Platform Matrix" + platform-list: + description: "Array of platforms" type: string required: true @@ -26,9 +26,9 @@ jobs: strategy: fail-fast: false matrix: - platform: ${{fromJSON(inputs.platform-matrix)}} + platform: ${{fromJSON(inputs.platform-list)}} runs-on: ["self-hosted", "enf-x86-beefy"] - container: ${{fromJSON(inputs.p)[matrix.platform].image}} + container: ${{fromJSON(inputs.platforms)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/performance_harness_run.yaml b/.github/workflows/performance_harness_run.yaml index ea49f4375b..ce037555a7 100644 --- a/.github/workflows/performance_harness_run.yaml +++ b/.github/workflows/performance_harness_run.yaml @@ -58,21 +58,19 @@ jobs: echo leap-prerelease=${{inputs.override-leap-prerelease}} >> $GITHUB_OUTPUT fi - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml - with: - override-build-matrix: ${{github.event.inputs.platform-choice}} + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: packages: write contents: read + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: .cicd/platforms reuse-build: name: Reuse leap build needs: [v] - if: | - always() && - needs.v.result == 'success' runs-on: ubuntu-latest outputs: build-artifact: ${{steps.downloadBuild.outputs.downloaded-file}} @@ -97,23 +95,20 @@ jobs: build-base: name: Run Build Workflow - needs: [platforms, reuse-build] - if: always() && needs.platforms.result == 'success' && needs.reuse-build.outputs.build-artifact == '' + needs: [platform-cache, reuse-build] + if: needs.reuse-build.outputs.build-artifact == '' uses: ./.github/workflows/build_base.yaml with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} - permissions: - packages: write - contents: read + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: '["${{github.event.inputs.platform-choice}}"]' tests: name: Tests - needs: [v, platforms, reuse-build, build-base] - if: always() && needs.platforms.result == 'success' && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success') + needs: [v, platform-cache, reuse-build, build-base] + if: always() && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success') runs-on: ["Leap-Perf-Ubuntu-22-16x64x600"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[github.event.inputs.platform-choice].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[github.event.inputs.platform-choice].image}} steps: - name: Download builddir uses: actions/download-artifact@v3 diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index a5cdd04b10..c5f0a5a277 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -12,36 +12,40 @@ defaults: shell: bash jobs: - platforms: - name: Run Platforms Workflow - uses: ./.github/workflows/platforms.yaml + platform-cache: + name: Platform Cache + uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1 permissions: - packages: write contents: read + packages: write + with: + runs-on: '["self-hosted", "enf-x86-beefy"]' + platform-files: | + .cicd/platforms + tools/reproducible.Dockerfile:builder build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml - needs: [platforms] + needs: [platform-cache] with: - p: ${{needs.platforms.outputs.p}} - platform-matrix: ${{needs.platforms.outputs.platform-matrix}} + platforms: ${{needs.platform-cache.outputs.platforms}} + platform-list: ${{needs.platform-cache.outputs.platforms}} permissions: packages: write contents: read tests: name: Tests - needs: [platforms, build-base] - if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' + needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: ${{fromJSON(needs.platforms.outputs.platform-matrix)}} + platform: ${{fromJSON(needs.platform-cache.outputs.platform-list)}} release: [3.1, 3.2, 4.0] runs-on: ["self-hosted", "enf-x86-lowtier"] container: - image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/platforms.yaml b/.github/workflows/platforms.yaml deleted file mode 100644 index e44ce0ffda..0000000000 --- a/.github/workflows/platforms.yaml +++ /dev/null @@ -1,88 +0,0 @@ -name: "Platforms" - -on: - workflow_call: - inputs: - override-build-matrix: - description: 'Override build matrix' - type: string - required: false - outputs: - p: - description: "Discovered Build Platforms" - value: ${{ jobs.d.outputs.p }} - platform-matrix: - description: "Overridden Platform Matrix" - value: ${{ jobs.pm.outputs.platform-matrix }} - -permissions: - packages: read - contents: read - -defaults: - run: - shell: bash - -jobs: - d: - name: Discover Platforms - runs-on: ubuntu-latest - outputs: - missing-platforms: ${{steps.discover.outputs.missing-platforms}} - p: ${{steps.discover.outputs.platforms}} - steps: - - name: Discover Platforms - id: discover - uses: AntelopeIO/discover-platforms-action@v1 - with: - platform-file: .cicd/platforms.json - password: ${{secrets.GITHUB_TOKEN}} - package-name: builders - - build-platforms: - name: Build Platforms - needs: d - if: needs.d.outputs.missing-platforms != '[]' - strategy: - fail-fast: false - matrix: - platform: ${{fromJSON(needs.d.outputs.missing-platforms)}} - runs-on: ["self-hosted", "enf-x86-beefy"] - permissions: - packages: write - contents: read - steps: - - name: Login to Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{github.repository_owner}} - password: ${{secrets.GITHUB_TOKEN}} - - name: Build and push - uses: docker/build-push-action@v3 - with: - push: true - tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}} - file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}} - - pm: - name: Platform Matrix - needs: [d] - if: always() && needs.d.result == 'success' - runs-on: ubuntu-latest - outputs: - platform-matrix: ${{steps.pm-results.outputs.platform-matrix}} - steps: - - name: Parse Platform Matrix - id: parse-pm - uses: actions/github-script@v6 - with: - script: return Object.keys(${{needs.d.outputs.p}}) - - name: Check | Override result - id: pm-results - run: | - echo 'platform-matrix=${{steps.parse-pm.outputs.result}}' >> $GITHUB_OUTPUT - - if [[ "${{inputs.override-build-matrix}}" != "" ]]; then - echo 'platform-matrix=["${{inputs.override-build-matrix}}"]' >> $GITHUB_OUTPUT - fi From 4f4f5d3350723758b25554be37fb8e6d57f50b12 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 2 Oct 2023 18:17:55 -0500 Subject: [PATCH 07/12] GH-1662 Remove do_eof() as not needed --- .../eosio/http_plugin/beast_http_session.hpp | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp index 4ad2d928f0..f6c039c28c 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp @@ -297,7 +297,7 @@ class beast_http_session : public detail::abstract_conn, if(ec) { // See on_read comment below if(ec == http::error::end_of_stream || ec == asio::error::connection_reset) - return do_eof(); + return; // socket destructor will be called return fail(ec, "read_header", plugin_state_.get_logger(), "closing connection"); } @@ -337,7 +337,7 @@ class beast_http_session : public detail::abstract_conn, // on another read. If the client disconnects, we may get // http::error::end_of_stream or asio::error::connection_reset. if(ec == http::error::end_of_stream || ec == asio::error::connection_reset) - return do_eof(); + return; // socket destructor will be called return fail(ec, "read", plugin_state_.get_logger(), "closing connection"); } @@ -367,7 +367,7 @@ class beast_http_session : public detail::abstract_conn, if(close) { // This means we should close the connection, usually because // the response indicated the "Connection: close" semantic. - return do_eof(); + return; // socket destructor will be called } // create a new response object @@ -383,7 +383,7 @@ class beast_http_session : public detail::abstract_conn, case continue_state_t::reject: // request body too large. After issuing 401 response, close connection continue_state_ = continue_state_t::none; - do_eof(); + // socket destructor will be called break; default: @@ -450,7 +450,7 @@ class beast_http_session : public detail::abstract_conn, res_->set(http::field::server, BOOST_BEAST_VERSION_STRING); send_response(std::move(err_str), static_cast(http::status::internal_server_error)); - do_eof(); + // socket destructor will be called } } @@ -492,26 +492,12 @@ class beast_http_session : public detail::abstract_conn, void run_session() { if(auto error_str = verify_max_requests_in_flight(); !error_str.empty()) { send_busy_response(std::move(error_str)); - return do_eof(); + return; // socket destructor will be called } do_read_header(); } - void do_eof() { - is_send_exception_response_ = false; - try { - // Send a shutdown signal - beast::error_code ec; - socket_.shutdown(Socket::shutdown_both, ec); - socket_.close(ec); - // At this point the connection is closed gracefully - } catch(...) { - handle_exception(); - } - } - - bool allow_host(const http::request& req) { if constexpr(std::is_same_v) { const std::string host_str(req["host"]); From a56cf349db60a7559f5c39cac5a9c5718129ebb1 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Mon, 2 Oct 2023 19:39:16 -0400 Subject: [PATCH 08/12] extended leap-util logs for smoke test --- libraries/chain/block_log.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 082aeb02f6..b97008a9df 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -326,7 +326,7 @@ namespace eosio { namespace chain { block_log_data log_data; block_log_index log_index; - block_log_bundle(std::filesystem::path block_file, std::filesystem::path index_file) + block_log_bundle(std::filesystem::path block_file, std::filesystem::path index_file, bool validate_indx) : block_file_name(std::move(block_file)), index_file_name(std::move(index_file)) { log_data.open(block_file_name); @@ -335,6 +335,15 @@ namespace eosio { namespace chain { EOS_ASSERT(!log_data.get_preamble().is_currently_pruned(), block_log_unsupported_version, "Block log is currently in pruned format, it must be vacuumed before doing this operation"); + if (validate_indx) + validate_index(); + } + + explicit block_log_bundle(const std::filesystem::path& block_dir, bool validate_index=true) + : block_log_bundle(block_dir / "blocks.log", block_dir / "blocks.index", validate_index) {} + + // throws if not valid + void validate_index() { uint32_t log_num_blocks = log_data.num_blocks(); uint32_t index_num_blocks = log_index.num_blocks(); @@ -345,9 +354,6 @@ namespace eosio { namespace chain { ("block_file_name", block_file_name)("log_num_blocks", log_num_blocks)( "index_num_blocks", index_num_blocks)("index_file_name", index_file_name)); } - - explicit block_log_bundle(const std::filesystem::path& block_dir) - : block_log_bundle(block_dir / "blocks.log", block_dir / "blocks.index") {} }; /// Used to traverse the block position (i.e. the last 8 bytes in each block log entry) of the blocks.log file @@ -1572,12 +1578,14 @@ namespace eosio { namespace chain { // static void block_log::smoke_test(const std::filesystem::path& block_dir, uint32_t interval) { - block_log_bundle log_bundle(block_dir); + block_log_bundle log_bundle(block_dir, false); ilog("block log version= ${version}",("version", log_bundle.log_data.version())); ilog("first block= ${first}",("first", log_bundle.log_data.first_block_num())); ilog("last block= ${last}",("last", log_bundle.log_data.last_block_num())); + log_bundle.validate_index(); + ilog("blocks.log and blocks.index agree on number of blocks"); if (interval == 0) { From 844513763c2ddc7b3dd9e5de91162a455ae115f8 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:59:12 -0400 Subject: [PATCH 09/12] remove reference to reproducible.Dockerfile that isn't on this branch yet --- .github/workflows/ph_backward_compatibility.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index c5f0a5a277..9fde864bd3 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -20,9 +20,7 @@ jobs: packages: write with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: | - .cicd/platforms - tools/reproducible.Dockerfile:builder + platform-files: .cicd/platforms build-base: name: Run Build Workflow From 381b3ed0e9f86dd349e9c5fcb391c6bf01f46ef9 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 2 Oct 2023 23:28:26 -0400 Subject: [PATCH 10/12] fix platform-list in PHBC workflow --- .github/workflows/ph_backward_compatibility.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index 9fde864bd3..4633108821 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -28,7 +28,7 @@ jobs: needs: [platform-cache] with: platforms: ${{needs.platform-cache.outputs.platforms}} - platform-list: ${{needs.platform-cache.outputs.platforms}} + platform-list: ${{needs.platform-cache.outputs.platform-list}} permissions: packages: write contents: read From 447813121e1407dbdaf697019e8ffa3606ceb287 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 3 Oct 2023 08:22:47 -0500 Subject: [PATCH 11/12] GH-1705 Advance read pointer when dropping trx. Also log at debug as nothing wrong. --- plugins/net_plugin/net_plugin.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 13166ce0bf..807660bd6f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3025,7 +3025,8 @@ namespace eosio { return true; } if (my_impl->sync_master->syncing_from_peer()) { - peer_wlog(this, "syncing, dropping trx"); + peer_dlog(this, "syncing, dropping trx"); + pending_message_buffer.advance_read_ptr( message_length ); return true; } From 4c45ac017024ad6d65a9ec9ca15215499e26d448 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 3 Oct 2023 09:43:41 -0500 Subject: [PATCH 12/12] Revert "GH-1662 Remove do_eof() as not needed" This reverts commit 4f4f5d3350723758b25554be37fb8e6d57f50b12. --- .../eosio/http_plugin/beast_http_session.hpp | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp index f6c039c28c..4ad2d928f0 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp @@ -297,7 +297,7 @@ class beast_http_session : public detail::abstract_conn, if(ec) { // See on_read comment below if(ec == http::error::end_of_stream || ec == asio::error::connection_reset) - return; // socket destructor will be called + return do_eof(); return fail(ec, "read_header", plugin_state_.get_logger(), "closing connection"); } @@ -337,7 +337,7 @@ class beast_http_session : public detail::abstract_conn, // on another read. If the client disconnects, we may get // http::error::end_of_stream or asio::error::connection_reset. if(ec == http::error::end_of_stream || ec == asio::error::connection_reset) - return; // socket destructor will be called + return do_eof(); return fail(ec, "read", plugin_state_.get_logger(), "closing connection"); } @@ -367,7 +367,7 @@ class beast_http_session : public detail::abstract_conn, if(close) { // This means we should close the connection, usually because // the response indicated the "Connection: close" semantic. - return; // socket destructor will be called + return do_eof(); } // create a new response object @@ -383,7 +383,7 @@ class beast_http_session : public detail::abstract_conn, case continue_state_t::reject: // request body too large. After issuing 401 response, close connection continue_state_ = continue_state_t::none; - // socket destructor will be called + do_eof(); break; default: @@ -450,7 +450,7 @@ class beast_http_session : public detail::abstract_conn, res_->set(http::field::server, BOOST_BEAST_VERSION_STRING); send_response(std::move(err_str), static_cast(http::status::internal_server_error)); - // socket destructor will be called + do_eof(); } } @@ -492,12 +492,26 @@ class beast_http_session : public detail::abstract_conn, void run_session() { if(auto error_str = verify_max_requests_in_flight(); !error_str.empty()) { send_busy_response(std::move(error_str)); - return; // socket destructor will be called + return do_eof(); } do_read_header(); } + void do_eof() { + is_send_exception_response_ = false; + try { + // Send a shutdown signal + beast::error_code ec; + socket_.shutdown(Socket::shutdown_both, ec); + socket_.close(ec); + // At this point the connection is closed gracefully + } catch(...) { + handle_exception(); + } + } + + bool allow_host(const http::request& req) { if constexpr(std::is_same_v) { const std::string host_str(req["host"]);