From e25b3d09e4eda05f58c1e4a6421d6c5b123724d5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 5 Jul 2018 09:45:34 -0500 Subject: [PATCH 1/5] Increase --abi-serializer-max-time-ms for slow test machines --- tests/testUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testUtils.py b/tests/testUtils.py index 40e98672e77..0fadc8b45f8 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1492,7 +1492,7 @@ def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", p2pPlugin="ne if self.staging: cmdArr.append("--nogen") - nodeosArgs="--max-transaction-time 5000 --filter-on * --p2p-max-nodes-per-host %d" % (totalNodes) + nodeosArgs="--max-transaction-time 5000 --abi-serializer-max-time-ms 5000 --filter-on * --p2p-max-nodes-per-host %d" % (totalNodes) if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" if self.enableMongo: From c20903d781487dce7632f5b4ec0e25b3eedbad92 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Thu, 5 Jul 2018 11:35:22 -0400 Subject: [PATCH 2/5] Remove rouge '0' from help text --- plugins/net_plugin/net_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index be2fa347cb1..e34d71a18a6 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2913,7 +2913,7 @@ namespace eosio { ( "p2p-listen-endpoint", bpo::value()->default_value( "0.0.0.0:9876" ), "The actual host:port used to listen for incoming p2p connections.") ( "p2p-server-address", bpo::value(), "An externally accessible host:port for identifying this node. Defaults to p2p-listen-endpoint.") ( "p2p-peer-address", bpo::value< vector >()->composing(), "The public endpoint of a peer node to connect to. Use multiple p2p-peer-address options as needed to compose a network.") - ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client0nodes from any single IP address") + ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "agent-name", bpo::value()->default_value("\"EOS Test Agent\""), "The name supplied to identify this node amongst the peers.") ( "allowed-connection", bpo::value>()->multitoken()->default_value({"any"}, "any"), "Can be 'any' or 'producers' or 'specified' or 'none'. If 'specified', peer-key must be specified at least once. If only 'producers', peer-key is not required. 'producers' and 'specified' may be combined.") ( "peer-key", bpo::value>()->composing()->multitoken(), "Optional public key of peer allowed to connect. May be used multiple times.") From 1a0a1fb3f1aec25b0647cadb5add8d5674819577 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 5 Jul 2018 10:40:32 -0500 Subject: [PATCH 3/5] 0.1s not enough time for slow darwin test machine --- libraries/testing/tester.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 8dd189c4cc1..262dba0d62f 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -91,7 +91,7 @@ namespace eosio { namespace testing { cfg.genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000"); cfg.genesis.initial_key = get_public_key( config::system_account_name, "active" ); - abi_serializer::set_max_serialization_time(fc::microseconds(100*1000)); // 100ms for slow test machines + abi_serializer::set_max_serialization_time(fc::seconds(1)); // 1s for slow test machines for(int i = 0; i < boost::unit_test::framework::master_test_suite().argc; ++i) { if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--binaryen")) From 0a29de7a896f5ca45cdba1f218d4e48d4aefa417 Mon Sep 17 00:00:00 2001 From: Bart Wyatt Date: Tue, 10 Jul 2018 18:19:37 -0400 Subject: [PATCH 4/5] consolidated security omnibus for 1.0.9 - Handle running out of fds on bnet accept - add basic validation to http `Host` headers on incoming requests * the header must exactly match an expected server[:port] * missing ports are assumed defaulted based on scheme to 80/443 - add new configuration option `http-alias` to add additional acceptable hosts * the host:port present in the http(s) addresses is automatically acceptable but must be exact (localhost === localhost, localhost !== 127.0.0.1, etc) - add new configuration option `http-validate-host` which defaults to true, if false these checks are not performed and any `Host` header is acceptable - correct cleos behavior which was not sending correct `Host` headers when the urls indicated non-default ports Co-authored-by: Bart Wyatt Co-authored-by: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> --- plugins/bnet_plugin/bnet_plugin.cpp | 20 +++++++++--- plugins/http_plugin/http_plugin.cpp | 49 +++++++++++++++++++++++++++-- programs/cleos/httpc.cpp | 15 ++++++++- programs/eosio-launcher/main.cpp | 1 + tests/p2p_tests/pump/run_test.pl | 3 +- tests/testUtils.py | 2 +- tests/validate-dirty-db.py | 2 +- 7 files changed, 82 insertions(+), 10 deletions(-) diff --git a/plugins/bnet_plugin/bnet_plugin.cpp b/plugins/bnet_plugin/bnet_plugin.cpp index 0f10bcbcc24..c1a76715279 100644 --- a/plugins/bnet_plugin/bnet_plugin.cpp +++ b/plugins/bnet_plugin/bnet_plugin.cpp @@ -1283,12 +1283,24 @@ namespace eosio { void listener::on_accept( boost::system::error_code ec ) { if( ec ) { + if( ec == boost::system::errc::too_many_files_open ) + do_accept(); return; } - auto newsession = std::make_shared( move( _socket ), _net_plugin ); - _net_plugin->async_add_session( newsession ); - newsession->_local_peer_id = _net_plugin->_peer_id; - newsession->run(); + std::shared_ptr newsession; + try { + newsession = std::make_shared( move( _socket ), _net_plugin ); + } + catch( std::exception& e ) { + //making a session creates an instance of std::random_device which may open /dev/urandom + // for example. Unfortuately the only defined error is a std::exception derivative + _socket.close(); + } + if( newsession ) { + _net_plugin->async_add_session( newsession ); + newsession->_local_peer_id = _net_plugin->_peer_id; + newsession->run(); + } do_accept(); } diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index 6d06a6ce8ee..98105858bd7 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace eosio { @@ -31,7 +32,10 @@ namespace eosio { namespace asio = boost::asio; using std::map; + using std::vector; + using std::set; using std::string; + using std::regex; using boost::optional; using boost::asio::ip::tcp; using std::shared_ptr; @@ -99,6 +103,28 @@ namespace eosio { websocket_server_tls_type https_server; + bool validate_host; + set valid_hosts; + + bool host_port_is_valid( const std::string& host_port ) { + return !validate_host || valid_hosts.find(host_port) != valid_hosts.end(); + } + + bool host_is_valid( const std::string& host, bool secure) { + if (!validate_host) { + return true; + } + + // normalise the incoming host so that it always has the explicit port + static auto has_port_expr = regex("[^:]:[0-9]+$"); /// ends in : without a preceeding colon which implies ipv6 + if (std::regex_search(host, has_port_expr)) { + return host_port_is_valid( host ); + } else { + // according to RFC 2732 ipv6 addresses should always be enclosed with brackets so we shouldn't need to special case here + return host_port_is_valid( host + ":" + std::to_string(secure ? websocketpp::uri_default_secure_port : websocketpp::uri_default_port )); + } + } + ssl_context_ptr on_tls_init(websocketpp::connection_hdl hdl) { ssl_context_ptr ctx = websocketpp::lib::make_shared(asio::ssl::context::sslv23_server); @@ -169,6 +195,13 @@ namespace eosio { template void handle_http_request(typename websocketpp::server>::connection_ptr con) { try { + auto& req = con->get_request(); + const auto& host_str = req.get_header("Host"); + if (host_str.empty() || !host_is_valid(host_str, con->get_uri()->get_secure())) { + con->set_status(websocketpp::http::status_code::bad_request); + return; + } + if( !access_control_allow_origin.empty()) { con->append_header( "Access-Control-Allow-Origin", access_control_allow_origin ); } @@ -181,8 +214,7 @@ namespace eosio { if( access_control_allow_credentials ) { con->append_header( "Access-Control-Allow-Credentials", "true" ); } - - auto& req = con->get_request(); + if(req.get_method() == "OPTIONS") { con->set_status(websocketpp::http::status_code::ok); return; @@ -230,6 +262,7 @@ namespace eosio { elog("error thrown from http io service"); } } + }; http_plugin::http_plugin():my(new http_plugin_impl()){} @@ -275,10 +308,18 @@ namespace eosio { "Specify if Access-Control-Allow-Credentials: true should be returned on each request.") ("max-body-size", bpo::value()->default_value(1024*1024), "The maximum body size in bytes allowed for incoming RPC requests") ("verbose-http-errors", bpo::bool_switch()->default_value(false), "Append the error log to HTTP responses") + ("http-validate-host", boost::program_options::value()->default_value(true), "If set to false, then any incoming \"Host\" header is considered valid") + ("http-alias", bpo::value>()->composing(), "Additionaly acceptable values for the \"Host\" header of incoming HTTP requests, can be specified multiple times. Includes http/s_server_address by default.") ; } void http_plugin::plugin_initialize(const variables_map& options) { + my->validate_host = options.at("http-validate-host").as(); + if( options.count( "http-alias" )) { + const auto& aliases = options["http-alias"].as>(); + my->valid_hosts.insert(aliases.begin(), aliases.end()); + } + tcp::resolver resolver(app().get_io_service()); if(options.count("http-server-address") && options.at("http-server-address").as().length()) { string lipstr = options.at("http-server-address").as(); @@ -291,6 +332,8 @@ namespace eosio { } catch(const boost::system::system_error& ec) { elog("failed to configure http to listen on ${h}:${p} (${m})", ("h",host)("p",port)("m", ec.what())); } + + my->valid_hosts.emplace(lipstr); } if(options.count("https-server-address") && options.at("https-server-address").as().length()) { @@ -315,6 +358,8 @@ namespace eosio { } catch(const boost::system::system_error& ec) { elog("failed to configure https to listen on ${h}:${p} (${m})", ("h",host)("p",port)("m", ec.what())); } + + my->valid_hosts.emplace(lipstr); } my->max_body_size = options.at("max-body-size").as(); diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 93b60bef787..5a3443d7213 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -150,6 +150,18 @@ namespace eosio { namespace client { namespace http { return resolved_url(url, std::move(resolved_addresses), *resolved_port, is_loopback); } + string format_host_header(const resolved_url& url) { + // common practice is to only make the port explicit when it is the non-default port + if ( + (url.scheme == "https" && url.resolved_port == 443) || + (url.scheme == "http" && url.resolved_port == 80) + ) { + return url.server; + } else { + return url.server + ":" + url.port; + } + } + fc::variant do_http_call( const connection_param& cp, const fc::variant& postdata, bool print_request, @@ -163,8 +175,9 @@ namespace eosio { namespace client { namespace http { boost::asio::streambuf request; std::ostream request_stream(&request); + auto host_header_value = format_host_header(url); request_stream << "POST " << url.path << " HTTP/1.0\r\n"; - request_stream << "Host: " << url.server << "\r\n"; + request_stream << "Host: " << host_header_value << "\r\n"; request_stream << "content-length: " << postjson.size() << "\r\n"; request_stream << "Accept: */*\r\n"; request_stream << "Connection: close\r\n"; diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index d9681d1a78d..139b3e5723b 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -1026,6 +1026,7 @@ launcher_def::write_config_file (tn_node_def &node) { cfg << "readonly = 0\n"; cfg << "send-whole-blocks = true\n"; cfg << "http-server-address = " << host->host_name << ":" << instance.http_port << "\n"; + cfg << "http-validate-host = false\n"; if (p2p == p2p_plugin::NET) { cfg << "p2p-listen-endpoint = " << host->listen_addr << ":" << instance.p2p_port << "\n"; cfg << "p2p-server-address = " << host->public_name << ":" << instance.p2p_port << "\n"; diff --git a/tests/p2p_tests/pump/run_test.pl b/tests/p2p_tests/pump/run_test.pl index 77ef2e8a779..11e467a5d2e 100755 --- a/tests/p2p_tests/pump/run_test.pl +++ b/tests/p2p_tests/pump/run_test.pl @@ -202,7 +202,8 @@ sub launch_nodes { my @cmdline = ($eosd, $gtsarg, "--data-dir=$data_dir[$i]", - "--verbose-http-errors"); + "--verbose-http-errors", + "--http-validate-host=false"); $pid[$i] = fork; if ($pid[$i] > 0) { my $pause = $i == 0 ? $first_pause : $launch_pause; diff --git a/tests/testUtils.py b/tests/testUtils.py index 0fadc8b45f8..79e5a84ad7c 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1232,7 +1232,7 @@ def launch(self): Utils.Print("ERROR: Wallet Manager wasn't configured to launch keosd") return False - cmd="%s --data-dir %s --config-dir %s --http-server-address=%s:%d --verbose-http-errors" % ( + cmd="%s --data-dir %s --config-dir %s --http-server-address=%s:%d --verbose-http-errors --http-validate-host=false" % ( Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) with open(WalletMgr.__walletLogFile, 'w') as sout, open(WalletMgr.__walletLogFile, 'w') as serr: diff --git a/tests/validate-dirty-db.py b/tests/validate-dirty-db.py index 9ca12543187..2d790cd7821 100755 --- a/tests/validate-dirty-db.py +++ b/tests/validate-dirty-db.py @@ -40,7 +40,7 @@ def errorExit(msg="", errorCode=1): def runNodeosAndGetOutput(myNodeId, myTimeout=3): """Startup nodeos, wait for timeout (before forced shutdown) and collect output. Stdout, stderr and return code are returned in a dictionary.""" Print("Launching nodeos process id: %d" % (myNodeId)) - cmd="programs/nodeos/nodeos --config-dir etc/eosio/node_bios --data-dir var/lib/node_bios --verbose-http-errors" + cmd="programs/nodeos/nodeos --config-dir etc/eosio/node_bios --data-dir var/lib/node_bios --verbose-http-errors --http-validate-host=false" Print("cmd: %s" % (cmd)) proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 4e3d05a349ecc878ed499dccd213aaaa0e1a2fa1 Mon Sep 17 00:00:00 2001 From: Bart Wyatt Date: Tue, 10 Jul 2018 22:45:22 -0400 Subject: [PATCH 5/5] bump verison to 1.0.9 --- CMakeLists.txt | 2 +- Docker/README.md | 6 +++--- Docker/docker-compose-eosio1.0.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3cee609e7a..67272ade5a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 0) -set(VERSION_PATCH 8) +set(VERSION_PATCH 9) set( CLI_CLIENT_EXECUTABLE_NAME cleos ) set( GUI_CLIENT_EXECUTABLE_NAME eosio ) diff --git a/Docker/README.md b/Docker/README.md index d7ccde53b17..b052b98ca53 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd eos/Docker docker build . -t eosio/eos ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v1.0.8 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v1.0.9 tag, you could do the following: ```bash -docker build -t eosio/eos:v1.0.8 --build-arg branch=v1.0.8 . +docker build -t eosio/eos:v1.0.9 --build-arg branch=v1.0.9 . ``` By default, the symbol in eosio.system is set to SYS. You can override this using the symbol argument while building the docker image. @@ -181,7 +181,7 @@ Note: if you want to use the mongo db plugin, you have to enable it in your `dat ``` # pull images -docker pull eosio/eos:v1.0.8 +docker pull eosio/eos:v1.0.9 # create volume docker volume create --name=nodeos-data-volume diff --git a/Docker/docker-compose-eosio1.0.yaml b/Docker/docker-compose-eosio1.0.yaml index 0d398e2d77c..1a50874ae45 100644 --- a/Docker/docker-compose-eosio1.0.yaml +++ b/Docker/docker-compose-eosio1.0.yaml @@ -2,7 +2,7 @@ version: "3" services: nodeosd: - image: eosio/eos:v1.0.8 + image: eosio/eos:v1.0.9 command: /opt/eosio/bin/nodeosd.sh --data-dir /opt/eosio/bin/data-dir -e hostname: nodeosd ports: @@ -14,7 +14,7 @@ services: - nodeos-data-volume:/opt/eosio/bin/data-dir keosd: - image: eosio/eos:v1.0.8 + image: eosio/eos:v1.0.9 command: /opt/eosio/bin/keosd --wallet-dir /opt/eosio/bin/data-dir --http-server-address=127.0.0.1:8900 hostname: keosd links: