From 53095bacf345b6df01225ffd74818de0889ea79b Mon Sep 17 00:00:00 2001 From: Bob Chen Date: Tue, 2 Apr 2024 09:24:16 +0800 Subject: [PATCH 1/6] httpfs add error log (#436) --- fs/httpfs/httpfs_v2.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/httpfs/httpfs_v2.cpp b/fs/httpfs/httpfs_v2.cpp index 57e07072..3a0b3fc8 100644 --- a/fs/httpfs/httpfs_v2.cpp +++ b/fs/httpfs/httpfs_v2.cpp @@ -206,6 +206,9 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile { } m_authorized = true; ret = op.resp.readv(iovec, iovcnt); + if (ret < 0) { + LOG_ERROR("HttpFs: read body failed, ", ERRNO()); + } return ret; } From b5f9430621aacc873abb4f3217fd8e40fcf117db Mon Sep 17 00:00:00 2001 From: Coldwings Date: Wed, 3 Apr 2024 16:38:56 +0800 Subject: [PATCH 2/6] Remove some outdated libcurl option usage in curl wrapper header (#440) (#443) --- net/curl.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/net/curl.h b/net/curl.h index 37fee819..8a4a4db5 100644 --- a/net/curl.h +++ b/net/curl.h @@ -240,7 +240,6 @@ class cURL { return; } setopt(CURLOPT_ERRORBUFFER, m_errmsg); - setopt(CURLOPT_DNS_USE_GLOBAL_CACHE, 0L); setopt(CURLOPT_NOSIGNAL, 1L); setopt(CURLOPT_TCP_NODELAY, 1L); m_errmsg[0] = '\0'; @@ -413,7 +412,6 @@ class cURL { set_read_stream(rstream); set_write_stream(wstream); setopt(CURLOPT_UPLOAD, 1L); - setopt(CURLOPT_PUT, 1L); setopt(CURLOPT_URL, url); setopt(CURLOPT_HTTPHEADER, headers.list); #if LIBCURL_VERSION_MAJOR > 7 || LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 37 From 5cbfb92987c6a018ab322c8eb2626e93fe0d7aa3 Mon Sep 17 00:00:00 2001 From: Bob Chen Date: Wed, 10 Apr 2024 20:55:13 +0800 Subject: [PATCH 3/6] zlib and uring source build didn't have -O3 (#451) --- CMake/build-from-src.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake index abfcbe1b..caeb2598 100644 --- a/CMake/build-from-src.cmake +++ b/CMake/build-from-src.cmake @@ -27,7 +27,7 @@ function(build_from_src [dep]) URL_MD5 9b8aa094c4e5765dabf4da391f00d15c UPDATE_DISCONNECTED ON BUILD_IN_SOURCE ON - CONFIGURE_COMMAND CFLAGS=-fPIC ./configure --prefix=${BINARY_DIR} --static + CONFIGURE_COMMAND sh -c "CFLAGS=\"-fPIC -O3\" ./configure --prefix=${BINARY_DIR} --static" BUILD_COMMAND $(MAKE) INSTALL_COMMAND $(MAKE) install ) @@ -43,7 +43,7 @@ function(build_from_src [dep]) UPDATE_DISCONNECTED ON BUILD_IN_SOURCE ON CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR} - BUILD_COMMAND V=1 CFLAGS=-fPIC $(MAKE) -C src + BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -g -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src" INSTALL_COMMAND $(MAKE) install ) set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) From cd03fc000465a5e6c0226ea673a0ac04871b509d Mon Sep 17 00:00:00 2001 From: Lanzheng Liu Date: Thu, 18 Apr 2024 17:41:27 +0800 Subject: [PATCH 4/6] fix dns discard (#459) Signed-off-by: liulanzheng --- net/utils.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/net/utils.cpp b/net/utils.cpp index f90e49a0..ed514ed9 100644 --- a/net/utils.cpp +++ b/net/utils.cpp @@ -312,6 +312,7 @@ class DefaultResolver : public Resolver { break; } } + ipaddr.recycle(ipaddr->empty()); } } From 76e032dbe8a657af2e7ad4149fe9798a07f2a275 Mon Sep 17 00:00:00 2001 From: Bob Chen Date: Fri, 19 Apr 2024 21:50:15 +0800 Subject: [PATCH 5/6] http client support uds (#461) --- fs/test/CMakeLists.txt | 6 +-- net/http/client.cpp | 22 +++++++++-- net/http/client.h | 15 ++++++-- net/http/test/client_function_test.cpp | 52 +++++++++++++++++++++++++- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/fs/test/CMakeLists.txt b/fs/test/CMakeLists.txt index 6eb5b9c5..fb7421b1 100644 --- a/fs/test/CMakeLists.txt +++ b/fs/test/CMakeLists.txt @@ -12,6 +12,6 @@ add_executable(test-filecopy test_filecopy.cpp) target_link_libraries(test-filecopy PRIVATE photon_shared) add_test(NAME test-filecopy COMMAND $) -add_executable(test-throttled test_throttledfile.cpp) -target_link_libraries(test-throttled PRIVATE photon_shared) -add_test(NAME test-throttled COMMAND $) \ No newline at end of file +add_executable(test-throttle-file test_throttledfile.cpp) +target_link_libraries(test-throttle-file PRIVATE photon_shared) +add_test(NAME test-throttle-file COMMAND $) \ No newline at end of file diff --git a/net/http/client.cpp b/net/http/client.cpp index 23ea5c25..802eb873 100644 --- a/net/http/client.cpp +++ b/net/http/client.cpp @@ -38,6 +38,7 @@ class PooledDialer { bool tls_ctx_ownership; std::unique_ptr tcpsock; std::unique_ptr tlssock; + std::unique_ptr udssock; std::unique_ptr resolver; //etsocket seems not support multi thread very well, use tcp_socket now. need to find out why @@ -49,6 +50,7 @@ class PooledDialer { auto tls_cli = new_tls_client(tls_ctx, new_tcp_socket_client(), true); tcpsock.reset(new_tcp_socket_pool(tcp_cli, -1, true)); tlssock.reset(new_tcp_socket_pool(tls_cli, -1, true)); + udssock.reset(new_uds_client()); } ~PooledDialer() { @@ -63,6 +65,8 @@ class PooledDialer { ISocketStream* dial(const T& x, uint64_t timeout = -1UL) { return dial(x.host_no_port(), x.port(), x.secure(), timeout); } + + ISocketStream* dial(std::string_view uds_path, uint64_t timeout = -1UL); }; ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool secure, uint64_t timeout) { @@ -96,6 +100,14 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec return nullptr; } +ISocketStream* PooledDialer::dial(std::string_view uds_path, uint64_t timeout) { + udssock->timeout(timeout); + auto stream = udssock->connect(uds_path.data()); + if (!stream) + LOG_ERRNO_RETURN(0, nullptr, "failed to dial to unix socket `", uds_path); + return stream; +} + constexpr uint64_t code3xx() { return 0; } template constexpr uint64_t code3xx(uint64_t x, Ts...xs) @@ -164,9 +176,13 @@ class ClientImpl : public Client { if (tmo.timeout() == 0) LOG_ERROR_RETURN(ETIMEDOUT, ROUNDTRIP_FAILED, "connection timedout"); auto &req = op->req; - auto s = (m_proxy && !m_proxy_url.empty()) - ? m_dialer.dial(m_proxy_url, tmo.timeout()) - : m_dialer.dial(req, tmo.timeout()); + ISocketStream* s; + if (m_proxy && !m_proxy_url.empty()) + s = m_dialer.dial(m_proxy_url, tmo.timeout()); + else if (!op->uds_path.empty()) + s = m_dialer.dial(op->uds_path, tmo.timeout()); + else + s = m_dialer.dial(req, tmo.timeout()); if (!s) { if (errno == ECONNREFUSED || errno == ENOENT) { LOG_ERROR_RETURN(0, ROUNDTRIP_FAST_RETRY, "connection refused") diff --git a/net/http/client.h b/net/http/client.h index 350766e3..c1a80cb3 100644 --- a/net/http/client.h +++ b/net/http/client.h @@ -48,7 +48,9 @@ class Client : public Object { uint16_t retry = 5; // default retry: 5 at most Response resp; // response int status_code = -1; // status code in response - bool enable_proxy; + bool enable_proxy = false; + std::string_view uds_path; // If set, Unix Domain Socket will be used instead of TCP. + // URL should still be the format of http://localhost/xxx IStream* body_stream = nullptr; // use body_stream as body using BodyWriter = Delegate; // or call body_writer if body_stream BodyWriter body_writer = {}; // is not set @@ -68,6 +70,11 @@ class Client : public Object { if (!_client) return -1; return _client->call(this); } + int call(std::string_view unix_socket_path) { + if (!_client) return -1; + uds_path = unix_socket_path; + return _client->call(this); + } protected: Client* _client; @@ -83,15 +90,15 @@ class Client : public Object { Operation(uint16_t buf_size) : req(_buf, buf_size) {} }; - Operation* new_operation(Verb v, std::string_view url, uint16_t buf_size = 64 * 1024 - 1) { + Operation* new_operation(Verb v, std::string_view url, uint16_t buf_size = UINT16_MAX) { return Operation::create(this, v, url, buf_size); } - Operation* new_operation(uint16_t buf_size = 64 * 1024 - 1) { + Operation* new_operation(uint16_t buf_size = UINT16_MAX) { return Operation::create(this, buf_size); } - template + template class OperationOnStack : public Operation { char _buf[BufferSize]; public: diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp index 4d68d3c1..494fa249 100644 --- a/net/http/test/client_function_test.cpp +++ b/net/http/test/client_function_test.cpp @@ -61,6 +61,21 @@ int timeout_writer(void *self, IStream* stream) { return 0; } +class SimpleHandler : public http::HTTPHandler { +public: + int handle_request(http::Request& req, http::Response& resp, std::string_view) { + std::string url_path(req.target()); + resp.set_result(200); + resp.headers.content_length(url_path.size()); + resp.headers.insert("Content-Type", "application/octet-stream"); + auto n = resp.write(url_path.data(), url_path.size()); + if (n != (ssize_t) url_path.size()) { + LOG_ERRNO_RETURN(0, -1, "send body failed"); + } + return 0; + } +}; + TEST(http_client, get) { system("mkdir -p /tmp/ease_ut/http_test/"); system("echo \"this is a http_client request body text for socket stream\" > /tmp/ease_ut/http_test/ease-httpclient-gettestfile"); @@ -523,6 +538,41 @@ TEST(DISABLED_http_client, ipv6) { // make sure runing in a ipv6-ready environm EXPECT_EQ(200, op->resp.status_code()); } +TEST(http_client, unix_socket) { + const char* uds_path = "test-http-client.sock"; + + auto http_server = new_http_server(); + DEFER(delete http_server); + http_server->add_handler(new SimpleHandler, true, "/simple-api"); + + auto socket_server = new_uds_server(true); + DEFER(delete socket_server); + + socket_server->set_handler(http_server->get_connection_handler()); + ASSERT_EQ(0, socket_server->bind(uds_path)); + ASSERT_EQ(0, socket_server->listen()); + ASSERT_EQ(0, socket_server->start_loop(false)); + + auto client = new_http_client(); + DEFER(delete client); + + Client::OperationOnStack<> op(client, Verb::GET, "http://localhost/simple-api"); + int ret = op.call(uds_path); + ASSERT_EQ(0, ret); + ASSERT_EQ(200, op.resp.status_code()); + + char buf[UINT16_MAX]; + ssize_t n = op.resp.read(buf, op.resp.body_size()); + ASSERT_EQ(n, (ssize_t) op.resp.body_size()); + LOG_INFO(buf); + + // A wrong hostname or HTTPS doesn't have effect on unix socket + Client::OperationOnStack<> op2(client, Verb::GET, "https://www.wrong.hostname/simple-api"); + ret = op2.call(uds_path); + ASSERT_EQ(0, ret); + ASSERT_EQ(200, op2.resp.status_code()); +} + TEST(url, url_escape_unescape) { EXPECT_EQ( url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"), @@ -582,5 +632,5 @@ int main(int argc, char** arg) { #endif set_log_output_level(ALOG_INFO); ::testing::InitGoogleTest(&argc, arg); - LOG_DEBUG("test result:`", RUN_ALL_TESTS()); + return RUN_ALL_TESTS(); } From 01a1864bf215bdbad1821168646bcb34cfc4a468 Mon Sep 17 00:00:00 2001 From: Bob Chen Date: Sun, 28 Apr 2024 11:03:24 +0800 Subject: [PATCH 6/6] epoll_wait allows to wait 0ms, instead of the previous 1ms (#466) --- io/epoll.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/io/epoll.cpp b/io/epoll.cpp index 7029606c..44296fed 100644 --- a/io/epoll.cpp +++ b/io/epoll.cpp @@ -203,10 +203,8 @@ ok: entry.interests |= eint; int do_epoll_wait(uint64_t timeout) { assert(_events_remain == 0); uint8_t cool_down_ms = 1; - // since timeout may less than 1ms - // in such condition, timeout_ms should be at least 1 - // or it may call epoll_wait without any idle - timeout = (timeout && timeout < 1024) ? 1 : timeout / 1024; + // The granularity of epoll_wait is milliseconds, but epoll_wait 0 finishes in microseconds + timeout /= 1000; while (_engine_fd > 0) { int ret = epoll_wait(_engine_fd, _events, LEN(_events), timeout); if (ret < 0) {