graft-project · AlexanderSuprunenko · Dec 11, 2018 · Dec 12, 2018 · Dec 12, 2018 · Dec 13, 2018
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,40 @@
 cmake_minimum_required(VERSION 3.10)
 
+if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
+   set(CMAKE_BUILD_TYPE RelWithDebInfo)
+endif(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
+
+if(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo)
+    message("==> The configuration is ${CMAKE_BUILD_TYPE}. Debug info will be extracted into separate files.")
+
+    function (add_executable _name)
+        _add_executable(${ARGV})
+
+        if (TARGET ${_name})
+            add_custom_command(TARGET ${_name} POST_BUILD
+                COMMAND echo "$<TARGET_FILE_NAME:${_name}>: extracting debug info"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> objcopy --only-keep-debug "$<TARGET_FILE_NAME:${_name}>" "$<TARGET_FILE_NAME:${_name}>.debug"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> strip --strip-debug --strip-unneeded "$<TARGET_FILE_NAME:${_name}>"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> objcopy --add-gnu-debuglink="$<TARGET_FILE_NAME:${_name}>.debug" "$<TARGET_FILE_NAME:${_name}>"
+            )
+        endif()
+    endfunction()
+
+    function (add_library _name _type)
+        _add_library(${ARGV})
+
+        if (TARGET ${_name} AND ${_type} STREQUAL SHARED)
+            add_custom_command(TARGET ${_name} POST_BUILD
+                COMMAND echo "$<TARGET_FILE_NAME:${_name}>: extracting debug info"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> objcopy --only-keep-debug "$<TARGET_FILE_NAME:${_name}>" "$<TARGET_FILE_NAME:${_name}>.debug"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> strip --strip-debug --strip-unneeded "$<TARGET_FILE_NAME:${_name}>"
+                COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:${_name}> objcopy --add-gnu-debuglink="$<TARGET_FILE_NAME:${_name}>.debug" "$<TARGET_FILE_NAME:${_name}>"
+            )
+        endif()
+    endfunction()
+
+endif(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo)
+
 project(graft_server)
 
 option(OPT_BUILD_TESTS "Build tests." OFF)
@@ -155,6 +190,7 @@ add_library(graft STATIC
     ${PROJECT_SOURCE_DIR}/src/lib/graft/mongoosex.cpp
     ${PROJECT_SOURCE_DIR}/src/lib/graft/router.cpp
     ${PROJECT_SOURCE_DIR}/src/lib/graft/task.cpp
+    ${PROJECT_SOURCE_DIR}/src/lib/graft/upstream_manager.cpp
     ${PROJECT_SOURCE_DIR}/modules/mongoose/mongoose.c
     ${PROJECT_SOURCE_DIR}/src/supernode/server.cpp
     ${PROJECT_SOURCE_DIR}/src/supernode/supernode.cpp
@@ -182,7 +218,7 @@ set_target_properties(graft PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/BUILD/libgraft
     )
 
-target_compile_definitions(graft PRIVATE MG_ENABLE_COAP=1 -DMONERO_DEFAULT_LOG_CATEGORY="supernode")
+target_compile_definitions(graft PRIVATE MG_ENABLE_SYNC_RESOLVER MG_ENABLE_COAP -DMONERO_DEFAULT_LOG_CATEGORY="supernode")
 if(ENABLE_SYSLOG)
     target_compile_definitions(graft PRIVATE -DELPP_SYSLOG)
 endif()
@@ -273,7 +309,7 @@ set_target_properties(supernode_common PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/BUILD/supernode_common
     )
 
-target_compile_definitions(supernode_common PRIVATE MG_ENABLE_COAP=1 -DMONERO_DEFAULT_LOG_CATEGORY="supernode")
+target_compile_definitions(supernode_common PRIVATE -DMONERO_DEFAULT_LOG_CATEGORY="supernode")
 if(ENABLE_SYSLOG)
     target_compile_definitions(supernode_common PRIVATE -DELPP_SYSLOG)
 endif()
@@ -369,7 +405,6 @@ if (OPT_BUILD_TESTS)
             )
 
 
-        target_compile_definitions(supernode_test PRIVATE MG_ENABLE_COAP=1)
         add_dependencies(supernode_test graft supernode_common googletest)
         set_target_properties(supernode_test PROPERTIES LINK_FLAGS "-Wl,-E")
         if(ENABLE_SYSLOG)

diff --git a/data/config.ini b/data/config.ini
@@ -1,4 +1,5 @@
 [cryptonode]
+;;rpc-address can be one of [upstream] values, like rpc-address=$default
 rpc-address=127.0.0.1:28681
 p2p-address=127.0.0.1:18980
 
@@ -44,13 +45,28 @@ requests-per-sec=100 ;; maximal amount of requests per second in the window, 0 t
 ban-ip-sec=300 ;; time duration in seconds to ban particular IP, 0 to ban forever
 
 [upstream]
-blah=https://127.0.0.1:8080
-walletnode=http://127.0.0.1:28694
-;format <name>=<uri>[,<max-active-connections>[,<keep-alive>[,<timeout-in-seconds>]]] [;; comment]
-;	where <keep-alive> - {true | false | 0 | 1} , false by default
-;	example:
-;wallet2=http://127.0.0.1:28694, 10, true, 2.55   ;; example
-;walletnode=http://127.0.0.1:28694,cntMax,true/false/1/0 always_open,timeout
+;;An entry in [upstream] section is a template with the following
+;;format:
+;;  <name>-uri=<uri>
+;;    - Upstream URI in the form of <protocol>://<host>:<port>/<end-point>.
+;;      Any part of the URI can be overwritten by the requester.
+;;  <name>-timeout=seconds
+;;    - optinal parameter, inactivity timeout in seconds,
+;;      [server] upstream-request-timeout by default
+;;  <name>-max-conns-upstream=count
+;;    - optinal parameter, maximum number of connections, associated with
+;;      the template. 0 by default, that means no restriction.
+;;  <name>-max-conns-address=count
+;;    - optinal parameter, is maximum number of connections, associated with
+;;      the template, for each host:port pair.
+;;      0 by default, that means no restriction.
+;;  <name>-keep-alive=[true|false]
+;;    - the parameter indicates the associated connections are kept opened
+;;      after receiving a response. false by default.
+;;
+default-uri=127.0.0.1:28681
+blah-uri=https://127.0.0.1:8080
+walletnode-uri=http://127.0.0.1:28694
 
 [graftlets]
 ;;dirs parameter, a list of directories to search graftlets separated by colons. If a directory is set relative it will be interpreted both relative to the current directory and relative to the executable location. By default, 'graftlets' directory will be used relative to the executable location.

diff --git a/graftlets/TestGraftlet.cpp b/graftlets/TestGraftlet.cpp
@@ -56,13 +56,19 @@ class TestGraftlet: public IGraftlet
         return graft::Status::Ok;
     }
 
+    std::string testRouting(const std::string& s)
+    {
+        return s;
+    }
+
     virtual void initOnce()
     {
 //        REGISTER_ACTION(TestGraftlet, testUndefined);
         REGISTER_ACTION(TestGraftlet, testInt1);
         REGISTER_ACTION(TestGraftlet, testInt2);
         REGISTER_ACTION(TestGraftlet, testString1);
         REGISTER_ACTION(TestGraftlet, testString2);
+        REGISTER_ACTION(TestGraftlet, testRouting);
 
         REGISTER_ENDPOINT("/URI/test/{id:[0-9]+}", METHOD_GET | METHOD_POST, TestGraftlet, testHandler);
         REGISTER_ENDPOINT("/URI/test1/{id:[0-9]+}", METHOD_GET | METHOD_POST, TestGraftlet, testHandler1);

diff --git a/include/lib/graft/GraftletLoader.h b/include/lib/graft/GraftletLoader.h
@@ -22,7 +22,9 @@
 
 #include "lib/graft/IGraftlet.h"
 #include "lib/graft/GraftletRegistry.h"
+#include "lib/graft/routing.h"
 #include "lib/graft/router.h"
+#include "lib/graft/common/utils.h"
 
 #include <map>
 #include <vector>
@@ -66,18 +68,9 @@ class GraftletHandlerT
     {
         ClsName_ cls;
         std::string method;
-        {
-            int pos = cls_method.find('.');
-            if(pos != std::string::npos)
-            {
-                cls = cls_method.substr(0, pos);
-                method = cls_method.substr(pos+1);
-            }
-            else
-            {
-                cls = cls_method;
-            }
-        }
+        bool res = graft::utils::split(cls_method, '.', cls, method);
+        if(!res) cls = cls_method;
+
         auto it = m_cls2any.find(cls);
         if(it == m_cls2any.end()) throw std::runtime_error("Cannot find graftlet class name:" + cls);
         std::shared_ptr<BaseT> concreteGraftlet = std::any_cast<std::shared_ptr<BaseT>>(it->second);
@@ -124,6 +117,18 @@ class GraftletLoader
         return getEndpointsT<IGraftlet>();
     }
 
+    template <class BaseT = IGraftlet>
+    std::string Routing(const std::string& gr_cls_method_ver, const std::string& value)
+    {
+        std::string_view grname;
+        std::string_view cls_method_ver;
+        graft::utils::split(gr_cls_method_ver, '.', grname, cls_method_ver);
+        if(grname.empty()) throw "cannot find graftlet name in " + gr_cls_method_ver;
+
+        GraftletHandlerT<BaseT> handler = buildAndResolveGraftletT<BaseT>( std::string(grname) );
+        return handler.template invoke<graft::RoutingFunctionSignature>( std::string(cls_method_ver), value );
+    }
+
     class DependencyGraph;
     friend class GraftletLoader::DependencyGraph;
 private:
@@ -263,7 +268,7 @@ class GraftletLoader
     std::map<DllName, std::tuple<boost::dll::shared_library, Version, DllPath, Dependencies>> m_name2lib;
     //dll name -> registry
     std::map<DllName, GraftletRegistry*> m_name2registries;
-    //dll (name, type_index of BaseT) -> (class name, any of BaseT)
+    //dll (name, type_index of BaseT) -> (class name -> any of BaseT)
     //it makes no sense to hold std::shared_ptr<IGraftlet> until the shared_ptr is returned from resolveGraftlet
     std::map< std::pair<DllName, std::type_index>, std::map<ClsName, std::any> > m_name2gls;
 };

diff --git a/include/lib/graft/common/utils.h b/include/lib/graft/common/utils.h
@@ -18,5 +18,8 @@ T random_number(T startRange, T endRange)
     return dist(mt);
 }
 
+bool split(const std::string_view& in, char delim, std::string_view& first, std::string_view& second);
+bool split(const std::string_view& in, char delim, std::string& first, std::string& second);
+
 }
 
diff --git a/include/lib/graft/connection.h b/include/lib/graft/connection.h
@@ -4,6 +4,8 @@
 #include "lib/graft/task.h"
 #include "lib/graft/blacklist.h"
 
+namespace graftlet { class GraftletLoader; }
+
 namespace graft {
 
 namespace details
@@ -72,11 +74,19 @@ class UpstreamSender : public SelfHolder<UpstreamSender>
         : m_bt(bt), m_onDone(onDone), m_keepAlive(true), m_connectioId(connectionId), m_upstream(upstream), m_timeout(timeout)
     { }
 
+    UpstreamSender(const BaseTaskPtr& bt, std::string error, std::function<void(UpstreamSender& uss)> onDoneError) : m_bt(bt)
+    {
+        setError(Status::Error, error);
+        onDoneError(*this);
+        releaseItself();
+    }
+
     BaseTaskPtr& getTask() { return m_bt; }
 
-    void send(TaskManager& manager, const std::string& uri);
+    void send(mg_mgr* mgr, int http_callback_port, const std::string& uri);
     Status getStatus() const { return m_status; }
     const std::string& getError() const { return m_error; }
+    size_t getRequestSize() const { return m_requestSize; }
 
     void ev_handler(mg_connection* upstream, int ev, void *ev_data);
 private:
@@ -94,14 +104,15 @@ class UpstreamSender : public SelfHolder<UpstreamSender>
     mg_connection* m_upstream = nullptr;
     Status m_status = Status::None;
     std::string m_error;
+    size_t m_requestSize = 0;
 };
 
 class ConnectionBase;
 
 class Looper final : public TaskManager
 {
 public:
-    Looper(const ConfigOpts& copts, ConnectionBase& connectionBase);
+    Looper(const ConfigOpts& copts, UpstreamManager& upstreamManager, ConnectionBase& connectionBase);
     virtual ~Looper();
 
     void serve();
@@ -160,6 +171,8 @@ class ConnectionManager
     Proto m_proto;
 };
 
+class UpstreamRoutingManager;
+
 class ConnectionBase final
 {
 public:
@@ -171,7 +184,7 @@ class ConnectionBase final
     void setSysInfoCounter(std::unique_ptr<SysInfoCounter>& counter);
     void createSystemInfoCounter();
     void loadBlacklist(const ConfigOpts& copts);
-    void createLooper(ConfigOpts& configOpts);
+    void createLooper(graftlet::GraftletLoader& graftletLoader,  ConfigOpts& configOpts);
     void initConnectionManagers();
     void bindConnectionManagers();
 
@@ -193,6 +206,7 @@ class ConnectionBase final
     std::unique_ptr<SysInfoCounter> m_sysInfo;
     std::atomic_bool m_looperReady{false};
     std::unique_ptr<Looper> m_looper;
+    std::unique_ptr<UpstreamManager> m_upstreamManager;
     std::map<ConnectionManager::Proto, std::unique_ptr<ConnectionManager>> m_conManagers;
 };
 

diff --git a/include/lib/graft/inout.h b/include/lib/graft/inout.h
@@ -204,14 +204,16 @@ namespace graft
          * Set uri, proto, host, port, path members if you need.
          * The function forms real URI substituting absent parts according to Config.ini.
          * It is public to be accessed from tests and other classes.
-         * \param default_uri - this parameter always comes from [cryptonode]rpc-address of Config.ini.
-         * \return
+         * \param default_uri - one of uri from lines in [upstream] of Config.ini.
+         * \param ip_port - output of resulting "ip:port".
+         * \param result_uri - output of resulting uri.
+         * \param resolve_cache - a cache with resolved host to ip map.
+         * \return true on success or false otherwise
          */
-        std::string makeUri(const std::string& default_uri) const;
+        bool makeUri(const std::string& default_uri, std::string& ip_port, std::string& result_uri, std::unordered_map<std::string,std::string>& resolve_cache) const;
 
         std::string port;
         std::string path;
-        static std::unordered_map<std::string, std::tuple<std::string,int,bool,double>> uri_substitutions;
     };
 
     class InHttp final : public InOutHttpBase

diff --git a/include/lib/graft/routing.h b/include/lib/graft/routing.h
@@ -0,0 +1,40 @@
+// Copyright (c) 2018, The Graft Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <string>
+#include <functional>
+
+namespace graft {
+
+using RoutingFunctionSignature = std::string (const std::string&);
+using RoutingFunction = std::function<RoutingFunctionSignature>;
+
+}//namespace graft
+
diff --git a/include/lib/graft/serveropts.h b/include/lib/graft/serveropts.h
@@ -2,6 +2,7 @@
 
 #include <string>
 #include <vector>
+#include <unordered_map>
 #include <cassert>
 
 namespace graft {
@@ -30,6 +31,8 @@ struct ConfigOpts
     std::vector<std::string> graftlet_dirs;
     int lru_timeout_ms;
     IPFilterOpts ipfilter;
+    std::unordered_map<std::string, std::tuple<std::string,int,bool,double>> uri_substitutions;
+    std::string default_uri_substitution_name;
 
     void check_asserts() const
     {

diff --git a/include/lib/graft/task.h b/include/lib/graft/task.h
@@ -38,6 +38,7 @@ namespace graft
 {
 extern std::string client_addr(mg_connection* client);
 extern std::string client_host(mg_connection* client);
+extern unsigned int port_from_uri(const std::string& uri);
 
 class UpstreamSender;
 class TaskManager;
@@ -192,7 +193,7 @@ class UpstreamManager;
 class TaskManager : private HandlerAPI
 {
 public:
-    TaskManager(const ConfigOpts& copts, SysInfoCounter& sysInfoCounter);
+    TaskManager(const ConfigOpts& copts, UpstreamManager& upstreamManager, SysInfoCounter& sysInfoCounter);
     virtual ~TaskManager();
     TaskManager(const TaskManager&) = delete;
     TaskManager& operator = (const TaskManager&) = delete;
@@ -259,6 +260,7 @@ class TaskManager : private HandlerAPI
 
     static inline size_t next_pow2(size_t val);
 
+    UpstreamManager& m_upstreamManager;
     SysInfoCounter& m_sysInfoCounter;
     GlobalContextMap m_gcm;
 
@@ -277,7 +279,6 @@ class TaskManager : private HandlerAPI
     std::deque<BaseTaskPtr> m_readyToResume;
     std::priority_queue<std::pair<std::chrono::time_point<std::chrono::steady_clock>,Context::uuid_t>> m_expireTaskQueue;
     std::unique_ptr<ExpiringList> m_futurePostponeUuids;
-    std::unique_ptr<UpstreamManager> m_upstreamManager;
 
     using PromiseItem = UpstreamTask::PromiseItem;
     using PromiseQueue = tp::MPMCBoundedQueue<PromiseItem>;