Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

air node gateway not required to load transport config #97

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions cpp/ppc-framework/io/DataBatch.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#pragma once
#include "../Common.h"
#include <bcos-utilities/Common.h>
#include <gperftools/malloc_extension.h>
#include <boost/lexical_cast.hpp>
#include <algorithm>
#include <memory>
Expand Down Expand Up @@ -210,8 +209,6 @@ class DataBatch
}
m_data->clear();
m_data.reset();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
}

private:
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${PIR_TARGET} ${SOURCES} ${OUT_TARS_H_LIST})
target_include_directories(${PIR_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ set(TEST_BINARY_NAME test-ppc-pir)
add_executable(${TEST_BINARY_NAME} ${SOURCES})
target_include_directories(${TEST_BINARY_NAME} PRIVATE .)

target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${CRYPTO_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${PIR_TARGET} ${BOOST_UNIT_TEST})
add_test(NAME test-pir WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${BS_ECDH_PSI_TARGET} ${SRCS})

target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb TCMalloc)
target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb)
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${CM2020_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${CM2020_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${ECDH_CONN_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${ECDH_CONN_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${ECDH_MULTI_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${ECDH_MULTI_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,8 @@ class CalculatorCache : public std::enable_shared_from_this<CalculatorCache>
for (auto const& it : m_plainData)
{
it->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
}
m_cipherRef.clear();
std::map<bcos::bytes, CipherRefDetail>().swap(m_cipherRef);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ void EcdhMultiPSIMaster::blindData()
});
// can release databatch after encrypted
dataBatch->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data success")
<< LOG_KV("dataSize", cipher.size()) << LOG_KV("task", m_taskID)
<< LOG_KV("seq", seq)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
<< LOG_KV("size", dataBatch->size())
<< printTaskInfo(m_taskState->task()) << LOG_KV("seq", seq);
dataBatch->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
} while (!m_taskState->sqlReader());
}
catch (std::exception& e)
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${ECDH_2PC_PSI_TARGET} ${SRCS})
target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET} TCMalloc)
target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ add_library(${LABELED_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${LABELED_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb TCMalloc)
target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ file(GLOB_RECURSE SRCS *.cpp)
add_library(${PSI_FRAMEWORK_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${PSI_FRAMEWORK_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)
target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET} TCMalloc)
target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC TCMalloc ${FRONT_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} ${IO_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${RA2018_PSI_TARGET} ${SRCS})
target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET})
target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
4 changes: 2 additions & 2 deletions cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ set(TEST_BINARY_NAME test-ppc-psi)
add_executable(${TEST_BINARY_NAME} ${SOURCES})
target_include_directories(${TEST_BINARY_NAME} PRIVATE .)

target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
add_test(NAME test-ppc-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
4 changes: 2 additions & 2 deletions cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ find_package(OpenSSL REQUIRED)
message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")

target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB} TCMalloc)
target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB})

if (ENABLE_IPP_CRYPTO)
find_package(ipp-crypto REQUIRED)
target_link_libraries(${CRYPTO_TARGET} PUBLIC ipp-crypto::crypto_mb)
endif ()
endif ()
2 changes: 1 addition & 1 deletion cpp/wedpr-helper/libhelper/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})

add_library(${HELPER_TARGET} ${SRCS})
find_package(Boost COMPONENTS filesystem program_options)
target_link_libraries(${HELPER_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} Boost::filesystem Boost::program_options)
target_link_libraries(${HELPER_TARGET} PUBLIC Boost::filesystem Boost::program_options)
2 changes: 1 addition & 1 deletion cpp/wedpr-helper/ppc-tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ aux_source_directory(src/config SRCS)

add_library(${TOOLS_TARGET} ${SRCS})

target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})
target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})

if (TESTS)
enable_testing()
Expand Down
20 changes: 13 additions & 7 deletions cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ using namespace ppc::protocol;
using namespace ppc::storage;
using namespace bcos;

void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport)
{
// load the network config
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config");
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config")
<< LOG_KV("requireTransport", requireTransport);
// gateway default enable-ssl
loadNetworkConfig(
m_gatewayConfig.networkConfig, _pt, "gateway", NetworkConfig::DefaultRpcListenPort, false);
Expand Down Expand Up @@ -73,11 +74,16 @@ void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
InvalidConfig() << errinfo_comment(
"The value of gateway.seq_sync_period_ms must no little than 3s"));
}
// load the grpcConfig
m_grpcConfig = loadGrpcConfig("transport", _pt);
// load the GrpcServerConfig
loadEndpointConfig(
m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
// only need load grpc config when requireTransport
if (requireTransport)
{
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load grpc config");
// load the grpcConfig
m_grpcConfig = loadGrpcConfig("transport", _pt);
// load the GrpcServerConfig
loadEndpointConfig(
m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
}
// the agencyID
m_agencyID = _pt.get<std::string>("agency.id", "");
if (m_agencyID.empty())
Expand Down
6 changes: 3 additions & 3 deletions cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,12 @@ class PPCConfig
loadRpcConfig(iniConfig);
}

void loadGatewayConfig(std::string const& _configPath)
void loadGatewayConfig(std::string const& _configPath, bool requireTransport)
{
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("path", _configPath);
boost::property_tree::ptree iniConfig;
boost::property_tree::read_ini(_configPath, iniConfig);
loadGatewayConfig(iniConfig);
loadGatewayConfig(iniConfig, requireTransport);
}

void loadFrontConfig(bool requireTransport,
Expand All @@ -180,7 +180,7 @@ class PPCConfig
loadNetworkConfig(m_rpcConfig, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true);
}

virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt);
virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport);


NetworkConfig const& rpcConfig() const { return m_rpcConfig; }
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/air-node/AirNodeInitializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ void AirNodeInitializer::initGateway(std::string const& _configPath)
INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
// not specify the certPath in air-mode
auto config = m_nodeInitializer->config();
config->loadGatewayConfig(_configPath);
config->loadGatewayConfig(_configPath, false);

auto threadPool = std::make_shared<bcos::ThreadPool>(
"gateway", config->gatewayConfig().networkConfig.threadPoolSize);
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/air-node/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
aux_source_directory(. SRC_LIST)

add_executable(${AIR_BINARY_NAME} ${SRC_LIST})
target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-main/gateway/GatewayInitializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ void GatewayInitializer::init(std::string const& _configPath)
INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
auto config = std::make_shared<PPCConfig>();

config->loadGatewayConfig(_configPath);
config->loadGatewayConfig(_configPath, true);
auto threadPool = std::make_shared<bcos::ThreadPool>(
"gateway", config->gatewayConfig().networkConfig.threadPoolSize);

Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/pro-node/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR})
aux_source_directory(./ SRC_LIST)
add_executable(${PRO_BINARY_NAME} ${SRC_LIST})

target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${IO_TARGET} ${SRCS})
# Note: the DataBatch depends on tbb
target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} TCMalloc ${HDFS_LIB} ${CPU_FEATURES_LIB})
target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB})
2 changes: 1 addition & 1 deletion cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ add_library(${STORAGE_TARGET} ${SRCS})
find_package(redis++ REQUIRED)
find_package(unofficial-mysql-connector-cpp REQUIRED)

target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static TCMalloc unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
target_link_libraries(${STORAGE_TARGET} PUBLIC redis++::redis++_static unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
Loading