deal with todo-lists

.github/workflows/cpp_full_node_workflow.yml

@@ -53,9 +53,9 @@ jobs:
             !c:/vcpkg/buildtrees
             !c:/vcpkg/packages
             !c:/vcpkg/downloads
-          key: build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
           restore-keys: |
-            build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
             build-all-${{ matrix.os }}-${{ github.base_ref }}-
             build-all-${{ matrix.os }}-
       - name: Build for linux
@@ -72,12 +72,6 @@ jobs:
           bash -x cpp/tools/install_depends.sh -o macos
           mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
-      - name: Publish Error
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-            name: config-x64-linux-dbg-err.log
-            path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log
       - uses: actions/upload-artifact@v4
         if: runner.os == 'macOS'      
         with:
@@ -123,9 +117,9 @@ jobs:
             /usr/local/share/vcpkg/packages
             /home/runner/.ccache
             /Users/runner/.ccache/
-          key: centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
           restore-keys: |
-            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
             centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
             centos-notest-all-${{ matrix.os }}-
       - name: Prepare centos tools
@@ -141,28 +135,35 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
-          yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
+          yum install -y wget lcov java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
           yum install -y git
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Check disk space
+        run: df . -h
+      - name: Free disk space
+        run: |
+          du -sch /usr/share/*
+          du -sch /opt/*
+      - name: Check disk space
+        run: |
+          df . -h
+          du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
+          df /tmp
+          du /tmp | sort -n
+          ls -lh /tmp
       - name: Build
         run: |
           bash -x cpp/tools/install_depends.sh -o centos
           alias cmake='cmake3'
-          . /opt/rh/devtoolset-11/enable 
+          . /opt/rh/devtoolset-11/enable
+          rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
           cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-          cmake3 --build . --parallel 3
-      - name: Publish Error
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-            name: vcpkg-manifest-install.log
-            path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
+          #cmake3 --build . --parallel 3
       #- name: Test
       #  run: |
       #    export OMP_NUM_THREADS=1
@@ -178,4 +179,4 @@ jobs:
       - uses: actions/upload-artifact@v3
         with:
           name: ppc-gateway-service-x64
-          path: ./cpp/build/bin/ppc-gateway-service
+          path: ./cpp/build/bin/ppc-gateway-service

.github/workflows/cpp_sdk_workflow.yml

@@ -53,9 +53,9 @@ jobs:
             ${{ env.VCPKG_ROOT }}/buildtrees
             ${{ env.VCPKG_ROOT }}/packages
             ${{ env.VCPKG_ROOT }}/downloads
-          key: build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
           restore-keys: |
-            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
             build-sdk-${{ matrix.os }}-${{ github.base_ref }}-
             build-sdk-${{ matrix.os }}-
       - name: Add MSbuild to PATH
@@ -127,9 +127,9 @@ jobs:
               /usr/local/share/vcpkg/packages
               /home/runner/.ccache
               /Users/runner/.ccache/
-            key: centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            key: centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
             restore-keys: |
-              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
               centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
               centos-sdk-notest-all-${{ matrix.os }}-
         - name: Prepare centos tools

.github/workflows/cpp_toolkit_workflow.yml

@@ -53,9 +53,9 @@ jobs:
             ${{ env.VCPKG_ROOT }}/buildtrees
             ${{ env.VCPKG_ROOT }}/packages
             ${{ env.VCPKG_ROOT }}/downloads
-          key: build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
           restore-keys: |
-            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
             build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-
             build-toolkit-${{ matrix.os }}-
       - name: Add MSbuild to PATH
@@ -118,9 +118,9 @@ jobs:
               /usr/local/share/vcpkg/packages
               /home/runner/.ccache
               /Users/runner/.ccache/
-            key: centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            key: centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
             restore-keys: |
-              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
               centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
               centos-toolkit-notest-all-${{ matrix.os }}-
         - name: Prepare centos tools

cpp/cmake/CompilerSettings.cmake

@@ -1,12 +1,6 @@
 set(CMAKE_CXX_STANDARD 20)
 set(Boost_NO_WARN_NEW_VERSIONS ON)
 message(STATUS "COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
-
-# export windows dll symbol
-if(WIN32)
-    message(STATUS "Compile on Windows")
-    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON")
-endif()
 if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
     find_program(CCACHE_PROGRAM ccache)
     if(CCACHE_PROGRAM)
@@ -132,38 +126,28 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
     endif()
     endif ()
 elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
-    add_compile_definitions(NOMINMAX)
-
-    # Only support visual studio 2017 and visual studio 2019
-    set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support
+       # Only support visual studio 2017 and visual studio 2019
+       set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support
 
-    message(STATUS "Compile On Windows, MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
-
-    if (MSVC_TOOLSET_VERSION EQUAL 141)
-        message(STATUS "Compile On Visual Studio 2017")
-    elseif(MSVC_TOOLSET_VERSION EQUAL 142)
-        message(STATUS "Compile On Visual Studio 2019")
-    else()
-        message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
-    endif()
-
-    add_compile_options(/std:c++latest)
-    add_compile_options(-bigobj)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
-    if(BUILD_SHARED_LIBS)
-        if(CMAKE_BUILD_TYPE MATCHES "Debug")
-            add_compile_options(/MDd)
-        else()
-            add_compile_options(/MD)
-        endif()
-    else ()
-        if(CMAKE_BUILD_TYPE MATCHES "Debug")
-            add_compile_options(/MTd)
-        else()
-            add_compile_options(/MT)
-        endif ()
-    endif ()
-    link_libraries(ws2_32 Crypt32 userenv)
+       message(STATUS "Compile On Windows, MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
+
+       if (MSVC_TOOLSET_VERSION EQUAL 141)
+           message(STATUS "Compile On Visual Studio 2017")
+       elseif(MSVC_TOOLSET_VERSION EQUAL 142)
+           message(STATUS "Compile On Visual Studio 2019")
+       else()
+           message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
+       endif()
+
+       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17")
+       add_compile_definitions(NOMINMAX)
+       add_compile_options(-bigobj)
+        # MSVC only support static build
+        set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
+        set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MT")
+        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
+        link_libraries(ws2_32 Crypt32 userenv)
 else ()
     message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.")
 endif ()

cpp/cmake/Options.cmake

@@ -169,7 +169,7 @@ macro(print_config NAME)
     message("------------------------------------------------------------------------")
     message("-- Configuring ${NAME} ${PROJECT_VERSION}${VERSION_SUFFIX}")
     message("------------------------------------------------------------------------")
-    message("-- CMake               Cmake version and location   ${CMAKE_VERSION} (${CMAKE_COMMAND})")
+    message("-- CMake               CMake version and location   ${CMAKE_VERSION} (${CMAKE_COMMAND})")
     message("-- Compiler            C++ compiler version         ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
     message("-- CMAKE_BUILD_TYPE    Build type                   ${CMAKE_BUILD_TYPE}")
     message("-- VCPKG_MANIFEST_FEATURES VCPKG manifest features  ${VCPKG_MANIFEST_FEATURES}")

cpp/cmake/grpc.cmake

@@ -41,7 +41,10 @@ if(NOT GRPC_CPP_REFLECTION AND TARGET gRPC::grpc++_reflection)
   endif()
 endif()
 
+get_target_property(GRPC_CPP_REFLECTION_INCLUDE gRPC::grpc++_reflection INTERFACE_INCLUDE_DIRECTORIES)
+include_directories(${GRPC_CPP_REFLECTION_INCLUDE})
+
 set(PROTOC_BINARY ${Protobuf_PROTOC_EXECUTABLE})
 message("# PROTOC_BINARY: ${Protobuf_PROTOC_EXECUTABLE}")
 message("# GRPC_CPP_PLUGIN: ${GRPC_CPP_PLUGIN}")
-message("# GRPC_CPP_REFLECTION: ${GRPC_CPP_REFLECTION}")
+message("# GRPC_CPP_REFLECTION_INCLUDE: ${GRPC_CPP_REFLECTION_INCLUDE}")

cpp/ppc-framework/protocol/GlobalConfig.h

@@ -33,7 +33,7 @@ class GlobalConfig
     GlobalConfig()
     {
         // set the supported-curves for ecdh-psi
-        auto key = calculateKey((uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        auto key = calculateKey((uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
 #ifdef ENABLE_CRYPTO_MB
 #ifdef ENABLE_CPU_FEATURES
         if (ppc::CPU_FEATURES.avx512ifma)

cpp/ppc-framework/protocol/Protocol.h

@@ -57,8 +57,7 @@ inline std::ostream& operator<<(std::ostream& _out, TaskType const& _type)
 }
 
 // the PSIAlgorithm type
-// TODO: rename PSIAlgorithmType to TaskAlgorithmType
-enum class PSIAlgorithmType : uint8_t
+enum class TaskAlgorithmType : uint8_t
 {
     // PSI implementation for https://eprint.iacr.org/2020/729.pdf(Private Set Intersection in the
     // Internet Setting from Lightweight Oblivious PRF)
@@ -76,34 +75,29 @@ enum class PSIAlgorithmType : uint8_t
     BS_ECDH_PSI = 0x06,
 };
 
-// enum class PSIAlgorithmType : uint8_t
-// {
-//     OT_PIR_2PC = 0x10,
-// };
-
-inline std::ostream& operator<<(std::ostream& _out, PSIAlgorithmType const& _type)
+inline std::ostream& operator<<(std::ostream& _out, TaskAlgorithmType const& _type)
 {
     switch (_type)
     {
-    case PSIAlgorithmType::CM_PSI_2PC:
+    case TaskAlgorithmType::CM_PSI_2PC:
         _out << "CM_PSI_2PC";
         break;
-    case PSIAlgorithmType::RA_PSI_2PC:
+    case TaskAlgorithmType::RA_PSI_2PC:
         _out << "RA_PSI_2PC";
         break;
-    case PSIAlgorithmType::LABELED_PSI_2PC:
+    case TaskAlgorithmType::LABELED_PSI_2PC:
         _out << "LABELED_PSI_2PC";
         break;
-    case PSIAlgorithmType::ECDH_PSI_2PC:
+    case TaskAlgorithmType::ECDH_PSI_2PC:
         _out << "ECDH_PSI_2PC";
         break;
-    case PSIAlgorithmType::OT_PIR_2PC:
+    case TaskAlgorithmType::OT_PIR_2PC:
         _out << "OT_PIR_2PC";
         break;
-    case PSIAlgorithmType::ECDH_PSI_MULTI:
+    case TaskAlgorithmType::ECDH_PSI_MULTI:
         _out << "ECDH_PSI_MULTI";
         break;
-    case PSIAlgorithmType::ECDH_PSI_CONN:
+    case TaskAlgorithmType::ECDH_PSI_CONN:
         _out << "ECDH_PSI_CONN";
         break;
     default:

cpp/ppc-framework/protocol/Task.h

@@ -162,7 +162,7 @@ inline std::string printTaskInfo(Task::ConstPtr _task)
     std::ostringstream stringstream;
     stringstream << LOG_KV("id", _task->id())
                  << LOG_KV("type", (ppc::protocol::TaskType)_task->type())
-                 << LOG_KV("algorithm", (ppc::protocol::PSIAlgorithmType)_task->algorithm())
+                 << LOG_KV("algorithm", (ppc::protocol::TaskAlgorithmType)_task->algorithm())
                  << LOG_KV("taskPtr", _task);
     if (_task->selfParty())
     {

cpp/test-utils/FakeFront.h

@@ -89,7 +89,7 @@ class FakeFront : public FrontInterface
         uint8_t type = _message->algorithmType();
         switch (type)
         {
-        case uint8_t(PSIAlgorithmType::RA_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::RA_PSI_2PC):
         {
             if (!m_agencyToRA2018.count(_agencyID))
             {
@@ -103,7 +103,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::LABELED_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::LABELED_PSI_2PC):
         {
             if (!m_agencyToLabeledPSI.count(_agencyID))
             {
@@ -117,7 +117,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::CM_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::CM_PSI_2PC):
         {
             if (!m_agencyToCM2020.count(_agencyID))
             {
@@ -131,7 +131,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::ECDH_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::ECDH_PSI_2PC):
         {
             if (!m_agencyToEcdhPSI.count(_agencyID))
             {
@@ -145,7 +145,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::ECDH_PSI_MULTI):
+        case uint8_t(TaskAlgorithmType::ECDH_PSI_MULTI):
         {
             if (!m_agencyToEcdhMutliPSI.count(_agencyID))
             {
@@ -159,7 +159,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::OT_PIR_2PC):
+        case uint8_t(TaskAlgorithmType::OT_PIR_2PC):
         {
             if (!m_agencyToOTPIR.count(_agencyID))
             {

cpp/tools/install_depends.sh

@@ -25,22 +25,6 @@ install_gsasl_depend()
         LOG_INFO "the libgasal.a has already exists!"
         return
     fi
-    LOG_INFO "download and install gsasl..."
-    wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/gsasl-1.8.0.tar.gz && tar -xvf gsasl-1.8.0.tar.gz 
-
-    # centos
-    if [[ "${os_type}" == "centos" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    # macos
-    if [[ "${os_type}" == "macos" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    # ubuntu
-    if [[ "${os_type}" == "ubuntu" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    LOG_INFO "download and install gsasl success..."
 
     LOG_INFO "download and install libgsasl..."
     wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz && tar -xvf libgsasl-1.8.0.tar.gz 
@@ -57,6 +41,8 @@ install_gsasl_depend()
     if [[ "${os_type}" == "ubuntu" ]];then
         cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
     fi
+    cd .. && rm -rf libgsasl-1.8.0 && rm -rf libgsasl-1.8.0.tar.gz
+    df -lh
     LOG_INFO "download and install libgsasl success..."
 }
 
@@ -74,6 +60,7 @@ install_nasm_depend()
     LOG_INFO "download and install nasm 2.15..."
     wget --no-check-certificate https://www.nasm.us/pub/nasm/releasebuilds/2.15/nasm-2.15.tar.gz && tar -xvf nasm-2.15.tar.gz
     cd nasm-2.15 && ./configure && make -j4 && make install
+    cd .. && rm -rf nasm-2.15.tar.gz && rm -rf nasm-2.15
     LOG_INFO "download and install nasm success..."
 }