Feat/s3: implementing pika storage on s3 using the rocksdb-cloud libr…

…ary (#1985) * feat: implementing pika storage on s3 using the rocksdb-cloud library
OpenAtomFoundation · Sep 27, 2023 · 35fc787 · 35fc787
1 parent d06ec36
commit 35fc787
Show file tree

Hide file tree

Showing 31 changed files with 817 additions and 61 deletions.
diff --git a/.github/workflows/pika_cloud.yml b/.github/workflows/pika_cloud.yml
@@ -0,0 +1,278 @@
+name: Pika_cloud
+
+on:
+  push:
+    branches: [ "unstable", "3.5.0" ]
+  pull_request:
+    branches: [ "unstable", "3.5.0" ]
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: RelWithDebInfo
+  LD_LIBRARY_PATH: /usr/local/lib
+
+jobs:
+  build_on_ubuntu:
+    # The CMake configure and build commands are platform-agnostic and should work equally well on Windows or Mac.
+    # You can convert this to a matrix build if you need cross-platform coverage.
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: cache dependencies
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
+            ~/.cache/pip
+          key: ${{ runner.os }}-dependencies
+
+      - name: install Deps
+        if: ${{ steps.cache.output.cache-hit != 'true' }}
+        run: |
+          sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
+          sudo apt-get install -y clang-tidy-12 python3-pip
+          python3 -m pip install --upgrade pip
+          python3 -m pip install redis
+      
+      - name: Set up AWS SDK for C++
+        run: |
+          sudo apt-get update --fix-missing
+          sudo apt-get install libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
+          git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp
+          mkdir sdk_build
+          cd sdk_build
+          cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${{ env.LD_LIBRARY_PATH }} DCMAKE_INSTALL_PREFIX=${{ env.LD_LIBRARY_PATH }} -DBUILD_ONLY="s3;core;transfer;kinesis" -DBUILD_SHARED_LIBS=ON
+          make
+          sudo make install
+      
+      - name: Set up and start MinIO
+        run: |
+          wget https://dl.minio.org.cn/server/minio/release/linux-amd64/minio
+          chmod +x minio
+          ./minio server minio_data&
+
+      - name: Configure CMake
+        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+        # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_S3=ON -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+
+      - name: Build
+        # Build your program with the given configuration
+        run: cmake --build build --config ${{ env.BUILD_TYPE }}
+
+      - name: Test
+        working-directory: ${{ github.workspace }}/build
+        # Execute tests defined by the CMake configuration.
+        # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+        run: ctest -C ${{ env.BUILD_TYPE }}
+
+
+      # - name: Unit Test
+      #   working-directory: ${{ github.workspace }}
+      #   run: ./pikatests.sh all
+
+        # master on port 9221, slave on port 9231, all with 2 db
+      - name: Start pika master and slave
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          chmod +x ../tests/integration/start_master_and_slave.sh
+          ../tests/integration/start_master_and_slave.sh
+
+      - name: Run Python E2E Tests
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          python3 ../tests/integration/pika_replication_test.py
+          python3 ../tests/unit/Blpop_Brpop_test.py
+
+      - name: Run Go E2E Tests
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          cd ../tests/integration/
+          chmod +x integrate_test.sh 
+          sh integrate_test.sh
+
+  # build_on_centos:
+  #   runs-on: ubuntu-latest
+  #   container:
+  #     image: centos:7
+
+  #   steps:
+  #     - name: Install deps
+  #       run: |
+  #         yum install -y wget git autoconf centos-release-scl gcc-c++
+  #         yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
+  #         yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which python3
+  #         python3 -m pip install --upgrade pip
+  #         python3 -m pip install redis
+
+  #     - name: Set up Go
+  #       uses: actions/setup-go@v3
+  #       with:
+  #         go-version: 1.19
+
+  #     - name: Install cmake
+  #       run: |
+  #         wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
+  #         bash ./cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
+
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #       with:
+  #         fetch-depth: 0
+
+  #     - name: Set up AWS SDK for C++
+  #       run: |
+  #         yum install -y libcurl-devel openssl-devel libuuid-devel pulseaudio-libs-devel
+  #         git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp
+  #         mkdir sdk_build
+  #         cd sdk_build
+  #         cmake ../aws-sdk-cpp -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${{ env.LD_LIBRARY_PATH }} -DCMAKE_INSTALL_PREFIX=${{ env.LD_LIBRARY_PATH }} -DBUILD_ONLY="s3;core;transfer;kinesis" -DBUILD_SHARED_LIBS=ON
+  #         make
+  #         make install
+
+  #     - name: Set up and start MinIO
+  #       run: |
+  #         wget https://dl.minio.org.cn/server/minio/release/linux-amd64/minio
+  #         chmod +x minio
+  #         ./minio server minio_data&
+
+  #     - name: Configure CMake
+  #       run: |
+  #         source /opt/rh/devtoolset-10/enable
+  #         cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_S3=ON -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+
+  #     - name: Build
+  #       run: |
+  #         source /opt/rh/devtoolset-10/enable
+  #         cmake --build build --config ${{ env.BUILD_TYPE }}
+
+  #     - name: Test
+  #       working-directory: ${{ github.workspace }}/build
+  #       run: ctest -C ${{ env.BUILD_TYPE }}
+
+  #     # - name: Unit Test
+  #     #   working-directory: ${{ github.workspace }}
+  #     #   run: ./pikatests.sh all
+
+  #     - name: Start pika master and slave
+  #       working-directory: ${{ github.workspace }}/build
+  #       run: |
+  #         chmod +x ../tests/integration/start_master_and_slave.sh
+  #         ../tests/integration/start_master_and_slave.sh
+
+  #     - name: Run Python E2E Tests
+  #       working-directory: ${{ github.workspace }}/build
+  #       run: |
+  #         python3 ../tests/integration/pika_replication_test.py
+  #         python3 ../tests/unit/Blpop_Brpop_test.py
+
+  # build_on_macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #     - uses: actions/checkout@v3
+
+  #     - name: Set up Go
+  #       uses: actions/setup-go@v3
+  #       with:
+  #         go-version: 1.19
+
+  #     - name: cache dependencies
+  #       uses: actions/cache@v2
+  #       id: cache
+  #       with:
+  #         path: |
+  #           ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
+  #           ~/.cache/pip
+  #         key: ${{ runner.os }}-dependencies
+
+  #     - name: install Deps
+  #       if: ${{ steps.cache.output.cache-hit != 'true' }}
+  #       run: |
+  #         brew update
+  #         brew install --overwrite python autoconf protobuf llvm wget git
+  #         brew install gcc@10 automake cmake make binutils
+  #         python3 -m pip install --upgrade pip
+  #         python3 -m pip install redis
+
+      # - name: Start UP MinIO
+      #   uses: infleet/[email protected]
+      #   with:
+      #     username: "minioadmin"
+      #     password: "minioadmin"
+
+      # - name: Configure CMake
+      #   run: |
+      #     export CC=/usr/local/opt/gcc@10/bin/gcc-10 
+      #     cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+      #   #  cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_S3=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+
+      # - name: Build
+      #   run: |
+      #     cmake --build build --config ${{ env.BUILD_TYPE }}
+
+      # - name: Test
+      #   working-directory: ${{ github.workspace }}/build
+      #   run: ctest -C ${{ env.BUILD_TYPE }}
+
+      # - name: Unit Test
+      #   working-directory: ${{ github.workspace }}
+      #   run: |
+      #     ./pikatests.sh all
+
+      # - name: Start pika master and slave
+      #   working-directory: ${{ github.workspace }}/build
+      #   run: |
+      #     chmod +x ../tests/integration/start_master_and_slave.sh
+      #     ../tests/integration/start_master_and_slave.sh
+
+      # - name: Run Python E2E Tests
+      #   working-directory: ${{ github.workspace }}/build
+      #   run: |
+      #     python3 ../tests/integration/pika_replication_test.py
+      #     python3 ../tests/unit/Blpop_Brpop_test.py
+
+      # - name: Run Go E2E Tests
+      #   working-directory: ${{ github.workspace }}/build
+      #   run: |
+      #     cd ../tests/integration/
+      #     chmod +x integrate_test.sh 
+      #     sh integrate_test.sh
+
+  # build_pika_image:
+  #   name: Build Pika Docker image
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Check out the repo
+  #       uses: actions/checkout@v3
+
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v2
+
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v2
+
+  #     - name: Extract metadata (tags, labels) for Docker
+  #       id: meta
+  #       uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+  #       with:
+  #         images: pikadb/pika
+
+  #     - name: Build Docker image
+  #       uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+  #       with:
+  #         context: .
+  #         file: ./Dockerfile
+  #         push: false
+  #         tags: ${{ steps.meta.outputs.tags }}
+  #         labels: ${{ steps.meta.outputs.labels }}
+