Debug CI

Signed-off-by: Aleksandr Ivanov <[email protected]>
bloomberg · Jul 15, 2024 · f86ec70 · f86ec70
1 parent d31b791
commit f86ec70
Show file tree

Hide file tree

Showing 2 changed files with 365 additions and 20 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -4,11 +4,9 @@ on:
   push:
     branches:
       - main
-      - add_sanitizers_to_ci
   pull_request:
     branches:
       - main
-      - add_sanitizers_to_ci
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -60,29 +58,282 @@ jobs:
           path: deps
           key: deps-${{ steps.get-hash.outputs.deps_hash }}
 
-  run_asan:
+  build_ubuntu:
+    name: Build [ubuntu]
+    runs-on: ubuntu-latest
     needs: build_dependencies
-    uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml
-    with:
-      sanitizer-name: 'asan'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get dependencies hash
+        id: get-hash
+        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
+      - uses: actions/cache/restore@v4
+        with:
+          path: deps
+          key: deps-${{ steps.get-hash.outputs.deps_hash }}
+      - name: Set up dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qy build-essential \
+            gdb \
+            curl \
+            python3.10 \
+            python3-pip \
+            cmake \
+            ninja-build \
+            pkg-config \
+            bison \
+            libfl-dev \
+            libbenchmark-dev \
+            libgmock-dev \
+            libz-dev
+      - name: Install cached non packaged dependencies
+        working-directory: deps
+        run: ../docker/build_deps.sh
+      - name: Build BlazingMQ
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
+        run: |
+          cmake -S . -B build/blazingmq -G Ninja \
+            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DBDE_BUILD_TARGET_SAFE=ON \
+            -DBDE_BUILD_TARGET_64=ON \
+            -DBDE_BUILD_TARGET_CPP17=ON \
+            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
+            -DCMAKE_INSTALL_LIBDIR=lib64
+          cmake --build build/blazingmq --parallel 8 --target all
+      - name: Clean-up build directories before caching
+        run: |
+          find . -name "*.o" -type f -delete
+          find . -name "*.a" -type f -delete
+      - uses: actions/cache@v4
+        with:
+          path: |
+            build/blazingmq
+            deps
+            /opt/bb/include
+          key: cache-${{ github.sha }}
 
-  run_msan:
+  unit_tests_cxx:
+    name: UT [c++]
+    runs-on: ubuntu-latest
     needs: build_dependencies
-    uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml
-    with:
-      sanitizer-name: 'msan'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get dependencies hash
+        id: get-hash
+        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
+      - uses: actions/cache/restore@v4
+        with:
+          path: deps
+          key: deps-${{ steps.get-hash.outputs.deps_hash }}
+      - name: Set up dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qy build-essential \
+            gdb \
+            curl \
+            python3.10 \
+            python3-pip \
+            cmake \
+            ninja-build \
+            pkg-config \
+            bison \
+            libfl-dev \
+            libbenchmark-dev \
+            libgmock-dev \
+            libz-dev
+      - name: Install cached non packaged dependencies
+        working-directory: deps
+        run: ../docker/build_deps.sh
+      - name: Build BlazingMQ UTs
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
+        run: |
+          cmake -S . -B build/blazingmq -G Ninja \
+            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DBDE_BUILD_TARGET_SAFE=ON \
+            -DBDE_BUILD_TARGET_64=ON \
+            -DBDE_BUILD_TARGET_CPP17=ON \
+            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
+            -DCMAKE_INSTALL_LIBDIR=lib64
+          cmake --build build/blazingmq --parallel 8 --target all.t
+      - name: Run C++ Unit Tests
+        run: |
+          cd ${{ github.workspace }}/build/blazingmq
+          ctest -E mwcsys_executil.t --output-on-failure
 
-  run_tsan:
-    needs: build_dependencies
-    uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml
-    with:
-      sanitizer-name: 'tsan'
+  unit_tests_python:
+    name: UT [python]
+    runs-on: ubuntu-latest
+    needs: build_ubuntu
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache/@v4
+        with:
+          path: |
+            build/blazingmq
+            deps
+            /opt/bb/include
+          key: cache-${{ github.sha }}
+      - name: Run Python Unit Tests
+        env:
+          PYTHONPATH: ${{ github.workspace }}/src/python
+        run: |
+          pip install -r ${{ github.workspace }}/src/python/requirements.txt
+          pip install --force-reinstall ruff==0.4.10
+          cd ${{ github.workspace }}
+          src/python/bin/schemagen
+          src/python/bin/tweakgen
+          pytest src/python
 
-  run_ubsan:
-    needs: build_dependencies
-    uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml
-    with:
-      sanitizer-name: 'ubsan'
+  integration_tests_ubuntu:
+    name: IT [${{ matrix.cluster }}/${{ matrix.mode }}]
+    strategy:
+      matrix:
+        mode: ["legacy_mode", "fsm_mode"]
+        cluster: ["single", "multi"]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    needs: build_ubuntu
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache/@v4
+        with:
+          path: |
+            build/blazingmq
+            deps
+            /opt/bb/include
+          key: cache-${{ github.sha }}
+      - name: Run Integration Tests
+        run: |
+          pip install -r ${{ github.workspace }}/src/python/requirements.txt
+          ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \
+            --log-level ERROR                   \
+            --log-file-level=info               \
+            --bmq-tolerate-dirty-shutdown       \
+            --bmq-log-dir=failure-logs          \
+            --bmq-log-level=INFO                \
+            --junitxml=integration-tests.xml    \
+            --tb long                           \
+            --reruns=3                          \
+            -n 4 -v
+
+  fuzz_tests_ubuntu:
+    name: Fuzz test [${{ matrix.request }}]
+    strategy:
+      matrix:
+        request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    needs: build_ubuntu
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache/@v4
+        with:
+          path: |
+            build/blazingmq
+            deps
+            /opt/bb/include
+          key: cache-${{ github.sha }}
+      - name: Run Fuzz Test
+        run: |
+          pip install -r ${{ github.workspace }}/src/python/requirements.txt
+          cd src/python
+          python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }}
+
+  build_macosx:
+    name: Build [macosx_${{ matrix.arch }}]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-14
+            arch: arm64
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up dependencies
+        run: |
+          brew install \
+            curl \
+            [email protected] \
+            ninja \
+            bison \
+            flex \
+            google-benchmark
+      - name: Build BlazingMQ
+        run: bin/build-darwin.sh
+
+  build_and_test_bmqprometheus_plugin:
+    name: "Build Prometheus plugin [ubuntu]"
+    runs-on: ubuntu-latest
+    needs: build_ubuntu
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache/@v4
+        with:
+          path: |
+            build/blazingmq
+            deps
+            /opt/bb/include
+          key: cache-${{ github.sha }}
+      - name: Set up plugins dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qy build-essential \
+            gdb \
+            curl \
+            python3.10 \
+            cmake \
+            ninja-build \
+            pkg-config \
+            bison \
+            libfl-dev \
+            libbenchmark-dev \
+            libgmock-dev \
+            libz-dev \
+            autoconf \
+            libtool
+      - name: Create dependency fetcher working directory
+        run: mkdir -p deps
+      - name: Fetch & Build non packaged plugins dependencies
+        working-directory: deps
+        run:  ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh
+      - name: Build plugins
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
+        run: |
+          cmake -S . -B build/blazingmq -G Ninja \
+            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DBDE_BUILD_TARGET_SAFE=ON \
+            -DBDE_BUILD_TARGET_64=ON \
+            -DBDE_BUILD_TARGET_CPP17=ON \
+            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
+            -DCMAKE_INSTALL_LIBDIR=lib64 \
+            -DINSTALL_TARGETS=prometheus
+          cmake --build build/blazingmq --parallel 8 --target all
+      - name: Create prometheus dir
+        run:  mkdir -p prometheus_dir
+      - name: Download Prometheus
+        run:  curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/
+      - name: Run Prometheus
+        run:  ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
+      - name: Run Pushgateway
+        run: |
+          docker pull prom/pushgateway
+          docker run -d -p 9091:9091 prom/pushgateway
+      - name: Run BMQPrometheus plugin integration test in "pull" mode
+        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker
+      - name: Clear Prometheus database and restart
+        run: |
+          curl -X POST "http://localhost:9090/-/quit"
+          rm -rf prometheus_dir/data
+          ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
+      - name: Run Prometheus plugin integration test in "push" mode
+        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker
 
   documentation:
     name: "Documentation"
@@ -104,3 +355,13 @@ jobs:
       - uses: actions/checkout@v4
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
+
+  docker_build_ubuntu:
+    name: "Docker [ubuntu]"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Docker Single-Node Workflow
+        run: docker compose -f docker/single-node/docker-compose.yaml up --build -d
+      - name: Docker Cluster Workflow
+        run: docker compose -f docker/cluster/docker-compose.yaml up --build -d