diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cd6325bbbe..2f83ddce62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,11 +4,9 @@ on: push: branches: - main - - add_sanitizers_to_ci pull_request: branches: - main - - add_sanitizers_to_ci concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -60,29 +58,282 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - run_asan: + build_ubuntu: + name: Build [ubuntu] + runs-on: ubuntu-latest needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'asan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all + - name: Clean-up build directories before caching + run: | + find . -name "*.o" -type f -delete + find . -name "*.a" -type f -delete + - uses: actions/cache@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} - run_msan: + unit_tests_cxx: + name: UT [c++] + runs-on: ubuntu-latest needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'msan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ UTs + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all.t + - name: Run C++ Unit Tests + run: | + cd ${{ github.workspace }}/build/blazingmq + ctest -E mwcsys_executil.t --output-on-failure - run_tsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'tsan' + unit_tests_python: + name: UT [python] + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Python Unit Tests + env: + PYTHONPATH: ${{ github.workspace }}/src/python + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + pip install --force-reinstall ruff==0.4.10 + cd ${{ github.workspace }} + src/python/bin/schemagen + src/python/bin/tweakgen + pytest src/python - run_ubsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'ubsan' + integration_tests_ubuntu: + name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] + strategy: + matrix: + mode: ["legacy_mode", "fsm_mode"] + cluster: ["single", "multi"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Integration Tests + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ + --log-level ERROR \ + --log-file-level=info \ + --bmq-tolerate-dirty-shutdown \ + --bmq-log-dir=failure-logs \ + --bmq-log-level=INFO \ + --junitxml=integration-tests.xml \ + --tb long \ + --reruns=3 \ + -n 4 -v + + fuzz_tests_ubuntu: + name: Fuzz test [${{ matrix.request }}] + strategy: + matrix: + request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Fuzz Test + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + cd src/python + python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} + + build_macosx: + name: Build [macosx_${{ matrix.arch }}] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-14 + arch: arm64 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + brew install \ + curl \ + python@3.10 \ + ninja \ + bison \ + flex \ + google-benchmark + - name: Build BlazingMQ + run: bin/build-darwin.sh + + build_and_test_bmqprometheus_plugin: + name: "Build Prometheus plugin [ubuntu]" + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Set up plugins dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev \ + autoconf \ + libtool + - name: Create dependency fetcher working directory + run: mkdir -p deps + - name: Fetch & Build non packaged plugins dependencies + working-directory: deps + run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh + - name: Build plugins + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ + -DINSTALL_TARGETS=prometheus + cmake --build build/blazingmq --parallel 8 --target all + - name: Create prometheus dir + run: mkdir -p prometheus_dir + - name: Download Prometheus + run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ + - name: Run Prometheus + run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Pushgateway + run: | + docker pull prom/pushgateway + docker run -d -p 9091:9091 prom/pushgateway + - name: Run BMQPrometheus plugin integration test in "pull" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker + - name: Clear Prometheus database and restart + run: | + curl -X POST "http://localhost:9090/-/quit" + rm -rf prometheus_dir/data + ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Prometheus plugin integration test in "push" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker documentation: name: "Documentation" @@ -104,3 +355,13 @@ jobs: - uses: actions/checkout@v4 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master + + docker_build_ubuntu: + name: "Docker [ubuntu]" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Docker Single-Node Workflow + run: docker compose -f docker/single-node/docker-compose.yaml up --build -d + - name: Docker Cluster Workflow + run: docker compose -f docker/cluster/docker-compose.yaml up --build -d diff --git a/.github/workflows/sanitizer-debug.yaml b/.github/workflows/sanitizer-debug.yaml new file mode 100644 index 0000000000..634e26c273 --- /dev/null +++ b/.github/workflows/sanitizer-debug.yaml @@ -0,0 +1,84 @@ +name: Debug + +on: + push: + branches: + - main + - add_sanitizers_to_ci + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build_dependencies: + name: Build deps [ubuntu] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - name: Cache lookup + uses: actions/cache/restore@v4 + id: cache-lookup + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + lookup-only: true + - name: Set up dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Fetch & Build non packaged dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + mkdir -p deps + cd deps + ../docker/build_deps.sh + - name: Cache save + if: steps.cache-lookup.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + + run_asan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'asan' + + run_msan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'msan' + + run_tsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'tsan' + + run_ubsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'ubsan'