Skip to content

Commit

Permalink
chore(CI): build deps using Bazel
Browse files Browse the repository at this point in the history
  • Loading branch information
ADD-SP committed Mar 9, 2024
1 parent 9bc643c commit 438190c
Show file tree
Hide file tree
Showing 11 changed files with 2,295 additions and 271 deletions.
14 changes: 14 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Enable Bzlmod for every Bazel command
common --enable_bzlmod
common --verbose_failures
common --registry=https://bcr.bazel.build

build --compilation_mode=dbg
build --copt="-g3"
build --strip=never
build --sandbox_debug

build:release --compilation_mode=opt
build:release --copt="-g3"
build:release --strip=never
build:release --sandbox_debug
1 change: 1 addition & 0 deletions .bazelversion
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7.0.2
224 changes: 33 additions & 191 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ on:
- current
- current-dev
pull_request:
schedule:
- cron: '0 0 * * SUN'
workflow_dispatch:


Expand All @@ -24,105 +22,15 @@ concurrency:
cancel-in-progress: true

jobs:
self-trigger:
runs-on: ubuntu-latest
if: ${{ github.event_name == 'schedule' }}
steps:
- name: Trigger
run: |
curl -H "Authorization: token ${{ secrets.ACCESS_TOKEN }}" \
-X POST \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/ADD-SP/ngx_waf/actions/workflows/${{ github.workflow }}.yml/dispatches \
-d '{"ref":"refs/heads/lts"}'
curl -H "Authorization: token ${{ secrets.ACCESS_TOKEN }}" \
-X POST \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/ADD-SP/ngx_waf/actions/workflows/${{ github.workflow }}.yml/dispatches \
-d '{"ref":"refs/heads/current"}'
build_deps:
name: Build dependencies
runs-on: ubuntu-latest

env:
BUILD_CACHE_NONCE: 0

steps:
- name: Set environment variables
run: |
echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}

- name: Checkout SpiderLabs/ModSecurity
uses: actions/checkout@v3
with:
repository: SpiderLabs/ModSecurity
path: ModSecurity

- name: Get ModSecurity commit hash
run: |
cd ModSecurity
echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV
- name: Lookup build cache
uses: actions/cache@v3
id: cache-deps
with:
path: ${{ env.CACHE_ROOT }}
key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }}

- name: Install dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
sudo apt-get --yes update
# https://github.com/actions/runner-images/issues/2139
sudo apt-get remove nginx-core nginx-full nginx-light nginx-extras
sudo apt-get remove libgd3
sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev
- name: Download and install ModSecurity
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
cd ModSecurity
./build.sh
git submodule init
git submodule update
./configure --prefix=/usr/local/modsecurity
make -j$(nproc)
sudo make install
- name: Package dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
mkdir -p ${{ env.CACHE_ROOT }}
mkdir -p /tmp/cache/usr/local
sudo cp -r /usr/local/modsecurity /tmp/cache/usr/local
tar -C /tmp/cache/ -cvzf ${{ env.CACHE_ROOT }}/cache.tar.gz .
native:
runs-on: ubuntu-latest
if: ${{ github.event_name != 'schedule' }}
needs: build_deps

env:
BUILD_CACHE_NONCE: 0

strategy:
matrix:
nginx-version: ['stable', 'mainline']
module-type: ['static', 'dynamic']
pcre-version: ['pcre', 'pcre2']

steps:
- name: Set environment variables
run: |
echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV
- uses: actions/setup-python@v2
with:
python-version: '3.x'
Expand All @@ -131,35 +39,30 @@ jobs:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}

- name: Checkout SpiderLabs/ModSecurity
uses: actions/checkout@v3
with:
repository: SpiderLabs/ModSecurity
path: ModSecurity

- name: Get ModSecurity commit hash
run: |
cd ModSecurity
echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV
- name: Lookup build cache
uses: actions/cache@v3
id: cache-deps
with:
path: ${{ env.CACHE_ROOT }}
key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }}

- name: Extract the deps
run: sudo tar -C / -xzvf ${{ env.CACHE_ROOT }}/cache.tar.gz
submodules: true

- name: Install dependencies
run: |
sudo apt-get --yes update
# https://github.com/actions/runner-images/issues/2139
sudo apt-get remove nginx-core nginx-full nginx-light nginx-extras
sudo apt-get remove libgd3
sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev
sudo apt-get remove nginx-core \
nginx-full \
nginx-light \
nginx-extras \
libgd3
sudo apt-get install --yes libsodium23 \
libsodium-dev \
build-essential \
zlib1g-dev \
libpcre3 \
libpcre3-dev \
libssl-dev \
libxslt1-dev \
libxml2-dev \
libgeoip-dev \
libgd-dev \
libperl-dev \
libcurl4-openssl-dev
- name: Download nginx-${{ matrix.nginx-version }}
run: |
Expand All @@ -170,44 +73,29 @@ jobs:
mkdir nginx
tar zxf nginx-*.tar.gz --directory nginx --strip-components=1
- name: Download PCRE
if: ${{ matrix.pcre-version == 'pcre' || (matrix.pcre-version == 'pcre2' && matrix.nginx-version == 'stable') }}
run: |
sudo chmod 777 -R /usr/local/src
cd /usr/local/src
wget https://cfhcable.dl.sourceforge.net/project/pcre/pcre/8.45/pcre-8.45.tar.gz -O pcre.tar.gz
mkdir pcre
tar zxf pcre.tar.gz --directory pcre --strip-components=1
sudo chmod 777 -R /usr/local/src
- name: Download PCRE2
if: ${{ matrix.pcre-version == 'pcre2' && matrix.nginx-version == 'mainline' }}
run: |
sudo chmod 777 -R /usr/local/src
cd /usr/local/src
wget https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.39/pcre2-10.39.tar.gz -O pcre.tar.gz
mkdir pcre
tar zxf pcre.tar.gz --directory pcre --strip-components=1
sudo chmod 777 -R /usr/local/src
- name: Download cjson
run: |
git clone -b v1.7.15 https://github.com/DaveGamble/cJSON.git lib/cjson
- name: Download uthash
run: |
git clone -b v2.3.0 https://github.com/troydhanson/uthash.git lib/uthash
- name: Configure ${{ matrix.module-type }}
run: |
bazel build --config release --registry=file://$(realpath ngx_waf_deps) //:deps
pushd bazel-bin
tar -zxf deps.tar.gz
echo "$(realpath deps/libmodsecurity/lib)" | sudo tee -a /etc/ld.so.conf.d/ngx_waf.conf
echo "$(realpath deps/libsodium/lib)" | sudo tee -a /etc/ld.so.conf.d/ngx_waf.conf
echo "$(realpath deps/libcjson/lib)" | sudo tee -a /etc/ld.so.conf.d/ngx_waf.conf
sudo ldconfig
export LIB_MODSECURITY=$(realpath deps/libmodsecurity)
export LIB_SODIUM=$(realpath deps/libsodium)
export LIB_CJSON=$(realpath deps/libcjson)
export LIB_UTHASH=$(realpath deps/uthash)
popd
cd /usr/local/src/nginx
if [ ${{ matrix.module-type }} = 'static module' ] ; then \
opt='--add-module' ;\
else \
opt='--add-dynamic-module' ;\
fi
export LIB_MODSECURITY=/usr/local/modsecurity
./configure ${opt}=${{ github.workspace }} --with-pcre=/usr/local/src/pcre --with-http_realip_module --with-cc-opt='-Wno-unused-but-set-variable -Wno-unused-function -fstack-protector-strong'
./configure ${opt}=${{ github.workspace }} --with-http_realip_module --with-cc-opt='-Wno-unused-but-set-variable -Wno-unused-function -fstack-protector-strong'
- name: Install nginx-${{ matrix.nginx-version }}
run: |
Expand All @@ -229,49 +117,3 @@ jobs:
export MODULE_TEST_PATH=/tmp/module_test
sh ./init.sh
exec sudo sh start.sh t/*.t
docker:
runs-on: ubuntu-latest
if: ${{ github.event_name != 'schedule' }}
strategy:
matrix:
version: ['stable', 'mainline']
target: ['glibc', 'musl']
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build module
run: docker build --file docker/Dockerfile.${{ matrix.target }} --build-arg=NGX_VER=${{ matrix.version }} .
triger:
runs-on: ubuntu-latest
if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/lts' || github.ref == 'refs/heads/current')) }}
needs: ['native', 'docker']
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- uses: actions/setup-python@v2
with:
python-version: '3.x'
- name: Set up lastversion
run: pip3 install lastversion
- name: 'Trigger workflow: docker'
if: ${{ github.ref == 'refs/heads/lts' || github.ref == 'refs/heads/current' }}
run: |
curl -H "Authorization: token ${{ secrets.ACCESS_TOKEN }}" \
-X POST \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/ADD-SP/ngx_waf/actions/workflows/docker.yml/dispatches \
-d "{ \
\"ref\":\"${{ github.ref }}\", \
\"inputs\": { \
\"ngx_stable_version_number\": \"`lastversion https://nginx.org --major stable`\", \
\"ngx_mainline_version_number\": \"`lastversion https://nginx.org --major mainline`\", \
\"module_version_number\": \"`cat assets/version.txt`\" \
} \
}"
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ lib/cjson
lib/uthash
test/test-nginx/t/servroot
test/test-nginx/t/*.t
test/test-nginx/gmon.out
test/test-nginx/gmon.out
bazel-*
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "ngx_waf_deps"]
path = ngx_waf_deps
url = https://github.com/ADD-SP/ngx_waf_deps
62 changes: 62 additions & 0 deletions BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@

genrule(
name = "deps",
srcs = [
"@uthash//:include/utarray.h",
"@uthash//:include/uthash.h",
"@uthash//:include/utlist.h",
"@uthash//:include/utringbuffer.h",
"@uthash//:include/utstack.h",
"@uthash//:include/utstring.h",
"@libsodium",
"@libmodsecurity",
"@libcjson",
"@libcjson//:cJSON.h",
],
outs = [
"deps.tar.gz"
],
cmd = """
mkdir -p deps/libsodium/include
mkdir -p deps/libsodium/lib
mkdir -p deps/uthash/include
mkdir -p deps/libmodsecurity/include
mkdir -p deps/libmodsecurity/lib
mkdir -p deps/libcjson/include
mkdir -p deps/libcjson/lib
cp -Lr $(location @uthash//:include/utarray.h) deps/uthash/include
cp -Lr $(location @uthash//:include/uthash.h) deps/uthash/include
cp -Lr $(location @uthash//:include/utlist.h) deps/uthash/include
cp -Lr $(location @uthash//:include/utringbuffer.h) deps/uthash/include
cp -Lr $(location @uthash//:include/utstack.h) deps/uthash/include
cp -Lr $(location @uthash//:include/utstring.h) deps/uthash/include
libsodium_base=$$(dirname $$(echo '$(locations @libsodium)' | awk '{print $$1}'))
cp -Lr $$libsodium_base/include/* deps/libsodium/include
cp -Lr $$libsodium_base/lib/* deps/libsodium/lib
libmodsecurity_base=$$(dirname $$(echo '$(locations @libmodsecurity)' | awk '{print $$1}'))
cp -Lr $$libmodsecurity_base/include/* deps/libmodsecurity/include
cp -Lr $$libmodsecurity_base/lib/* deps/libmodsecurity/lib
cp -Lr $(location @libcjson//:cJSON.h) deps/libcjson/include
for f in $(locations @libcjson); do
# copy .a and .so
if [ $$(echo $$f | grep -E '\\.a$$') ]; then
cp -L $$f deps/libcjson/lib
elif [ $$(echo $$f | grep -E '\\.so$$') ]; then
cp -L $$f deps/libcjson/lib
fi
done
# chmod
find deps -type d -exec chmod 755 {} +
find deps -type f -exec chmod 644 {} +
rm -f $(RULEDIR)/deps.tar.gz
tar -zcvf $(RULEDIR)/deps.tar.gz deps
rm -rf deps
""",
)
10 changes: 10 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
module(
name = "ngx_waf",
repo_name = "ngx_waf",
)

bazel_dep(name = "rules_foreign_cc", version = "0.10.1")
bazel_dep(name = "uthash", version = "2.3.0")
bazel_dep(name = "libsodium", version = "1.0.19")
bazel_dep(name = "libmodsecurity", version = "3.0.12")
bazel_dep(name = "libcjson", version = "1.7.17")
Loading

0 comments on commit 438190c

Please sign in to comment.