From 52b13f1c953037f9d0b606c6c1fd9cf56a7bdb81 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Wed, 30 Oct 2024 10:58:21 +0100 Subject: [PATCH 01/15] test(dockerfile: toolchain): update LLVM version to 16.0.6 Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/build.gradle | 2 +- appsec/tests/integration/gradle/images.gradle | 2 +- .../src/docker/toolchain/CHECKSUMS | 8 - .../src/docker/toolchain/Dockerfile | 28 +-- .../integration/src/docker/toolchain/Makefile | 161 ------------------ .../src/docker/toolchain/ToolchainGCC.cmake | 1 - .../src/docker/toolchain/alltypes.h.diff | 8 +- .../src/docker/toolchain/locale.h.diff | 2 +- 8 files changed, 24 insertions(+), 188 deletions(-) delete mode 100644 appsec/tests/integration/src/docker/toolchain/Makefile diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 95dde0a654..077b6de4fd 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -329,7 +329,7 @@ def buildAppSecTask = { String version, String variant -> ''' git config --global --add safe.directory '*' cd /appsec - test -f CMakeCache.txt || \\ + // test -f CMakeCache.txt || \\ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ -DCMAKE_INSTALL_PREFIX=/appsec \\ -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\ diff --git a/appsec/tests/integration/gradle/images.gradle b/appsec/tests/integration/gradle/images.gradle index f590530456..a4652f4cfb 100644 --- a/appsec/tests/integration/gradle/images.gradle +++ b/appsec/tests/integration/gradle/images.gradle @@ -33,7 +33,7 @@ tasks.register('buildToolchain', Exec) { description = "Build the toolchain image" inputs.dir 'src/docker/toolchain' outputs.upToDateWhen imageUpToDate(inputs, "$repo:toolchain") - commandLine 'docker', 'build', '-t', "$repo:toolchain", 'src/docker/toolchain' + commandLine 'docker', 'build', '-t', "$repo:toolchain", 'src/docker/toolchain', "--build-arg", "ARCH=$arch" } tasks.register('buildPhpDeps', Exec) { diff --git a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS index 8c7b6ff368..f44d511fd9 100644 --- a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS +++ b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS @@ -1,10 +1,2 @@ -9b0b259cc43d5e4d20200676be153de81b485b3fdf065623ff71c7e1894ec5c8ed9d99c1416f70246ab0f417cbba6d1d4af9f3769e2e6577400681a791346231 compiler-rt-11.1.0.src.tar.xz -a1d2e3f5ad529a04f87059903b31fc3c9803cd86f44aed1aebd87ce7e423d8dd2b6776be12e85a0374a6215f581420438d224c130aad5e6355920af32c02aa7b compiler-rt-10.0.1.src.tar.xz -d77145858cda538127b631a8072fafc3fa01a5a9648d4ca1cb6d563009061c56f93ca606f4f7f9e706d5cc0dd8f4e0895f496439ca173f175a1c8ee740b5d30a gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -af5333da5b90f4a46a5184532164f4c6522e3c03a580131627c0f167ab98fb3e71b3e15518d6e22414141484ec5ab0d184294ae7f10034ebfed28e7072836b28 libcxx-11.1.0.src.tar.xz -0bf3806fd9382ca6790ca2a8e991424caf64e81415386875243565034243f2ac7442c596e3c55ece80932c2ec59b71801e3e415dedc9db4dd4c3f66b6a893558 libcxxabi-11.1.0.src.tar.xz -507f29cf1a318d9761fe6306b2e9b57c02a342f138b47ec5420dce527132a33f7affcd386913792c472ceeb9fb1c1b105bba3234a1575aae0f68024e94c8d596 libunwind-11.1.0.src.tar.xz -07bf9973384151a18d5cc2892103e5f28a88c632e8e49662fde56d123632f2ed1b3710fa7a87b6b821955d0ec44160ff36f2aa4f233e389e14d628e9bf8dc764 llvm-11.1.0.src.tar.xz -5344b581bd6463d71af8c13e91792fa51f25a96a1ecbea81e42664b63d90b325aeb421dfbc8c22e187397ca08e84d9296a0c0c299ba04fa2b751d6864914bd82 musl-1.2.2.tar.gz 9591360672ba6192c606404caf70101538728a1cd5d548efcbb952f663f182bd1954d63743ffc9dd18f5c649a62a042c5e36d1ff423634dfd074f672dd1f4af9 cmake-3.28.0-linux-x86_64.tar.gz 48a20095711870b23bd5db342de0e058a7c6876bafad4c6ce9ff9bce672ca1e95ed9ac890d519b0884cd277d091575eda7e60a97cad377ee57c1e20dee25feb1 cmake-3.28.0-linux-aarch64.tar.gz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index 07867d0d93..b5d9aabf91 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -1,14 +1,20 @@ -FROM debian@sha256:08db48d59c0a91afb802ebafc921be3154e200c452e4d0b19634b426b03e0e25 AS toolchain -RUN apt-get update && \ - apt-get install -y curl xz-utils make file lld clang git patchelf gcc libgcc-s1 sed autoconf wget libssl-dev wget libxml2 +FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain -RUN ln -s /bin/sed /usr/bin/sed -RUN mkdir /build -ADD . /build/ +ARG LLVM_VERSION=16 +ARG ARCH -RUN wget https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ - grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ - tar --strip-components=1 -C /usr/local -xvzf cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ - rm cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz +COPY . /build -RUN cd /build && make install && make clean +RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" + +RUN apt-get update -y && \ + apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 + +RUN wget "https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ + grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ + tar --strip-components=1 -C /usr/local -xvzf "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ + rm "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all diff --git a/appsec/tests/integration/src/docker/toolchain/Makefile b/appsec/tests/integration/src/docker/toolchain/Makefile deleted file mode 100644 index cc09881c17..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/Makefile +++ /dev/null @@ -1,161 +0,0 @@ -MUSL_VERSION := 1.2.2 -LLVM_VERSION := 11.1.0 -LLVM_SUFFIX := 11 -SHELL := /bin/bash -RELTYPE := RelWithDebInfo - -# need to be in sync with Toolchain*.cmake files -MUSL_SYSROOT := $(CURDIR)/muslsysroot - -TARGET_ARCH := $(shell arch) -TARGET := $(TARGET_ARCH)-none-linux-musl - - -install: $(MUSL_SYSROOT)/lib/libglibc_compat.a .libcxx-installed .libcxxabi-installed -clean: - rm -rf src/ build/ *.tar.xz *.tar.gz \ - .compiler-rt-installed .gcc-toolchain-installed .libcxxabi-installed \ - .libcxx-installed .libunwind-installed .musl-installed -.PHONY: install clean - -CC_TOOLCHAIN := /usr -GCC_TOOL_PREFIX := /usr/bin/ -GCC_TOOLCHAIN_SYSROOT := / - -$(MUSL_SYSROOT)/lib/libglibc_compat.a: glibc_compat.c .musl-installed - mkdir -p $(MUSL_SYSROOT)/lib - clang --sysroot $(MUSL_SYSROOT) -fpie -O2 -fno-omit-frame-pointer \ - -ggdb3 -c glibc_compat.c -o /tmp/glibc_compat.o && \ - ar rcs $@ /tmp/glibc_compat.o && \ - rm /tmp/glibc_compat.o - -.gcc-toolchain-installed: - cp -v /lib/$(TARGET_ARCH)-linux-gnu/libgcc_s.so.1 /usr/lib/gcc/$(TARGET_ARCH)-linux-gnu/10/libgcc_s.so.1 - touch $@ - -musl-$(MUSL_VERSION).tar.gz: - curl -o $@ -Lf https://musl.libc.org/releases/musl-1.2.2.tar.gz - grep -F $@ CHECKSUMS | sha512sum --check - -libcxx-$(LLVM_VERSION).src.tar.xz \ - libcxxabi-$(LLVM_VERSION).src.tar.xz \ - libunwind-$(LLVM_VERSION).src.tar.xz \ - llvm-$(LLVM_VERSION).src.tar.xz \ - compiler-rt-$(LLVM_VERSION).src.tar.xz: - curl -o $@ -Lf https://github.com/llvm/llvm-project/releases/download/llvmorg-$(LLVM_VERSION)/$@ - grep -F $@ CHECKSUMS | sha512sum --check - - -src/musl/.finger: musl-$(MUSL_VERSION).tar.gz - mkdir -p src/musl && \ - tar -xzf musl-$(MUSL_VERSION).tar.gz --strip-components=1 -C src/musl && \ - touch src/musl/.finger - -src/%/.finger: %-$(LLVM_VERSION).src.tar.xz - mkdir -p $(subst /.finger,,$@) && \ - tar -xJf $< --strip-components=1 -C $(subst /.finger,,$@) && \ - touch $@ - -.musl-installed: src/musl/.finger .gcc-toolchain-installed - mkdir -p build/musl && \ - pushd build/musl && \ - CC=$(GCC_TOOL_PREFIX)gcc \ - AR=$(GCC_TOOL_PREFIX)ar \ - RANLIB=$(GCC_TOOL_PREFIX)ranlib \ - ../../src/musl/configure --prefix=$(MUSL_SYSROOT) && \ - $(MAKE) -j $(shell nproc) && \ - $(MAKE) install && \ - popd && \ - pushd $(MUSL_SYSROOT)/include && \ - patch -p0 < /build/locale.h.diff && \ - patch -p0 < /build/alltypes.h.diff && \ - popd && \ - touch $@ - -VERBOSE := 1 -export VERBOSE -COMMON_CXX_FLAGS := -DCMAKE_CXX_FLAGS="-resource-dir $(MUSL_SYSROOT)" -COMMON_CMAKE_OPTIONS := -DCMAKE_BUILD_TYPE=$(RELTYPE) \ - -DCMAKE_INSTALL_PREFIX=$(MUSL_SYSROOT) \ - -DCMAKE_SYSROOT=$(MUSL_SYSROOT) \ - -DCMAKE_AR=/usr/bin/llvm-ar-${LLVM_SUFFIX} \ - -DCMAKE_ASM_COMPILER_TARGET=$(TARGET) \ - -DCMAKE_C_COMPILER=/usr/bin/clang-${LLVM_SUFFIX} \ - -DCMAKE_C_COMPILER_TARGET=$(TARGET) \ - -DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN=$(GCC_TOOLCHAIN) \ - -DCMAKE_C_FLAGS="-resource-dir $(MUSL_SYSROOT)" \ - -DCMAKE_CXX_COMPILER=/usr/bin/clang++-${LLVM_SUFFIX} \ - -DCMAKE_CXX_COMPILER_TARGET=$(TARGET) \ - -DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN=$(GCC_TOOLCHAIN) \ - -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS="-v -fuse-ld=lld" \ - -DCMAKE_NM=/usr/bin/llvm-nm-${LLVM_SUFFIX} \ - -DCMAKE_RANLIB=/usr/bin/llvm-ranlib-${LLVM_SUFFIX} - -.compiler-rt-installed: src/compiler-rt/.finger .musl-installed - mkdir -p build/compiler-rt && \ - pushd build/compiler-rt && \ - cmake $(COMMON_CMAKE_OPTIONS) $(COMMON_CXX_FLAGS) \ - -DCOMPILER_RT_BUILD_BUILTINS=ON \ - -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ - -DCOMPILER_RT_BUILD_MEMPROF=OFF \ - -DCOMPILER_RT_BUILD_PROFILE=OFF \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ - -DCOMPILER_RT_BUILD_CRT=ON \ - ../../src/compiler-rt && \ - $(MAKE) -j $(shell nproc) && $(MAKE) install && \ - popd && \ - touch $@ - -.libunwind-installed: src/libunwind/.finger src/libcxx/.finger .compiler-rt-installed - mkdir -p build/libunwind && \ - pushd build/libunwind && \ - cmake $(COMMON_CMAKE_OPTIONS) \ - -DCMAKE_CXX_FLAGS="-resource-dir $(MUSL_SYSROOT) -nostdinc++ -isystem $(realpath .)/src/libcxx/include/" \ - -DLLVM_PATH=../../src/llvm \ - -DLIBUNWIND_USE_COMPILER_RT=ON \ - ../../src/libunwind && \ - $(MAKE) -j $(shell nproc) && $(MAKE) install && \ - popd && \ - touch $@ - -.libcxxabi-installed: src/libcxxabi/.finger src/libcxx/.finger src/llvm/.finger src/libunwind/.finger .compiler-rt-installed .libunwind-installed - mkdir -p build/libcxxabi && \ - pushd build/libcxxabi && \ - cmake $(COMMON_CMAKE_OPTIONS) $(COMMON_CXX_FLAGS) \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_INCLUDE_TESTS=OFF \ - -DLIBCXXABI_USE_COMPILER_RT=ON \ - -DLLVM_PATH=../../src/llvm \ - -DLIBCXXABI_LIBUNWIND_PATH=../../src/libunwind \ - -DLIBCXXABI_LIBCXX_INCLUDES=../../src/libcxx/include \ - ../../src/libcxxabi && \ - $(MAKE) -j $(shell nproc) && $(MAKE) install && \ - popd && \ - touch $@ - -.orig-sysroot-copied: .musl-installed - for dir in $(GCC_TOOLCHAIN_SYSROOT)usr/include/{linux,asm,asm-generic,$$($(GCC_TOOL_PREFIX)gcc -print-multiarch)/asm}; do \ - test ! -d "$$dir" || cp -av "$$dir" $(MUSL_SYSROOT)/include/; done && \ - cp $$(dirname $$($(GCC_TOOL_PREFIX)gcc -print-libgcc-file-name))/crtbegin{,S,T}.o $(MUSL_SYSROOT)/lib && \ - cp $$(dirname $$($(GCC_TOOL_PREFIX)gcc -print-libgcc-file-name))/crtend{,S}.o $(MUSL_SYSROOT)/lib && \ - cp $(GCC_TOOLCHAIN_SYSROOT)usr/include/$$($(GCC_TOOL_PREFIX)gcc -print-multiarch)/fpu_control.h $(MUSL_SYSROOT)/include/ && \ - touch $@ - - -.libcxx-installed: src/libcxx/.finger src/llvm/.finger src/libunwind/.finger .orig-sysroot-copied .compiler-rt-installed .libunwind-installed .libcxxabi-installed - mkdir -p build/libcxx && \ - pushd build/libcxx && \ - cmake $(COMMON_CMAKE_OPTIONS) $(COMMON_CXX_FLAGS) \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_CXX_ABI=libcxxabi \ - -DLIBCXX_CXX_ABI_INCLUDE_PATHS=../../src/libcxxabi/include \ - -DLIBCXX_USE_COMPILER_RT=ON \ - -DLIBCXX_ENABLE_FILESYSTEM=ON \ - -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=0 \ - ../../src/libcxx && \ - $(MAKE) -j $(shell nproc) && $(MAKE) install && \ - popd && \ - touch $@ diff --git a/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake b/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake index d147056712..f5a442e915 100644 --- a/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake +++ b/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake @@ -11,4 +11,3 @@ set(CMAKE_CXX_COMPILER_TARGET ${target}) set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN /usr) set(CMAKE_NM ${tool_prefix}nm) set(CMAKE_RANLIB ${tool_prefix}ranlib) - diff --git a/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff b/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff index bd9b86ffef..daa64e502d 100644 --- a/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff +++ b/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff @@ -1,8 +1,8 @@ --- bits/alltypes.h 2023-11-06 11:49:18.000000000 +0000 +++ bits/alltypes.h 2024-04-22 09:30:09.927560000 +0000 @@ -383,12 +383,20 @@ - - + + #if defined(__NEED_pthread_attr_t) && !defined(__DEFINED_pthread_attr_t) -typedef struct { union { int __i[sizeof(long)==8?14:9]; volatile int __vi[sizeof(long)==8?14:9]; unsigned long __s[sizeof(long)==8?7:9]; } __u; } pthread_attr_t; +typedef struct { union { int __i[sizeof(long)==8?14:9]; volatile int __vi[sizeof(long)==8?14:9]; unsigned long __s[sizeof(long)==8?7:9]; @@ -12,7 +12,7 @@ +} __u; } pthread_attr_t; #define __DEFINED_pthread_attr_t #endif - + #if defined(__NEED_pthread_mutex_t) && !defined(__DEFINED_pthread_mutex_t) -typedef struct { union { int __i[sizeof(long)==8?10:6]; volatile int __vi[sizeof(long)==8?10:6]; volatile void *volatile __p[sizeof(long)==8?5:6]; } __u; } pthread_mutex_t; +typedef struct { union { int __i[sizeof(long)==8?10:6]; volatile int __vi[sizeof(long)==8?10:6]; volatile void *volatile __p[sizeof(long)==8?5:6]; @@ -22,4 +22,4 @@ +} __u; } pthread_mutex_t; #define __DEFINED_pthread_mutex_t #endif - + diff --git a/appsec/tests/integration/src/docker/toolchain/locale.h.diff b/appsec/tests/integration/src/docker/toolchain/locale.h.diff index 36de614b57..408367ec34 100644 --- a/appsec/tests/integration/src/docker/toolchain/locale.h.diff +++ b/appsec/tests/integration/src/docker/toolchain/locale.h.diff @@ -6,6 +6,6 @@ #define LC_MESSAGES_MASK (1< Date: Mon, 4 Nov 2024 13:56:40 +0100 Subject: [PATCH 02/15] test(dockerfile: toolchain): wip Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/build.gradle | 1 - .../src/docker/toolchain/CHECKSUMS | 3 +- .../src/docker/toolchain/Dockerfile | 54 +++++++++++++++---- .../src/docker/toolchain/Dockerfile.bak | 15 ++++++ .../src/docker/toolchain/Toolchain.cmake | 19 ++++--- .../src/docker/toolchain/Toolchain.env | 8 --- .../src/docker/toolchain/locale.h.diff | 32 +++++++---- .../src/docker/toolchain/wchar.h.diff | 11 ++++ 8 files changed, 101 insertions(+), 42 deletions(-) create mode 100644 appsec/tests/integration/src/docker/toolchain/Dockerfile.bak delete mode 100644 appsec/tests/integration/src/docker/toolchain/Toolchain.env create mode 100644 appsec/tests/integration/src/docker/toolchain/wchar.h.diff diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 077b6de4fd..ed9f94bb8a 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -329,7 +329,6 @@ def buildAppSecTask = { String version, String variant -> ''' git config --global --add safe.directory '*' cd /appsec - // test -f CMakeCache.txt || \\ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ -DCMAKE_INSTALL_PREFIX=/appsec \\ -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\ diff --git a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS index f44d511fd9..d62c3ab33f 100644 --- a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS +++ b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS @@ -1,2 +1 @@ -9591360672ba6192c606404caf70101538728a1cd5d548efcbb952f663f182bd1954d63743ffc9dd18f5c649a62a042c5e36d1ff423634dfd074f672dd1f4af9 cmake-3.28.0-linux-x86_64.tar.gz -48a20095711870b23bd5db342de0e058a7c6876bafad4c6ce9ff9bce672ca1e95ed9ac890d519b0884cd277d091575eda7e60a97cad377ee57c1e20dee25feb1 cmake-3.28.0-linux-aarch64.tar.gz +89a67ebfbbc764cc456e8825ecfa90707741f8835b1b2adffae0b227ab1fe5ca9cce75b0efaffc9ca8431cae528dc54fd838867a56a2b645344d9e82d19ab1b7 llvm-project-16.0.6.src.tar.xz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index b5d9aabf91..6c1fe3949c 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -1,20 +1,52 @@ -FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain +FROM debian:latest AS toolchain -ARG LLVM_VERSION=16 +ARG LLVM_VERSION=16.0.6 ARG ARCH -COPY . /build +COPY Toolchain.cmake /build/Toolchain.cmake +COPY CHECKSUMS /CHECKSUMS RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" -RUN apt-get update -y && \ - apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 - -RUN wget "https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ - grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ - tar --strip-components=1 -C /usr/local -xvzf "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ - rm "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" +RUN apt-get update && apt-get install -y \ + wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} all + ./llvm.sh 16 all + +RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \ + grep -F llvm-project-${LLVM_VERSION}.src.tar.xz /CHECKSUMS | sha512sum --check && \ + tar -xvf llvm-project-${LLVM_VERSION}.src.tar.xz + +COPY wchar.h.diff /wchar.h.diff +RUN patch /usr/include/wchar.h < /wchar.h.diff + +RUN cd llvm-project-${LLVM_VERSION}.src && mkdir -p build && cd build && \ + cmake \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_C_COMPILER=clang-16 \ + -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ + -DCMAKE_CXX_COMPILER=clang++-16 \ + -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DLIBUNWIND_ENABLE_STATIC=ON \ + -DLIBUNWIND_USE_COMPILER_RT=ON \ + -DLIBCXXABI_ENABLE_SHARED=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ + -DLIBCXXABI_USE_COMPILER_RT=ON \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_HAS_MUSL_LIBC=ON \ + -DLIBCXX_USE_COMPILER_RT=ON \ + -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_EXTERNAL_LIT=/usr/bin/lit ../runtimes && \ + make -j$(nproc) install-unwind install + +RUN cd /usr/lib && ln -s gcc/*/*/ resource_dir +RUN cd /usr/lib && ln -s clang/${LLVM_VERSION%%.*}/lib/linux/libclang_rt.builtins-*.a libclang_rt.builtins.a + +RUN rm -rf /llvm-project-${LLVM_VERSION}.src +RUN rm -f llvm-project-${LLVM_VERSION}.src.tar.xz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak b/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak new file mode 100644 index 0000000000..904a99093e --- /dev/null +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak @@ -0,0 +1,15 @@ +FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain + +ARG LLVM_VERSION=16 +ARG ARCH + +COPY . /build + +RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" + +RUN apt-get update -y && \ + apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 cmake + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all diff --git a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake b/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake index a444fc21c1..b88959a21f 100644 --- a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake +++ b/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake @@ -9,22 +9,21 @@ if(ARCHITECTURE MATCHES "x86_64") else() set(ARCH aarch64) endif() -set(CMAKE_SYSROOT /build/muslsysroot) -set(CMAKE_AR /usr/bin/llvm-ar-11) +set(CMAKE_AR /usr/bin/llvm-ar-16) set(triple ${ARCH}-none-linux-musl) set(CMAKE_ASM_COMPILER_TARGET ${triple}) -set(CMAKE_C_COMPILER /usr/bin/clang-11) +set(CMAKE_C_COMPILER /usr/bin/clang-16) set(CMAKE_C_COMPILER_TARGET ${triple}) -set(c_cxx_flags "-nostdinc -isystem${CMAKE_SYSROOT}/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir ${CMAKE_SYSROOT} -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc") +set(c_cxx_flags "-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer") set(CMAKE_C_FLAGS_INIT ${c_cxx_flags}) -set(CMAKE_CXX_COMPILER /usr/bin/clang++-11) +set(CMAKE_CXX_COMPILER /usr/bin/clang++-16) set(CMAKE_CXX_COMPILER_TARGET ${triple}) -set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem${CMAKE_SYSROOT}/include/c++/v1 ${c_cxx_flags}") -set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a") -set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a") +set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem/usr/lib/clang/16.0.6/include/c++/v1 ${c_cxx_flags}") +set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") +set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") set(CMAKE_C_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") set(CMAKE_CXX_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") -set(CMAKE_NM /usr/bin/llvm-nm-11) -set(CMAKE_RANLIB /usr/bin/llvm-ranlib-11) +set(CMAKE_NM /usr/bin/llvm-nm-16) +set(CMAKE_RANLIB /usr/bin/llvm-ranlib-16) set(CMAKE_STRIP /usr/bin/strip) # llvm-strip doesn't seem to work correctly diff --git a/appsec/tests/integration/src/docker/toolchain/Toolchain.env b/appsec/tests/integration/src/docker/toolchain/Toolchain.env deleted file mode 100644 index 7403fda0d3..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/Toolchain.env +++ /dev/null @@ -1,8 +0,0 @@ -export CXXFLAGS="-stdlib=libc++ -isystem/build/muslsysroot/include/c++/v1 -nostdinc -isystem/build/muslsysroot/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir /build/muslsysroot -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc" -export CFLAGS="-nostdinc -isystem/build/muslsysroot/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir /build/muslsysroot -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc" -export LDFLAGS="-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /build/muslsysroot/lib/linux/libclang_rt.builtins-x86_64.a -lunwind -lc /build/muslsysroot/lib/linux/libclang_rt.builtins-x86_64.a" -export CC="/usr/bin/clang-11" -export CXX="/usr/bin/clang++-11" -export AR="/usr/bin/llvm-ar-11" -export NM="/usr/bin/llvm-nm-11" -export RANLIB="/usr/bin/llvm-ranlib-11" diff --git a/appsec/tests/integration/src/docker/toolchain/locale.h.diff b/appsec/tests/integration/src/docker/toolchain/locale.h.diff index 408367ec34..52d046665a 100644 --- a/appsec/tests/integration/src/docker/toolchain/locale.h.diff +++ b/appsec/tests/integration/src/docker/toolchain/locale.h.diff @@ -1,11 +1,23 @@ --- locale.h -+++ locale.h -@@ -71,7 +71,7 @@ - #define LC_COLLATE_MASK (1< Date: Tue, 5 Nov 2024 16:20:52 +0100 Subject: [PATCH 03/15] test(dockerfile: toolchain): simplify Dockerfile Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/build.gradle | 3 +- .../src/docker/toolchain/Dockerfile | 38 +------------------ 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index ed9f94bb8a..4faa03b7fb 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -329,10 +329,9 @@ def buildAppSecTask = { String version, String variant -> ''' git config --global --add safe.directory '*' cd /appsec - cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ -DCMAKE_INSTALL_PREFIX=/appsec \\ -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\ - -DCMAKE_TOOLCHAIN_FILE=/build/Toolchain.cmake \\ -DDD_APPSEC_TESTING=ON /project/appsec make -j extension ddappsec-helper && \\ touch ddappsec.so libddappsec-helper.so diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index 6c1fe3949c..4600bec712 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -9,44 +9,10 @@ COPY CHECKSUMS /CHECKSUMS RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" RUN apt-get update && apt-get install -y \ - wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg + build-essential wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ ./llvm.sh 16 all -RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \ - grep -F llvm-project-${LLVM_VERSION}.src.tar.xz /CHECKSUMS | sha512sum --check && \ - tar -xvf llvm-project-${LLVM_VERSION}.src.tar.xz - -COPY wchar.h.diff /wchar.h.diff -RUN patch /usr/include/wchar.h < /wchar.h.diff - -RUN cd llvm-project-${LLVM_VERSION}.src && mkdir -p build && cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_C_COMPILER=clang-16 \ - -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ - -DCMAKE_CXX_COMPILER=clang++-16 \ - -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_ENABLE_STATIC=ON \ - -DLIBUNWIND_USE_COMPILER_RT=ON \ - -DLIBCXXABI_ENABLE_SHARED=ON \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ - -DLIBCXXABI_USE_COMPILER_RT=ON \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_USE_COMPILER_RT=ON \ - -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_EXTERNAL_LIT=/usr/bin/lit ../runtimes && \ - make -j$(nproc) install-unwind install - -RUN cd /usr/lib && ln -s gcc/*/*/ resource_dir -RUN cd /usr/lib && ln -s clang/${LLVM_VERSION%%.*}/lib/linux/libclang_rt.builtins-*.a libclang_rt.builtins.a - -RUN rm -rf /llvm-project-${LLVM_VERSION}.src -RUN rm -f llvm-project-${LLVM_VERSION}.src.tar.xz +RUN ln -s /usr/bin/cmake /usr/local/bin/cmake From f2143b742c590e4e61e3b03543719af4d3edc76c Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:38:43 +0000 Subject: [PATCH 04/15] Fixes: * Create linux_users group with gid=1000 * Create linux_user with uid=1000 * Simplify toolchain and include cmake release * Reinstate checksums and delete unnecessary files --- .../src/docker/apache2-fpm/Dockerfile | 6 +- .../src/docker/apache2-mod/Dockerfile | 6 +- .../src/docker/nginx-fpm/Dockerfile | 6 +- .../integration/src/docker/php/Dockerfile | 3 + .../src/docker/toolchain/CHECKSUMS | 3 +- .../src/docker/toolchain/Dockerfile | 23 +-- .../src/docker/toolchain/Dockerfile.bak | 15 -- .../src/docker/toolchain/Toolchain.cmake | 29 --- .../src/docker/toolchain/ToolchainGCC.cmake | 13 -- .../src/docker/toolchain/alltypes.h.diff | 25 --- .../src/docker/toolchain/glibc_compat.c | 187 ------------------ .../src/docker/toolchain/locale.h.diff | 23 --- .../src/docker/toolchain/wchar.h.diff | 11 -- 13 files changed, 26 insertions(+), 324 deletions(-) delete mode 100644 appsec/tests/integration/src/docker/toolchain/Dockerfile.bak delete mode 100644 appsec/tests/integration/src/docker/toolchain/Toolchain.cmake delete mode 100644 appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake delete mode 100644 appsec/tests/integration/src/docker/toolchain/alltypes.h.diff delete mode 100644 appsec/tests/integration/src/docker/toolchain/glibc_compat.c delete mode 100644 appsec/tests/integration/src/docker/toolchain/locale.h.diff delete mode 100644 appsec/tests/integration/src/docker/toolchain/wchar.h.diff diff --git a/appsec/tests/integration/src/docker/apache2-fpm/Dockerfile b/appsec/tests/integration/src/docker/apache2-fpm/Dockerfile index dbf502e566..72fc5b9210 100644 --- a/appsec/tests/integration/src/docker/apache2-fpm/Dockerfile +++ b/appsec/tests/integration/src/docker/apache2-fpm/Dockerfile @@ -3,8 +3,10 @@ ARG VARIANT FROM datadog/dd-appsec-php-ci:php-$PHP_VERSION-$VARIANT RUN apt-get update && apt-get install -y \ - apache2 \ - && rm -rf /var/lib/apt/lists/* + apache2 \ + && rm -rf /var/lib/apt/lists/* + +RUN usermod -a -G linux_users www-data RUN rm -rf /var/www/html RUN sed -i '//,/<\/Directory>/s/AllowOverride None/AllowOverride All/' /etc/apache2/apache2.conf diff --git a/appsec/tests/integration/src/docker/apache2-mod/Dockerfile b/appsec/tests/integration/src/docker/apache2-mod/Dockerfile index 9d9e1ae252..c15985d1fd 100644 --- a/appsec/tests/integration/src/docker/apache2-mod/Dockerfile +++ b/appsec/tests/integration/src/docker/apache2-mod/Dockerfile @@ -3,8 +3,10 @@ ARG VARIANT FROM datadog/dd-appsec-php-ci:php-$PHP_VERSION-$VARIANT RUN apt-get update && apt-get install -y \ - apache2 \ - && rm -rf /var/lib/apt/lists/* + apache2 \ + && rm -rf /var/lib/apt/lists/* + +RUN usermod -a -G linux_users www-data RUN rm -rf /var/www/html RUN sed -i 's@/var/www/html@/var/www/public@' /etc/apache2/sites-available/000-default.conf diff --git a/appsec/tests/integration/src/docker/nginx-fpm/Dockerfile b/appsec/tests/integration/src/docker/nginx-fpm/Dockerfile index f9475e2b4e..1c9b1fc67d 100644 --- a/appsec/tests/integration/src/docker/nginx-fpm/Dockerfile +++ b/appsec/tests/integration/src/docker/nginx-fpm/Dockerfile @@ -4,8 +4,10 @@ ARG VARIANT FROM datadog/dd-appsec-php-ci:php-$PHP_VERSION-$VARIANT RUN apt-get update && apt-get install -y \ - nginx \ - && rm -rf /var/lib/apt/lists/* + nginx \ + && rm -rf /var/lib/apt/lists/* + +RUN usermod -a -G linux_users www-data RUN rm -rf /var/www/html diff --git a/appsec/tests/integration/src/docker/php/Dockerfile b/appsec/tests/integration/src/docker/php/Dockerfile index e85c7f012b..00900e3ad8 100644 --- a/appsec/tests/integration/src/docker/php/Dockerfile +++ b/appsec/tests/integration/src/docker/php/Dockerfile @@ -35,4 +35,7 @@ ENV CARGO_HOME=/root/.cargo ENV RUSTUP_HOME=/root/.rustup ENV LD_LIBRARY_PATH=/root/php/icu-60/lib +RUN groupadd -g 1000 linux_users && \ + useradd -G linux_users -u 1000 linux_user + ENTRYPOINT ["php"] diff --git a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS index d62c3ab33f..f44d511fd9 100644 --- a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS +++ b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS @@ -1 +1,2 @@ -89a67ebfbbc764cc456e8825ecfa90707741f8835b1b2adffae0b227ab1fe5ca9cce75b0efaffc9ca8431cae528dc54fd838867a56a2b645344d9e82d19ab1b7 llvm-project-16.0.6.src.tar.xz +9591360672ba6192c606404caf70101538728a1cd5d548efcbb952f663f182bd1954d63743ffc9dd18f5c649a62a042c5e36d1ff423634dfd074f672dd1f4af9 cmake-3.28.0-linux-x86_64.tar.gz +48a20095711870b23bd5db342de0e058a7c6876bafad4c6ce9ff9bce672ca1e95ed9ac890d519b0884cd277d091575eda7e60a97cad377ee57c1e20dee25feb1 cmake-3.28.0-linux-aarch64.tar.gz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index 4600bec712..4c1cc7d069 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -1,18 +1,13 @@ -FROM debian:latest AS toolchain - -ARG LLVM_VERSION=16.0.6 -ARG ARCH - -COPY Toolchain.cmake /build/Toolchain.cmake -COPY CHECKSUMS /CHECKSUMS - -RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" +FROM debian@sha256:08db48d59c0a91afb802ebafc921be3154e200c452e4d0b19634b426b03e0e25 AS toolchain RUN apt-get update && apt-get install -y \ - build-essential wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg + build-essential wget binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg -RUN wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - ./llvm.sh 16 all +RUN ln -s /bin/sed /usr/bin/sed +RUN mkdir /build +ADD . /build/ -RUN ln -s /usr/bin/cmake /usr/local/bin/cmake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ + grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ + tar --strip-components=1 -C /usr/local -xvzf cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ + rm cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak b/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak deleted file mode 100644 index 904a99093e..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak +++ /dev/null @@ -1,15 +0,0 @@ -FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain - -ARG LLVM_VERSION=16 -ARG ARCH - -COPY . /build - -RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" - -RUN apt-get update -y && \ - apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 cmake - -RUN wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} all diff --git a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake b/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake deleted file mode 100644 index b88959a21f..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake +++ /dev/null @@ -1,29 +0,0 @@ -set(CMAKE_SYSTEM_NAME Linux) -execute_process( - COMMAND arch - OUTPUT_VARIABLE ARCHITECTURE - OUTPUT_STRIP_TRAILING_WHITESPACE -) -if(ARCHITECTURE MATCHES "x86_64") - set(ARCH x86_64) -else() - set(ARCH aarch64) -endif() -set(CMAKE_AR /usr/bin/llvm-ar-16) -set(triple ${ARCH}-none-linux-musl) -set(CMAKE_ASM_COMPILER_TARGET ${triple}) -set(CMAKE_C_COMPILER /usr/bin/clang-16) -set(CMAKE_C_COMPILER_TARGET ${triple}) -set(c_cxx_flags "-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer") -set(CMAKE_C_FLAGS_INIT ${c_cxx_flags}) -set(CMAKE_CXX_COMPILER /usr/bin/clang++-16) -set(CMAKE_CXX_COMPILER_TARGET ${triple}) -set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem/usr/lib/clang/16.0.6/include/c++/v1 ${c_cxx_flags}") -set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") -set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") -set(CMAKE_C_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") -set(CMAKE_CXX_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") - -set(CMAKE_NM /usr/bin/llvm-nm-16) -set(CMAKE_RANLIB /usr/bin/llvm-ranlib-16) -set(CMAKE_STRIP /usr/bin/strip) # llvm-strip doesn't seem to work correctly diff --git a/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake b/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake deleted file mode 100644 index f5a442e915..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/ToolchainGCC.cmake +++ /dev/null @@ -1,13 +0,0 @@ -set(target x86_64-none-linux-gnu) -set(tool_prefix "") -set(CMAKE_SYSROOT /) -set(CMAKE_AR ${tool_prefix}ar) -set(CMAKE_ASM_COMPILER_TARGET ${target}) -set(CMAKE_C_COMPILER ${tool_prefix}gcc) -set(CMAKE_C_COMPILER_TARGET ${target}) -set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN /usr) -set(CMAKE_CXX_COMPILER ${tool_prefix}g++) -set(CMAKE_CXX_COMPILER_TARGET ${target}) -set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN /usr) -set(CMAKE_NM ${tool_prefix}nm) -set(CMAKE_RANLIB ${tool_prefix}ranlib) diff --git a/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff b/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff deleted file mode 100644 index daa64e502d..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/alltypes.h.diff +++ /dev/null @@ -1,25 +0,0 @@ ---- bits/alltypes.h 2023-11-06 11:49:18.000000000 +0000 -+++ bits/alltypes.h 2024-04-22 09:30:09.927560000 +0000 -@@ -383,12 +383,20 @@ - - - #if defined(__NEED_pthread_attr_t) && !defined(__DEFINED_pthread_attr_t) --typedef struct { union { int __i[sizeof(long)==8?14:9]; volatile int __vi[sizeof(long)==8?14:9]; unsigned long __s[sizeof(long)==8?7:9]; } __u; } pthread_attr_t; -+typedef struct { union { int __i[sizeof(long)==8?14:9]; volatile int __vi[sizeof(long)==8?14:9]; unsigned long __s[sizeof(long)==8?7:9]; -+#ifdef __aarch64__ -+ char __glibc_compat[64]; -+#endif -+} __u; } pthread_attr_t; - #define __DEFINED_pthread_attr_t - #endif - - #if defined(__NEED_pthread_mutex_t) && !defined(__DEFINED_pthread_mutex_t) --typedef struct { union { int __i[sizeof(long)==8?10:6]; volatile int __vi[sizeof(long)==8?10:6]; volatile void *volatile __p[sizeof(long)==8?5:6]; } __u; } pthread_mutex_t; -+typedef struct { union { int __i[sizeof(long)==8?10:6]; volatile int __vi[sizeof(long)==8?10:6]; volatile void *volatile __p[sizeof(long)==8?5:6]; -+#ifdef __aarch64__ -+ char __glibc_compat[48]; -+#endif -+} __u; } pthread_mutex_t; - #define __DEFINED_pthread_mutex_t - #endif - diff --git a/appsec/tests/integration/src/docker/toolchain/glibc_compat.c b/appsec/tests/integration/src/docker/toolchain/glibc_compat.c deleted file mode 100644 index 68d5e1f978..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/glibc_compat.c +++ /dev/null @@ -1,187 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#if defined(__linux__) && !defined(__GLIBC__) - -# ifdef __x86_64__ -float ceilf(float x) -{ - float result; - // NOLINTNEXTLINE(hicpp-no-assembler) - __asm__("roundss $0x0A, %[x], %[result]" - : [result] "=x"(result) - : [x] "x"(x)); - return result; -} -double ceil(double x) -{ - double result; - // NOLINTNEXTLINE(hicpp-no-assembler) - __asm__("roundsd $0x0A, %[x], %[result]" - : [result] "=x"(result) - : [x] "x"(x)); - return result; -} -# endif - -# ifdef __aarch64__ -float ceilf(float x) -{ - float result; - __asm__("frintp %s0, %s1\n" : "=w"(result) : "w"(x)); - return result; -} -double ceil(double x) -{ - double result; - __asm__("frintp %d0, %d1\n" : "=w"(result) : "w"(x)); - return result; -} -# endif - -# ifdef __aarch64__ -# define _STAT_VER 0 -# else -# define _STAT_VER 1 -# endif - -// glibc before 2.33 (2021) doesn't have these -int stat(const char *restrict path, void *restrict buf) -{ - int __xstat(int, const char *restrict, void *restrict); - return __xstat(_STAT_VER, path, buf); -} - -int fstat(int fd, void *buf) -{ - int __fxstat(int, int, void *); - return __fxstat(_STAT_VER, fd, buf); -} - -int lstat(const char *restrict path, void *restrict buf) -{ - int __lxstat(int, const char *restrict, void *restrict); - return __lxstat(_STAT_VER, path, buf); -} - -// glibc doesn't define pthread_atfork on aarch64. We need to delegate to -// glibc's __register_atfork() instead. __register_atfork() takes an extra -// argument, __dso_handle, which is a pointer to the DSO that is registering the -// fork handlers. This is used to ensure that the handlers are not called after -// the DSO is unloaded. glibc on amd64 also implements pthread_atfork() in terms -// of __register_atfork(). (musl never unloads modules so that potential -// problem doesn't exist) - -// On amd64, even though pthread_atfork is exported by glibc, it should not be -// used. Code that uses pthread_atfork will compile to an import to -// __register_atfork(), but here we're compiling against musl, resulting in an -// an import to pthread_atfork. This will cause a runtime error after the test -// that unloads our module. The reason is that when we call pthread_atfork in -// glibc, __register_atfork() is called with the __dso_handle of libc6.so, not -// the __dso_handle of our module. So the fork handler is not unregistered when -// our module is unloaded. - -extern void *__dso_handle __attribute__((weak)); -int __register_atfork(void (*prepare)(void), void (*parent)(void), - void (*child)(void), void *__dso_handle) __attribute__((weak)); - -int pthread_atfork( - void (*prepare)(void), void (*parent)(void), void (*child)(void)) -{ - // glibc - if (__dso_handle && __register_atfork) { - return __register_atfork(prepare, parent, child, __dso_handle); - } - - static int (*real_atfork)(void (*)(void), void (*)(void), void (*)(void)); - - if (!real_atfork) { - // dlopen musl -# ifdef __aarch64__ - void *handle = dlopen("ld-musl-aarch64.so.1", RTLD_LAZY); - if (!handle) { - (void)fprintf( - // NOLINTNEXTLINE(concurrency-mt-unsafe) - stderr, "dlopen of ld-musl-aarch64.so.1 failed: %s\n", - dlerror()); - abort(); - } -# else - void *handle = dlopen("libc.musl-x86_64.so.1", RTLD_LAZY); - if (!handle) { - (void)fprintf( - // NOLINTNEXTLINE(concurrency-mt-unsafe) - stderr, "dlopen of libc.musl-x86_64.so.1 failed: %s\n", - dlerror()); - abort(); - } -# endif - real_atfork = dlsym(handle, "pthread_atfork"); - if (!real_atfork) { - (void)fprintf( - // NOLINTNEXTLINE(concurrency-mt-unsafe) - stderr, "dlsym of pthread_atfork failed: %s\n", dlerror()); - abort(); - } - } - - return real_atfork(prepare, parent, child); -} - -// the symbol strerror_r in glibc is not the POSIX version; it returns char * -// __xpg_sterror_r is exported by both glibc and musl -int strerror_r(int errnum, char *buf, size_t buflen) -{ - int __xpg_strerror_r(int, char *, size_t); - return __xpg_strerror_r(errnum, buf, buflen); -} - -// when compiling with --coverage, some references to atexit show up. -// glibc doesn't provide atexit for similar reasons as pthread_atfork presumably -int __cxa_atexit(void (*func)(void *), void *arg, void *dso_handle); -int atexit(void (*function)(void)) -{ - if (!__dso_handle) { - (void)fprintf(stderr, "Aborting because __dso_handle is NULL\n"); - abort(); - } - - // the cast is harmless on amd64 and aarch64. Passing an extra argument to a - // function that expects none causes no problems - return __cxa_atexit((void (*)(void *))function, 0, __dso_handle); -} - -// introduced in glibc 2.25 -ssize_t getrandom(void *buf, size_t buflen, unsigned int flags) { - // SYS_getrandom is 318 (amd64) or 278 (aarch64) - // This was only added in Linux 3.17 (2014), so don't use it - // return syscall(SYS_getrandom, buf, buflen, flags); - int fd; - size_t bytes_read = 0; - - fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return -1; - } - - while (bytes_read < buflen) { - ssize_t result = read(fd, (char*)buf + bytes_read, buflen - bytes_read); - if (result < 0) { - if (errno == EINTR) { - continue; - } - close(fd); - return -1; - } - bytes_read += result; - } - - close(fd); - return (ssize_t)bytes_read; -} - -#endif diff --git a/appsec/tests/integration/src/docker/toolchain/locale.h.diff b/appsec/tests/integration/src/docker/toolchain/locale.h.diff deleted file mode 100644 index 52d046665a..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/locale.h.diff +++ /dev/null @@ -1,23 +0,0 @@ ---- /usr/include/locale.h -+++ /usr/include/locale.h -@@ -157,19 +157,7 @@ - # define LC_TELEPHONE_MASK (1 << __LC_TELEPHONE) - # define LC_MEASUREMENT_MASK (1 << __LC_MEASUREMENT) - # define LC_IDENTIFICATION_MASK (1 << __LC_IDENTIFICATION) --# define LC_ALL_MASK (LC_CTYPE_MASK \ -- | LC_NUMERIC_MASK \ -- | LC_TIME_MASK \ -- | LC_COLLATE_MASK \ -- | LC_MONETARY_MASK \ -- | LC_MESSAGES_MASK \ -- | LC_PAPER_MASK \ -- | LC_NAME_MASK \ -- | LC_ADDRESS_MASK \ -- | LC_TELEPHONE_MASK \ -- | LC_MEASUREMENT_MASK \ -- | LC_IDENTIFICATION_MASK \ -- ) -+# define LC_ALL_MASK 0x1fbf - - /* Return a duplicate of the set of locale in DATASET. All usage - counters are increased if necessary. */ diff --git a/appsec/tests/integration/src/docker/toolchain/wchar.h.diff b/appsec/tests/integration/src/docker/toolchain/wchar.h.diff deleted file mode 100644 index 60dc2a04d5..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/wchar.h.diff +++ /dev/null @@ -1,11 +0,0 @@ ---- /usr/include/wchar.h 2024-10-31 15:25:55.995768004 +0000 -+++ /usr/include/wchar.h.fix 2024-10-31 15:25:35.478137009 +0000 -@@ -485,7 +485,7 @@ - int __base, locale_t __loc) __THROW; - - __extension__ --extern unsigned long long int wcstoull_l (const wchar_t *__restrict __nptr, -+extern long long wcstoull_l (const wchar_t *__restrict __nptr, - wchar_t **__restrict __endptr, - int __base, locale_t __loc) - __THROW; From 18c0c781cefb1dcfe0576105b09d7a4d1129df01 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Wed, 6 Nov 2024 10:42:26 +0100 Subject: [PATCH 05/15] chore(test: gradle): restore cache check Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 4faa03b7fb..448d89b3fa 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -329,7 +329,8 @@ def buildAppSecTask = { String version, String variant -> ''' git config --global --add safe.directory '*' cd /appsec - cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ + test -f CMakeCache.txt || \\ + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ -DCMAKE_INSTALL_PREFIX=/appsec \\ -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\ -DDD_APPSEC_TESTING=ON /project/appsec From ed92c43ae6f5203051a70847b501f4c40ccf8bc3 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Wed, 6 Nov 2024 11:20:07 +0100 Subject: [PATCH 06/15] chore(tests: integration): format Dockerfile Signed-off-by: Alexandre Rulleau --- .../src/docker/toolchain/Dockerfile | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index 4c1cc7d069..aad0eb595f 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -1,13 +1,24 @@ FROM debian@sha256:08db48d59c0a91afb802ebafc921be3154e200c452e4d0b19634b426b03e0e25 AS toolchain RUN apt-get update && apt-get install -y \ - build-essential wget binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg + binutils \ + build-essential \ + curl \ + git \ + gnupg \ + libncurses5-dev \ + lld \ + lsb-release \ + patchelf \ + software-properties-common \ + wget \ + wget \ + xz-utils RUN ln -s /bin/sed /usr/bin/sed -RUN mkdir /build -ADD . /build/ +COPY . /build/ RUN wget https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ - grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ - tar --strip-components=1 -C /usr/local -xvzf cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ - rm cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz + grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ + tar --strip-components=1 -C /usr/local -xvzf cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz && \ + rm cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz From 45c33a27507abba6ff4fedb7ee6e7f6fa218b1ee Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Wed, 6 Nov 2024 17:33:57 +0100 Subject: [PATCH 07/15] fix(tests: integration): fix recommended settings Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/src/test/waf/recommended.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appsec/tests/integration/src/test/waf/recommended.json b/appsec/tests/integration/src/test/waf/recommended.json index f06f7f29fc..0fbc7b4c01 100644 --- a/appsec/tests/integration/src/test/waf/recommended.json +++ b/appsec/tests/integration/src/test/waf/recommended.json @@ -6881,7 +6881,7 @@ "type": "redirect_request", "parameters": { "status_code": "303", - "location": "datadoghq.com" + "location": "https://datadoghq.com" } } ] From 6affbc9a57a209435e8e2ea1e9ee6baf8a89bd59 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 28 Oct 2024 10:59:21 +0100 Subject: [PATCH 08/15] feat(libddwaf): bump to 1.20.0 Signed-off-by: Alexandre Rulleau --- appsec/third_party/libddwaf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appsec/third_party/libddwaf b/appsec/third_party/libddwaf index f18e6e286f..9be9b7c38f 160000 --- a/appsec/third_party/libddwaf +++ b/appsec/third_party/libddwaf @@ -1 +1 @@ -Subproject commit f18e6e286f5f62af5c809c4e0a84a9c624553b2c +Subproject commit 9be9b7c38f15a486096a74919c1decd8486c487d From 4317f0a32f63027dd4a12600c5442d14458fba88 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 28 Oct 2024 11:25:00 +0100 Subject: [PATCH 09/15] test(libddwaf): bump to 1.20.0 Signed-off-by: Alexandre Rulleau --- appsec/tests/helper/client_test.cpp | 4 ++-- appsec/tests/helper/waf_test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appsec/tests/helper/client_test.cpp b/appsec/tests/helper/client_test.cpp index 50e8c2c117..62095be19a 100644 --- a/appsec/tests/helper/client_test.cpp +++ b/appsec/tests/helper/client_test.cpp @@ -155,7 +155,7 @@ TEST(ClientTest, ClientInit) EXPECT_STREQ(msg_res->status.c_str(), "ok"); EXPECT_EQ(msg_res->meta.size(), 2); EXPECT_STREQ( - msg_res->meta[std::string(tag::waf_version)].c_str(), "1.18.0"); + msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.0"); EXPECT_STREQ( msg_res->meta[std::string(tag::event_rules_errors)].c_str(), "{}"); @@ -264,7 +264,7 @@ TEST(ClientTest, ClientInitInvalidRules) EXPECT_STREQ(msg_res->status.c_str(), "ok"); EXPECT_EQ(msg_res->meta.size(), 2); EXPECT_STREQ( - msg_res->meta[std::string(tag::waf_version)].c_str(), "1.18.0"); + msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.0"); rapidjson::Document doc; doc.Parse(msg_res->meta[std::string(tag::event_rules_errors)]); diff --git a/appsec/tests/helper/waf_test.cpp b/appsec/tests/helper/waf_test.cpp index 0c3d533ce5..a4eb340aa0 100644 --- a/appsec/tests/helper/waf_test.cpp +++ b/appsec/tests/helper/waf_test.cpp @@ -48,7 +48,7 @@ TEST(WafTest, InitWithInvalidRules) waf::instance::from_settings(cs, ruleset, meta, metrics)}; EXPECT_EQ(meta.size(), 2); - EXPECT_STREQ(meta[std::string(tag::waf_version)].c_str(), "1.18.0"); + EXPECT_STREQ(meta[std::string(tag::waf_version)].c_str(), "1.20.0"); rapidjson::Document doc; doc.Parse(meta[std::string(tag::event_rules_errors)]); From eb6621d64a0d3cd28d9ac87eafea97cc98713d7c Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 28 Oct 2024 14:13:26 +0100 Subject: [PATCH 10/15] test(libddwaf): bump to 1.20.1 Signed-off-by: Alexandre Rulleau --- appsec/third_party/libddwaf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appsec/third_party/libddwaf b/appsec/third_party/libddwaf index 9be9b7c38f..de06f7afb2 160000 --- a/appsec/third_party/libddwaf +++ b/appsec/third_party/libddwaf @@ -1 +1 @@ -Subproject commit 9be9b7c38f15a486096a74919c1decd8486c487d +Subproject commit de06f7afb2112152b9f7e137109358cf4762f90d From 7734dc26927c71beffed94bc40b0bac65a7254a3 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 28 Oct 2024 14:16:41 +0100 Subject: [PATCH 11/15] test(libddwaf): bump to 1.20.1 Signed-off-by: Alexandre Rulleau --- appsec/tests/helper/client_test.cpp | 4 ++-- appsec/tests/helper/waf_test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appsec/tests/helper/client_test.cpp b/appsec/tests/helper/client_test.cpp index 62095be19a..b12cf89e16 100644 --- a/appsec/tests/helper/client_test.cpp +++ b/appsec/tests/helper/client_test.cpp @@ -155,7 +155,7 @@ TEST(ClientTest, ClientInit) EXPECT_STREQ(msg_res->status.c_str(), "ok"); EXPECT_EQ(msg_res->meta.size(), 2); EXPECT_STREQ( - msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.0"); + msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.1"); EXPECT_STREQ( msg_res->meta[std::string(tag::event_rules_errors)].c_str(), "{}"); @@ -264,7 +264,7 @@ TEST(ClientTest, ClientInitInvalidRules) EXPECT_STREQ(msg_res->status.c_str(), "ok"); EXPECT_EQ(msg_res->meta.size(), 2); EXPECT_STREQ( - msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.0"); + msg_res->meta[std::string(tag::waf_version)].c_str(), "1.20.1"); rapidjson::Document doc; doc.Parse(msg_res->meta[std::string(tag::event_rules_errors)]); diff --git a/appsec/tests/helper/waf_test.cpp b/appsec/tests/helper/waf_test.cpp index a4eb340aa0..a78f5476a7 100644 --- a/appsec/tests/helper/waf_test.cpp +++ b/appsec/tests/helper/waf_test.cpp @@ -48,7 +48,7 @@ TEST(WafTest, InitWithInvalidRules) waf::instance::from_settings(cs, ruleset, meta, metrics)}; EXPECT_EQ(meta.size(), 2); - EXPECT_STREQ(meta[std::string(tag::waf_version)].c_str(), "1.20.0"); + EXPECT_STREQ(meta[std::string(tag::waf_version)].c_str(), "1.20.1"); rapidjson::Document doc; doc.Parse(meta[std::string(tag::event_rules_errors)]); From 3072923ad9700860b50e1b0ddad08e642602248b Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 28 Oct 2024 15:28:20 +0100 Subject: [PATCH 12/15] test(helper): fix tests Signed-off-by: Alexandre Rulleau --- appsec/tests/helper/client_test.cpp | 4 ++-- appsec/tests/helper/engine_test.cpp | 5 ++--- appsec/tests/helper/main.cpp | 2 +- .../helper/remote_config/listeners/engine_listener_test.cpp | 6 ++---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/appsec/tests/helper/client_test.cpp b/appsec/tests/helper/client_test.cpp index b12cf89e16..8303494bec 100644 --- a/appsec/tests/helper/client_test.cpp +++ b/appsec/tests/helper/client_test.cpp @@ -637,8 +637,8 @@ TEST(ClientTest, EventWithMultipleActions) EXPECT_EQ(msg_res->actions.size(), 3); // Block is not generated since there is a redirect EXPECT_STREQ(msg_res->actions[0].verdict.c_str(), "redirect"); - EXPECT_STREQ( - msg_res->actions[0].parameters["location"].c_str(), "localhost"); + EXPECT_STREQ(msg_res->actions[0].parameters["location"].c_str(), + "https://localhost"); EXPECT_STREQ( msg_res->actions[0].parameters["status_code"].c_str(), "303"); EXPECT_STREQ(msg_res->actions[1].verdict.c_str(), diff --git a/appsec/tests/helper/engine_test.cpp b/appsec/tests/helper/engine_test.cpp index 1778e17337..06a8b0c576 100644 --- a/appsec/tests/helper/engine_test.cpp +++ b/appsec/tests/helper/engine_test.cpp @@ -4,14 +4,13 @@ // This product includes software developed at Datadog // (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. #include "common.hpp" -#include "json_helper.hpp" #include #include #include #include const std::string waf_rule = - R"({"version":"2.1","rules":[{"id":"1","name":"rule1","tags":{"type":"flow1","category":"category1"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg1","key_path":[]}],"regex":"^string.*"}},{"operator":"match_regex","parameters":{"inputs":[{"address":"arg2","key_path":[]}],"regex":".*"}}]}]})"; + R"({"version":"2.1","rules":[{"id":"1","name":"rule1","tags":{"type":"flow1","category":"category1"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg1","key_path":[]}],"regex":"^string.*"}},{"operator":"match_regex","parameters":{"inputs":[{"address":"arg2","key_path":[]}],"regex":".*"}}]},{"id":"2","name":"rule2","tags":{"type":"flow2","category":"category2"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg3","key_path":[]}],"regex":"^string.*"}}]}]})"; const std::string waf_rule_with_data = R"({"version":"2.1","rules":[{"id":"blk-001-001","name":"Block IP Addresses","tags":{"type":"block_ip","category":"security_response"},"conditions":[{"parameters":{"inputs":[{"address":"http.client_ip"}],"data":"blocked_ips"},"operator":"ip_match"}],"transformers":[],"on_match":["block"]}]})"; @@ -704,7 +703,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions) R"({"rules_override": [{"rules_target":[{"rule_id":"1"}], "on_match": ["redirect"]}], "actions": [{"id": "redirect", "type": "redirect_request", "parameters": {"status_code": "303", - "location": "localhost"}}]})"); + "location": "https://localhost"}}]})"); e->update(update, meta, metrics); } diff --git a/appsec/tests/helper/main.cpp b/appsec/tests/helper/main.cpp index 5be5e7a7e9..064f9e1ce5 100644 --- a/appsec/tests/helper/main.cpp +++ b/appsec/tests/helper/main.cpp @@ -190,7 +190,7 @@ std::string create_sample_rules_ok() "id": "redirect", "type": "redirect_request", "parameters": { - "location": "localhost" + "location": "https://localhost" } } ] diff --git a/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp b/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp index 1e3a3d4bb1..48281376ed 100644 --- a/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp +++ b/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp @@ -6,18 +6,16 @@ #include "../../common.hpp" #include "../mocks.hpp" -#include "base64.h" #include "engine.hpp" #include "json_helper.hpp" #include "remote_config/exception.hpp" #include "remote_config/listeners/engine_listener.hpp" -#include "remote_config/product.hpp" #include "subscriber/waf.hpp" #include #include const std::string waf_rule = - R"({"version":"2.1","rules":[{"id":"1","name":"rule1","tags":{"type":"flow1","category":"category1"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg1","key_path":[]}],"regex":".*"}}]}]})"; + R"({"version":"2.1","rules":[{"id":"1","name":"rule1","tags":{"type":"flow1","category":"category1"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg1","key_path":[]}],"regex":".*"}}]},{"id":"2","name":"rule2","tags":{"type":"flow2","category":"category2"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"dummy","key_path":[]}],"regex":".*"}}]}]})"; namespace dds::remote_config { @@ -978,7 +976,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideAndActionsUpdate) } const std::string update = R"({"actions": [{"id": "redirect", "type": "redirect_request", "parameters": - {"status_code": "303", "location": "localhost"}}],"rules_override": + {"status_code": "303", "location": "https://localhost"}}],"rules_override": [{"rules_target": [{"rule_id": "1"}], "on_match": ["redirect"]}]})"; listener.on_update(get_config("ASM", update)); From be3cb96ef565cc8bffa93fdbb29154bcb286138a Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Wed, 6 Nov 2024 18:17:10 +0100 Subject: [PATCH 13/15] fix(tests: integration): fix RoadRunnerTests Signed-off-by: Alexandre Rulleau --- .../com/datadog/appsec/php/integration/RoadRunnerTests.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RoadRunnerTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RoadRunnerTests.groovy index 4a5503e363..c1910f1a42 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RoadRunnerTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RoadRunnerTests.groovy @@ -132,7 +132,7 @@ class RoadRunnerTests { .header('X-Forwarded-For', '80.80.80.81').GET().build() def trace = CONTAINER.traceFromRequest(req, ofString()) { HttpResponse re -> assert re.statusCode() == 303 - assert re.headers().firstValue('Location').get() == 'datadoghq.com' + assert re.headers().firstValue('Location').get() == 'https://datadoghq.com' } Span span = trace.first() @@ -156,7 +156,7 @@ class RoadRunnerTests { HttpRequest req = CONTAINER.buildReq('/?user=user2023').GET().build() def trace = CONTAINER.traceFromRequest(req, ofString()) { HttpResponse it -> assert it.statusCode() == 303 - assert it.headers().firstValue('Location').get() == 'datadoghq.com' + assert it.headers().firstValue('Location').get() == 'https://datadoghq.com' } assert trace.first().meta."appsec.blocked" == "true" } From 48f24e17cfe3e6e5f23a10c7dec0fb56b7639587 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 8 Nov 2024 13:07:50 +0100 Subject: [PATCH 14/15] fix(tests: integration): remove useless build arg Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/gradle/images.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appsec/tests/integration/gradle/images.gradle b/appsec/tests/integration/gradle/images.gradle index a4652f4cfb..f590530456 100644 --- a/appsec/tests/integration/gradle/images.gradle +++ b/appsec/tests/integration/gradle/images.gradle @@ -33,7 +33,7 @@ tasks.register('buildToolchain', Exec) { description = "Build the toolchain image" inputs.dir 'src/docker/toolchain' outputs.upToDateWhen imageUpToDate(inputs, "$repo:toolchain") - commandLine 'docker', 'build', '-t', "$repo:toolchain", 'src/docker/toolchain', "--build-arg", "ARCH=$arch" + commandLine 'docker', 'build', '-t', "$repo:toolchain", 'src/docker/toolchain' } tasks.register('buildPhpDeps', Exec) { From e8009f467afed3e1e5e099bcec260626e37e4fc8 Mon Sep 17 00:00:00 2001 From: Gustavo Lopes Date: Fri, 8 Nov 2024 14:22:07 +0000 Subject: [PATCH 15/15] appsec inttests: support building 8.4 images --- appsec/tests/integration/gradle/images.gradle | 1 + appsec/tests/integration/src/docker/php/Dockerfile | 5 +++++ appsec/tests/integration/src/docker/php/build_dev_php.sh | 8 +++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/appsec/tests/integration/gradle/images.gradle b/appsec/tests/integration/gradle/images.gradle index f590530456..4b6e82041b 100644 --- a/appsec/tests/integration/gradle/images.gradle +++ b/appsec/tests/integration/gradle/images.gradle @@ -11,6 +11,7 @@ def phpVersions = [ '8.1': '8.1.26', '8.2': '8.2.13', '8.3': '8.3.0', + '8.4': '8.4.0', ] def arch = System.getProperty('os.arch') diff --git a/appsec/tests/integration/src/docker/php/Dockerfile b/appsec/tests/integration/src/docker/php/Dockerfile index 00900e3ad8..6f5357e69f 100644 --- a/appsec/tests/integration/src/docker/php/Dockerfile +++ b/appsec/tests/integration/src/docker/php/Dockerfile @@ -23,7 +23,12 @@ ARG VARIANT ENV NO_APX_WRAPPER=1 +# PHP 8.4 checks if /usr/sbin/apache2 exists +RUN touch /usr/sbin/apache2 +RUN chmod +x /usr/sbin/apache2 RUN USER=root /build/php/build_dev_php.sh "$PHP_VERSION" "$VARIANT" +RUN rm /usr/sbin/apache2 + ADD php.ini /root/php/php.ini ENV PHPRC=/root/php/php.ini diff --git a/appsec/tests/integration/src/docker/php/build_dev_php.sh b/appsec/tests/integration/src/docker/php/build_dev_php.sh index 4b81f7e18c..4221002b61 100755 --- a/appsec/tests/integration/src/docker/php/build_dev_php.sh +++ b/appsec/tests/integration/src/docker/php/build_dev_php.sh @@ -36,6 +36,8 @@ function download_php { local download_url if [[ $version_id -lt 50400 ]]; then download_url="http://museum.php.net/php5/php-${version}.tar.gz" + elif [[ $version_id -ge 80400 ]]; then + download_url="https://downloads.php.net/~saki/php-8.4.0RC3.tar.gz" else download_url="https://www.php.net/distributions/php-${version}.tar.gz" fi @@ -85,10 +87,10 @@ function get_xdebug_version { echo '2.8.1' elif [[ $version_id -lt 80000 ]]; then echo '2.9.8' - elif [[ $version_id -ge 80300 ]]; then + elif [[ $version_id -lt 80400 ]]; then echo '3.3.1' - else - echo '3.2.2' + elif [[ $version_id -ge 80400 ]]; then + echo '3.4.0beta1' fi }