add NIC+WAFv5 tests (#6456)

.github/actions/smoke-tests/action.yaml

@@ -31,6 +31,9 @@ inputs:
  azure-ad-secret:
  description: Azure Active Directory secret for JWKs
  required: false
+ registry-token:
+ description: JWT token for accessing container registry
+ required: false
 
 outputs:
  test-results-name:
@@ -76,13 +79,17 @@ runs:
  docker run --rm \
  --name test-runner-${{ github.run_id }} \
  --network=kind \
+ -v "/var/run/docker.sock:/var/run/docker.sock" \
+ -v ~/.docker:/root/.docker \
  -v ${{ github.workspace }}/tests:/workspace/tests \
  -v ${{ github.workspace }}/deployments:/workspace/deployments \
  -v ${{ github.workspace }}/charts:/workspace/charts \
  -v ${{ github.workspace }}/config:/workspace/config \
  -v ${{ github.workspace }}/pyproject.toml:/workspace/pyproject.toml \
  -v ${{ steps.k8s.outputs.test_output_path }}:${{ steps.k8s.outputs.test_output_path }} \
  -v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
+ --docker-registry-user=oauth2accesstoken \
+ --docker-registry-token=${{ inputs.registry-token }} \
  --context=kind-${{ github.run_id }} \
  --image=${{ inputs.image-name }}:${{ inputs.tag }} \
  --image-pull-policy=Never \

.github/data/matrix-smoke-nap.json

@@ -32,6 +32,14 @@
  "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr'",
  "platforms": "linux/amd64"
  },
+ {
+ "label": "AP_WAF_V5 1/1",
+ "image": "debian-plus-nap-v5",
+ "type": "plus",
+ "nap_modules": "waf",
+ "marker": "appprotect_waf_v5",
+ "platforms": "linux/amd64"
+ },
  {
  "label": "AP_DOS 1/3",
  "image": "debian-plus-nap",

.github/workflows/build-test-image.yml

@@ -55,7 +55,7 @@ jobs:
  context: "."
  cache-from: type=gha,scope=test-runner
  tags: |
- gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') }}
+ gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') }}
  gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:latest
  pull: true
  push: true

.github/workflows/ci.yml

@@ -548,7 +548,7 @@ jobs:
  - name: Check if test image exists
  id: check-image
  run: |
- docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}
+ docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}
  shell: bash
  continue-on-error: true
  if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
@@ -559,7 +559,7 @@ jobs:
  file: tests/Dockerfile
  context: "."
  cache-from: type=gha,scope=test-runner
- tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+ tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
  pull: true
  push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
  load: false

.github/workflows/regression.yml

@@ -271,7 +271,8 @@ jobs:
  k8s-version: ${{ matrix.k8s }}
  label: ${{ matrix.images.label }}
  azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
- test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+ registry-token: ${{ steps.auth.outputs.access_token }}
+ test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
 
  - name: Upload Test Results
  uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0

.github/workflows/setup-smoke.yml

@@ -54,7 +54,7 @@ jobs:
  - name: Set image variables
  id: image_details
  run: |
- echo "name=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic${{ contains(inputs.nap-modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap-modules, 'waf') && '-nap' || '' }}/nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
+ echo "name=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic${{ contains(inputs.nap-modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap-modules, 'waf') && '-nap' || '' }}${{ contains(inputs.image, 'v5') && '-v5' || '' }}/nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
  echo "build_tag=${{ inputs.build-tag }}${{ contains(inputs.image, 'ubi-9') && '-ubi' || '' }}${{ contains(inputs.image, 'ubi-8') && '-ubi8' || '' }}${{ contains(inputs.image, 'alpine') && '-alpine' || '' }}${{ contains(inputs.target, 'aws') && '-mktpl' || '' }}${{ contains(inputs.image, 'fips') && '-fips' || ''}}" >> $GITHUB_OUTPUT
  echo "stable_tag=${{ inputs.stable-tag }}${{ contains(inputs.image, 'ubi-9') && '-ubi' || '' }}${{ contains(inputs.image, 'ubi-8') && '-ubi8' || '' }}${{ contains(inputs.image, 'alpine') && '-alpine' || '' }}${{ contains(inputs.target, 'aws') && '-mktpl' || '' }}${{ contains(inputs.image, 'fips') && '-fips' || ''}}" >> $GITHUB_OUTPUT
 
@@ -108,7 +108,7 @@ jobs:
  - name: Check if test image exists
  id: check-image
  run: |
- docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+ docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
  shell: bash
  continue-on-error: true
  if: ${{ inputs.authenticated }}
@@ -119,7 +119,7 @@ jobs:
  file: tests/Dockerfile
  context: "."
  cache-from: type=gha,scope=test-runner
- tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+ tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
  pull: true
  push: ${{ inputs.authenticated }}
  load: ${{ !inputs.authenticated }}
@@ -147,6 +147,11 @@ jobs:
  ${{ contains(inputs.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
  if: ${{ !inputs.authenticated }}
 
+ - name: Generate WAF v5 tgz from JSON
+ run: |
+ docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.2.0 -p /data/wafv5.json -o /data/wafv5.tgz
+ if: ${{ contains(inputs.image, 'nap-v5')}}
+
  - name: Run Smoke Tests
  id: smoke-tests
  uses: ./.github/actions/smoke-tests
@@ -158,7 +163,8 @@ jobs:
  label: ${{ inputs.label }}
  k8s-version: ${{ inputs.k8s-version }}
  azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
- test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+ registry-token: ${{ steps.auth.outputs.access_token }}
+ test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
  if: ${{ steps.stable_exists.outputs.exists != 'true' }}
 
  - name: Upload Test Results

.github/workflows/single-image-regression.yml

@@ -107,4 +107,5 @@ jobs:
  label: "${{ inputs.image }} regression"
  k8s-version: ${{ inputs.k8s-version }}
  azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
+ registry-token: ${{ steps.auth.outputs.access_token }}
  test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ inputs.test-image-tag }}"

pyproject.toml

@@ -32,6 +32,7 @@ markers =[
  "appprotect_waf_policies_block",
  "appprotect_waf_policies_grpc",
  "appprotect_waf_policies_vsr",
+ "appprotect_waf_v5",
  "appprotect_watch",
  "appprotect_batch",
  "basic_auth",

tests/Dockerfile

@@ -23,6 +23,9 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s
  && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
  && apt-get update && apt-get install -y apache2-utils
 
+RUN apt update -y \
+ && curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && mv /tmp/docker/docker /usr/bin/docker
+
 COPY --link tests /workspace/tests
 
 COPY --link pyproject.toml /workspace/

tests/conftest.py

@@ -126,6 +126,18 @@ def pytest_addoption(parser) -> None:
  default="1",
  help="Number of resources to deploy for upgrade tests",
  )
+ parser.addoption(
+ "--docker-registry-user",
+ action="store",
+ default="",
+ help="Docker registry username",
+ )
+ parser.addoption(
+ "--docker-registry-token",
+ action="store",
+ default="",
+ help="Docker registry token",
+ )
 
 
 # import fixtures into pytest global namespace

tests/data/ap-waf-v5/policies/waf.yaml

@@ -0,0 +1,8 @@
+apiVersion: k8s.nginx.org/v1
+kind: Policy
+metadata:
+ name: waf-policy
+spec:
+ waf:
+ enable: true
+ apBundle: "wafv5.tgz"

tests/data/ap-waf-v5/standard/virtual-server.yaml

@@ -0,0 +1,20 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+ name: virtual-server
+spec:
+ host: virtual-server.example.com
+ upstreams:
+ - name: backend2
+ service: backend2-svc
+ port: 80
+ - name: backend1
+ service: backend1-svc
+ port: 80
+ routes:
+ - path: "/backend1"
+ action:
+ pass: backend1
+ - path: "/backend2"
+ action:
+ pass: backend2

tests/data/ap-waf-v5/virtual-server-route-waf-subroute.yaml

@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+ name: backends
+spec:
+ host: virtual-server-route.example.com
+ upstreams:
+ - name: backend1
+ service: backend1-svc
+ port: 80
+ - name: backend3
+ service: backend3-svc
+ port: 80
+ subroutes:
+ - path: "/backends/backend1"
+ policies:
+ - name: waf-policy
+ action:
+ pass: backend1
+ - path: "/backends/backend3"
+ action:
+ pass: backend3

tests/data/ap-waf-v5/virtual-server-waf-route.yaml

@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+ name: virtual-server
+spec:
+ host: virtual-server.example.com
+ upstreams:
+ - name: backend2
+ service: backend2-svc
+ port: 80
+ - name: backend1
+ service: backend1-svc
+ port: 80
+ routes:
+ - path: "/backend1"
+ policies:
+ - name: waf-policy
+ action:
+ pass: backend1
+ - path: "/backend2"
+ action:
+ pass: backend2

tests/data/ap-waf-v5/virtual-server-waf-spec.yaml

@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+ name: virtual-server
+spec:
+ host: virtual-server.example.com
+ policies:
+ - name: waf-policy
+ upstreams:
+ - name: backend2
+ service: backend2-svc
+ port: 80
+ - name: backend1
+ service: backend1-svc
+ port: 80
+ routes:
+ - path: "/backend1"
+ action:
+ pass: backend1
+ - path: "/backend2"
+ action:
+ pass: backend2

tests/data/ap-waf-v5/wafv5.json

@@ -0,0 +1,112 @@
+{
+ "policy": {
+ "name": "app_protect_api_security_policy",
+ "description": "NGINX App Protect API Security Policy. The policy is intended to be used with an OpenAPI file",
+ "template": {
+ "name": "POLICY_TEMPLATE_NGINX_BASE"
+ },
+ "open-api-files": [],
+ "blocking-settings": {
+ "violations": [
+ {
+ "block": true,
+ "description": "Mandatory request body is missing",
+ "name": "VIOL_MANDATORY_REQUEST_BODY"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter location",
+ "name": "VIOL_PARAMETER_LOCATION"
+ },
+ {
+ "block": true,
+ "description": "Mandatory parameter is missing",
+ "name": "VIOL_MANDATORY_PARAMETER"
+ },
+ {
+ "block": true,
+ "description": "JSON data does not comply with JSON schema",
+ "name": "VIOL_JSON_SCHEMA"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter array value",
+ "name": "VIOL_PARAMETER_ARRAY_VALUE"
+ },
+ {
+ "block": true,
+ "description": "Illegal Base64 value",
+ "name": "VIOL_PARAMETER_VALUE_BASE64"
+ },
+ {
+ "block": true,
+ "description": "Illegal request content type",
+ "name": "VIOL_URL_CONTENT_TYPE"
+ },
+ {
+ "block": true,
+ "description": "Illegal static parameter value",
+ "name": "VIOL_PARAMETER_STATIC_VALUE"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter value length",
+ "name": "VIOL_PARAMETER_VALUE_LENGTH"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter data type",
+ "name": "VIOL_PARAMETER_DATA_TYPE"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter numeric value",
+ "name": "VIOL_PARAMETER_NUMERIC_VALUE"
+ },
+ {
+ "block": true,
+ "description": "Parameter value does not comply with regular expression",
+ "name": "VIOL_PARAMETER_VALUE_REGEXP"
+ },
+ {
+ "block": true,
+ "description": "Illegal URL",
+ "name": "VIOL_URL"
+ },
+ {
+ "block": true,
+ "description": "Illegal parameter",
+ "name": "VIOL_PARAMETER"
+ },
+ {
+ "block": true,
+ "description": "Illegal empty parameter value",
+ "name": "VIOL_PARAMETER_EMPTY_VALUE"
+ },
+ {
+ "block": true,
+ "description": "Illegal repeated parameter name",
+ "name": "VIOL_PARAMETER_REPEATED"
+ },
+ {
+ "block": true,
+ "description": "Illegal method",
+ "name": "VIOL_METHOD"
+ },
+ {
+ "block": true,
+ "description": "Illegal gRPC method",
+ "name": "VIOL_GRPC_METHOD"
+ }
+ ]
+ },
+ "xml-profiles": [
+ {
+ "name": "Default",
+ "defenseAttributes": {
+ "maximumNameLength": 1024
+ }
+ }
+ ]
+ }
+}

tests/settings.py

@@ -30,3 +30,7 @@
 BATCH_RELOAD_NUMBER = 2
 # Number of namespaces to deploy to measure Pod performance, used in test_multiple_ns_perf.py
 NS_COUNT = 0
+# Nginx registry address to pull waf components from
+NGX_REG = "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr"
+# WAF component version to pull from above registry
+WAF_V5_VERSION = "5.2.0"