diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 000000000000..d40366529ee2 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,37 @@ +# NB: sematics here are not the same as .gitignore +# see https://github.com/bazelbuild/bazel/issues/8106 +# Ignore backup files. +*~ +# Ignore Vim swap files. +.*.swp +# Ignore files generated by IDEs. +/.aswb/ +/.cache/ +/.classpath +/.clwb/ +/.factorypath +/.idea/ +/.ijwb/ +/.project +/.settings +/.vscode/ +/bazel.iml +# Ignore all bazel-* symlinks. There is no full list since this can change +# based on the name of the directory bazel is cloned into. +/bazel-* +# Ignore outputs generated during Bazel bootstrapping. +/output/ +# Ignore jekyll build output. +/production +/.sass-cache +# Bazelisk version file +.bazelversion +# User-specific .bazelrc +user.bazelrc + +/t/ +/spec/ +/spec-ee/ +/servroot/ +/autodoc/ +/.github/ diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000000..50990b609368 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,51 @@ +# Bazel doesn't need more than 200MB of memory for local build based on memory profiling: +# https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling +# The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large +# enough to consume all memory constrained by cgroup in large host. +# Limiting JVM heapsize here to let it do GC more when approaching the limit to +# leave room for compiler/linker. +# The number 3G is chosen heuristically to both support large VM and small VM with RBE. +# Startup options cannot be selected via config. +startup --host_jvm_args=-Xmx512m + +run --color=yes + +common --color=yes +common --curses=auto + +build --experimental_ui_max_stdouterr_bytes=10485760 + +build --show_progress_rate_limit=0 +build --show_timestamps +build --worker_verbose + +build --incompatible_strict_action_env + +# Enable --platforms API based cpu,compiler,crosstool_top selection; remove this in 7.0.0 as it's enabled by default +build --incompatible_enable_cc_toolchain_resolution + +# Pass PATH, CC, CXX variables from the environment. +build --action_env=CC --host_action_env=CC +build --action_env=CXX --host_action_env=CXX +build --action_env=PATH --host_action_env=PATH + +build --action_env=BAZEL_BUILD=1 + +# temporary fix for https://github.com/bazelbuild/bazel/issues/12905 on macOS +build --features=-debug_prefix_map_pwd_is_dot + +# Build flags. +build --action_env=BUILD_NAME=kong-dev +build --action_env=INSTALL_DESTDIR=MANAGED +build --strip=never + +# Release flags +build:release --//:debug=false +build:release --action_env=BUILD_NAME=kong-dev +build:release --action_env=INSTALL_DESTDIR=/usr/local +build:release --copt="-g" +build:release --strip=never + +build --spawn_strategy=local + +build --action_env=GITHUB_TOKEN --host_action_env=GITHUB_TOKEN diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000000..dfda3e0b4f01 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +6.1.0 diff --git a/.github/labeler.yml b/.github/labeler.yml index f401408e9cf1..b54040139c27 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -20,10 +20,14 @@ core/db/migrations: core/db: - any: ['kong/db/**/*', '!kong/db/migrations/**/*'] +changelog: +- CHANGELOG.md + core/docs: -- kong/autodoc/**/* -- ./**/*.md -- ./*.md +- any: ['**/*.md', '!CHANGELOG.md'] + +autodoc: +- 'autodoc/**/*' core/language/go: - kong/runloop/plugin_servers/* @@ -49,6 +53,10 @@ core/router: core/templates: - kong/templates/* +core/tracing: +- kong/tracing/**/* +- kong/pdk/tracing.lua + chore: - .github/**/* - .devcontainer/**/* @@ -107,19 +115,14 @@ plugins/key-auth: plugins/ldap-auth: - kong/plugins/ldap-auth/**/* -plugins/log-serializers: -- kong/plugins/log-serializers/**/* - plugins/loggly: - kong/plugins/loggly/**/* plugins/oauth2: - kong/plugins/oauth2/**/* -plugins/post-function: +plugins/serverless-functions: - kong/plugins/post-function/**/* - -plugins/pre-function: - kong/plugins/pre-function/**/* plugins/prometheus: @@ -164,4 +167,21 @@ plugins/udp-log: plugins/zipkin: - kong/plugins/zipkin/**/* +plugins/opentelemetry: +- kong/plugins/opentelemetry/**/* + +copyright: +- COPYRIGHT + +schema-change-noteworthy: +- kong/db/schema/entities/**/* +build/bazel: +- '**/*.bazel' +- '**/*.bzl' +- build/**/* +- WORKSPACE +- .bazelignore +- .bazelrc +- .bazelversion +- scripts/build-*.sh diff --git a/.github/matrix-commitly.yml b/.github/matrix-commitly.yml new file mode 100644 index 000000000000..7685340597c3 --- /dev/null +++ b/.github/matrix-commitly.yml @@ -0,0 +1,24 @@ +# please see matrix-full.yml for meaning of each field +build-packages: +- label: ubuntu-22.04 + os: ubuntu-22.04 + package: deb + check-manifest-suite: ubuntu-22.04-amd64 + +build-images: +- label: ubuntu + base-image: ubuntu:22.04 + package: deb + artifact-from: ubuntu-22.04 + +smoke-tests: +- label: ubuntu + +scan-vulnerabilities: +- label: ubuntu + +release-packages: + +release-images: +- label: ubuntu + package: deb diff --git a/.github/matrix-full.yml b/.github/matrix-full.yml new file mode 100644 index 000000000000..3b81e34f5466 --- /dev/null +++ b/.github/matrix-full.yml @@ -0,0 +1,193 @@ +build-packages: +# label: used to distinguish artifacts for later use; +# ensure its value ends with -arm64 if it's a cross-compiled for ARM64 target +# image: docker image name if the build is running in side a container +# package: package type +# package-type: the nfpm packaging target, //:kong_{package} target; only used when package is rpm +# bazel-args: additional bazel build flags +# check-manifest-suite: the check manifest suite as defined in scripts/explain_manifest/config.py + +# Ubuntu +- label: ubuntu-18.04 + image: ubuntu:18.04 + package: deb + check-manifest-suite: ubuntu-18.04-amd64 +- label: ubuntu-20.04 + image: ubuntu:20.04 + package: deb + check-manifest-suite: ubuntu-20.04-amd64 +- label: ubuntu-22.04 + package: deb + check-manifest-suite: ubuntu-22.04-amd64 + +# Debian +- label: debian-10 + image: debian:10 + package: deb + check-manifest-suite: debian-10-amd64 +- label: debian-11 + image: debian:11 + package: deb + check-manifest-suite: debian-11-amd64 + +# Alpine +- label: alpine + os: ubuntu-22.04 + package: apk + bazel-args: --platforms=//:alpine-crossbuild-x86_64 --//:opentracing=true + check-manifest-suite: alpine-amd64 + +# CentOS +- label: centos-7 + os: ubuntu-22.04 + image: centos:7 + package: rpm + package-type: el7 + check-manifest-suite: el7-amd64 + +# RHEL +- label: rhel-7 + image: centos:7 + package: rpm + package-type: el7 + check-manifest-suite: el7-amd64 + +- label: rhel-8 + image: rockylinux:8 + package: rpm + package-type: el8 + check-manifest-suite: el8-amd64 + +# AmazonLinux +- label: amazonlinux-2 + image: amazonlinux:2 + package: rpm + bazel-args: --//:opentracing=true + package-type: aws2 + check-manifest-suite: amazonlinux-2-amd64 + +build-images: +# Only build images for the latest version of each major release. +# label: used as compose docker image label ${github.sha}-${label} +# base-image: docker image to use as base +# package: package type +# package-distro: the distro name to use in package name, for example el7, aws2; only applicable to rpm +# artifact-from: label of build-packages to use +# artifact-from-alt: another label of build-packages to use for downloading package (to build multi-arch image) +# docker-platforms: comma separated list of docker buildx platforms to build for + +# Ubuntu +- label: ubuntu + base-image: ubuntu:22.04 + package: deb + artifact-from: ubuntu-22.04 + +# Centos +- label: centos7 + base-image: centos:7 + package: rpm + package-distro: el7 + artifact-from: centos-7 + +- label: rhel7 + base-image: registry.access.redhat.com/ubi7 + package: rpm + package-distro: el7 + artifact-from: rhel-7 + +- label: rhel8 + base-image: registry.access.redhat.com/ubi8 + package: rpm + package-distro: el8 + artifact-from: rhel-8 + +# Alpine +- label: alpine + base-image: alpine:3.16 + package: apk + artifact-from: alpine + +release-packages: +# Ubuntu +- label: ubuntu-18.04 + package: deb + artifact-from: ubuntu-18.04 + artifact-version: 18.04 + artifact-type: ubuntu + artifact: kong.all.deb +- label: ubuntu-20.04 + package: deb + artifact-from: ubuntu-20.04 + artifact-version: 20.04 + artifact-type: ubuntu + artifact: kong.all.deb +- label: ubuntu-22.04 + package: deb + artifact-from: ubuntu-22.04 + artifact-version: 22.04 + artifact-type: ubuntu + artifact: kong.all.deb + +# Debian +- label: debian-10 + package: deb + artifact-from: debian-10 + artifact-version: 10 + artifact-type: debian + artifact: kong.all.deb +- label: debian-11 + package: deb + artifact-from: debian-11 + artifact-version: 11 + artifact-type: debian + artifact: kong.all.deb + +# CentOS +- label: centos-7 + package: rpm + artifact-from: centos-7 + artifact-version: 7 + artifact-type: centos + artifact: kong.el7.all.rpm + +# RHEL +- label: rhel-7 + package: rpm + artifact-from: rhel-7 + artifact-version: 7 + artifact-type: rhel + artifact: kong.el7.all.rpm + +- label: rhel-8 + package: rpm + artifact-from: rhel-8 + artifact-version: 8 + artifact-type: rhel + artifact: kong.el8.all.rpm + +# Amazon Linux +- label: amazonlinux-2 + package: rpm + artifact-from: amazonlinux-2 + artifact-version: 2 + artifact-type: amazonlinux + artifact: kong.aws2.all.rpm + +# Alpine +- label: alpine + package: apk + artifact-from: alpine + artifact-type: alpine + artifact: kong.all.apk.tar.gz + +release-images: +- label: centos7 + package: rpm +- label: rhel7 + package: rpm +- label: rhel8 + package: rpm +- label: alpine + package: apk +- label: ubuntu + package: deb diff --git a/.github/workflows/auto-assignee.yml b/.github/workflows/auto-assignee.yml deleted file mode 100644 index 12fa2933c44a..000000000000 --- a/.github/workflows/auto-assignee.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Add assignee to PRs -on: - pull_request: - types: [ opened, reopened ] -permissions: - pull-requests: write -jobs: - assign-author: - runs-on: ubuntu-latest - steps: - - uses: toshimaru/auto-author-assign@2daaeb2988aef24bf37e636fe733f365c046aba0 - diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000000..9a129c7a6c0c --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,24 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + name: Backport + runs-on: ubuntu-latest + if: > + github.event.pull_request.merged + && ( + github.event.action == 'closed' + || ( + github.event.action == 'labeled' + && contains(github.event.label.name, 'backport') + ) + ) + steps: + - uses: tibdex/backport@2e217641d82d02ba0603f46b1aeedefb258890ac # v2.0.3 + with: + github_token: ${{ secrets.PAT }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1a7137a62eb1..ec10e3aec5fe 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,62 +1,88 @@ name: Build & Test -on: [push, pull_request] +on: + pull_request: + paths-ignore: + # ignore markdown files (CHANGELOG.md, README.md, etc.) + - '**/*.md' + - '.github/workflows/release.yml' + push: + paths-ignore: + # ignore markdown files (CHANGELOG.md, README.md, etc.) + - '**/*.md' + # ignore PRs for the generated COPYRIGHT file + - 'COPYRIGHT' + branches: + - master + - release/* + - test-please/* + +# cancel previous runs if new commits are pushed to the PR, but run for each commit on master +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: build: name: Build dependencies - runs-on: ubuntu-20.04 - - env: - DOWNLOAD_ROOT: $HOME/download-root + runs-on: ubuntu-22.04 steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV + - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} + + - name: Install packages + if: steps.cache-deps.outputs.cache-hit != 'true' + run: sudo apt update && sudo apt install libyaml-dev valgrind libprotobuf-dev - - name: Checkout kong-build-tools + - name: Build Kong if: steps.cache-deps.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + run: | + bazel build //build:kong --verbose_failures + export PATH="$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$PATH" + chmod +rw -R $INSTALL_ROOT + nginx -V + ldd $(which nginx) + luarocks + + - name: Bazel Outputs + uses: actions/upload-artifact@v3 + if: failure() with: - repository: Kong/kong-build-tools - path: kong-build-tools - ref: master + name: bazel-outputs + path: | + bazel-out/_tmp/actions - name: Checkout go-pluginserver if: steps.cache-deps.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: Kong/go-pluginserver path: go-pluginserver - name: Add to Path if: steps.cache-deps.outputs.cache-hit != 'true' - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - - name: Install packages - if: steps.cache-deps.outputs.cache-hit != 'true' - run: sudo apt update && sudo apt install libyaml-dev valgrind libprotobuf-dev - - - name: Build Kong dependencies + - name: Build Dev Kong dependencies if: steps.cache-deps.outputs.cache-hit != 'true' run: | - source .ci/setup_env_github.sh + eval `luarocks path` make dev lint-doc-and-unit-tests: name: Lint, Doc and Unit tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: build env: @@ -77,22 +103,20 @@ jobs: steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} - name: Add to Path - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - name: Check autodoc generation run: | @@ -107,7 +131,16 @@ jobs: - name: Lint Lua code run: | eval `luarocks path` - luacheck -q . + make lint + + - name: Validate rockspec file + run: | + eval `luarocks path` + scripts/validate-rockspec + + - name: Check spec file misspelling + run: | + scripts/check_spec_files_spelling.sh - name: Unit tests run: | @@ -117,10 +150,11 @@ jobs: integration-tests-postgres: name: Postgres ${{ matrix.suite }} - ${{ matrix.split }} tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: build strategy: + fail-fast: false matrix: suite: [integration, plugins] split: [first (01-04), second (>= 05)] @@ -129,6 +163,9 @@ jobs: KONG_TEST_PG_DATABASE: kong KONG_TEST_PG_USER: kong KONG_TEST_DATABASE: postgres + KONG_SPEC_TEST_GRPCBIN_PORT: "15002" + KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" + KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json TEST_SUITE: ${{ matrix.suite }} TEST_SPLIT: ${{ matrix.split }} @@ -144,7 +181,7 @@ jobs: options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8 grpcbin: - image: moul/grpcbin + image: kong/grpcbin ports: - 15002:9000 - 15003:9001 @@ -153,7 +190,9 @@ jobs: image: redis ports: - 6379:6379 - options: --entrypoint redis-server + - 6380:6380 + options: >- + --name kong_redis zipkin: image: openzipkin/zipkin:2.19 @@ -163,28 +202,49 @@ jobs: steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} - name: Add to Path - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools:$INSTALL_ROOT/go-pluginserver" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - name: Add gRPC test host names run: | echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts + - name: Enable SSL for Redis + if: ${{ matrix.suite == 'plugins' }} + run: | + docker cp ${{ github.workspace }} kong_redis:/workspace + docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh + docker restart kong_redis + docker logs kong_redis + + - name: Run OpenTelemetry Collector + if: ${{ matrix.suite == 'plugins' }} + run: | + mkdir -p ${{ github.workspace }}/tmp/otel + touch ${{ github.workspace }}/tmp/otel/file_exporter.json + sudo chmod 777 -R ${{ github.workspace }}/tmp/otel + docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \ + -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \ + -v ${{ github.workspace }}/tmp/otel:/etc/otel \ + --name opentelemetry-collector -d \ + otel/opentelemetry-collector-contrib:0.52.0 \ + --config=/etc/otel-collector-config.yaml + sleep 2 + docker logs opentelemetry-collector + - name: Tests run: | eval `luarocks path` @@ -193,13 +253,16 @@ jobs: integration-tests-dbless: name: DB-less integration tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: build env: KONG_TEST_PG_DATABASE: kong KONG_TEST_PG_USER: kong KONG_TEST_DATABASE: 'off' + KONG_SPEC_TEST_GRPCBIN_PORT: "15002" + KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" + KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json TEST_SUITE: dbless services: @@ -212,28 +275,40 @@ jobs: steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} - name: Add to Path - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools:$INSTALL_ROOT/go-pluginserver" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - name: Add gRPC test host names run: | echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts + - name: Run OpenTelemetry Collector + run: | + mkdir -p ${{ github.workspace }}/tmp/otel + touch ${{ github.workspace }}/tmp/otel/file_exporter.json + sudo chmod 777 -R ${{ github.workspace }}/tmp/otel + docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \ + -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \ + -v ${{ github.workspace }}/tmp/otel:/etc/otel \ + --name opentelemetry-collector -d \ + otel/opentelemetry-collector-contrib:0.52.0 \ + --config=/etc/otel-collector-config.yaml + sleep 2 + docker logs opentelemetry-collector + - name: Tests run: | eval `luarocks path` @@ -242,10 +317,11 @@ jobs: integration-tests-cassandra: name: C* ${{ matrix.cassandra_version }} ${{ matrix.suite }} - ${{ matrix.split }} tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: build strategy: + fail-fast: false matrix: suite: [integration, plugins] cassandra_version: [3] @@ -253,6 +329,9 @@ jobs: env: KONG_TEST_DATABASE: cassandra + KONG_SPEC_TEST_GRPCBIN_PORT: "15002" + KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" + KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json TEST_SUITE: ${{ matrix.suite }} TEST_SPLIT: ${{ matrix.split }} @@ -276,7 +355,9 @@ jobs: image: redis ports: - 6379:6379 - options: --entrypoint redis-server + - 6380:6380 + options: >- + --name kong_redis zipkin: image: openzipkin/zipkin:2.19 @@ -286,28 +367,49 @@ jobs: steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} - name: Add to Path - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools:$INSTALL_ROOT/go-pluginserver" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - name: Add gRPC test host names run: | echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts + - name: Enable SSL for Redis + if: ${{ matrix.suite == 'plugins' }} + run: | + docker cp ${{ github.workspace }} kong_redis:/workspace + docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh + docker restart kong_redis + docker logs kong_redis + + - name: Run OpenTelemetry Collector + if: ${{ matrix.suite == 'plugins' }} + run: | + mkdir -p ${{ github.workspace }}/tmp/otel + touch ${{ github.workspace }}/tmp/otel/file_exporter.json + sudo chmod 777 -R ${{ github.workspace }}/tmp/otel + docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \ + -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \ + -v ${{ github.workspace }}/tmp/otel:/etc/otel \ + --name opentelemetry-collector -d \ + otel/opentelemetry-collector-contrib:0.52.0 \ + --config=/etc/otel-collector-config.yaml + sleep 2 + docker logs opentelemetry-collector + - name: Tests run: | eval `luarocks path` @@ -316,7 +418,7 @@ jobs: pdk-tests: name: PDK tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: build env: @@ -325,33 +427,31 @@ jobs: steps: - name: Set environment variables run: | - echo "INSTALL_ROOT=$HOME/install-root" >> $GITHUB_ENV - echo "DOWNLOAD_ROOT=$HOME/download-root" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$INSTALL_ROOT/openssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "INSTALL_ROOT=$(pwd)/bazel-bin/build/kong-dev" >> $GITHUB_ENV - name: Checkout Kong source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lookup build cache - uses: actions/cache@v2 id: cache-deps + uses: actions/cache@v3 with: path: ${{ env.INSTALL_ROOT }} - key: ${{ hashFiles('.ci/setup_env_github.sh') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }}-${{ hashFiles('.requirements') }}-${{ hashFiles('kong-*.rockspec') }} + key: ${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '**/*.bzl', '**/*.bazel', '.github/workflows/build_and_test.yml') }} - name: Add to Path - run: echo "$INSTALL_ROOT/openssl/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin:$INSTALL_ROOT/luarocks/bin:$GITHUB_WORKSPACE/kong-build-tools/openresty-build-tools:$DOWNLOAD_ROOT/cpanm" >> $GITHUB_PATH + run: echo "$INSTALL_ROOT/bin:$INSTALL_ROOT/openresty/nginx/sbin:$INSTALL_ROOT/openresty/bin" >> $GITHUB_PATH - name: Install Test::Nginx run: | - CPAN_DOWNLOAD=$DOWNLOAD_ROOT/cpanm + CPAN_DOWNLOAD=./cpanm mkdir -p $CPAN_DOWNLOAD curl -o $CPAN_DOWNLOAD/cpanm https://cpanmin.us chmod +x $CPAN_DOWNLOAD/cpanm echo "Installing CPAN dependencies..." - cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) - cpanm --notest Test::Nginx + $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) + $CPAN_DOWNLOAD/cpanm --notest Test::Nginx - name: Tests run: | diff --git a/.github/workflows/buildifier.yml b/.github/workflows/buildifier.yml new file mode 100644 index 000000000000..726aa8c94223 --- /dev/null +++ b/.github/workflows/buildifier.yml @@ -0,0 +1,55 @@ +name: Buildifier + +on: + pull_request: + paths: + - '**/*.bzl' + - '**/*.bazel' + - 'BUILD*' + - 'WORKSPACE*' + push: + paths: + - '**/*.bzl' + - '**/*.bazel' + - 'BUILD*' + - 'WORKSPACE*' + branches: + - master + - release/* + +jobs: + + autoformat: + name: Auto-format and Check + runs-on: ubuntu-22.04 + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Install Dependencies + run: | + sudo wget -O /bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/5.1.0/buildifier-linux-amd64 + sudo chmod +x /bin/buildifier + + - name: Run buildifier + run: | + buildifier -mode=fix $(find . -name 'BUILD*' -o -name 'WORKSPACE*' -o -name '*.bzl' -o -name '*.bazel' -type f) + + - name: Verify buildifier + shell: bash + run: | + # From: https://backreference.org/2009/12/23/how-to-match-newlines-in-sed/ + # This is to leverage this workaround: + # https://github.com/actions/toolkit/issues/193#issuecomment-605394935 + function urlencode() { + sed ':begin;$!N;s/\n/%0A/;tbegin' + } + if [[ $(git diff-index --name-only HEAD --) ]]; then + for x in $(git diff-index --name-only HEAD --); do + echo "::error file=$x::Please run buildifier.%0A$(git diff $x | urlencode)" + done + echo "${{ github.repository }} is out of style. Please run buildifier." + exit 1 + fi + echo "${{ github.repository }} is formatted correctly." diff --git a/.github/workflows/label-check.yml b/.github/workflows/label-check.yml new file mode 100644 index 000000000000..00df7b9de5fa --- /dev/null +++ b/.github/workflows/label-check.yml @@ -0,0 +1,13 @@ +name: Pull Request Label Checker +on: + pull_request: + types: [synchronize, opened, reopened, labeled, unlabeled] +jobs: + check-labels: + name: prevent merge labels + runs-on: ubuntu-latest + + steps: + - name: do-not-merge label found + run: echo "do-not-merge label found, this PR will not be merged"; exit 1 + if: ${{ contains(github.event.*.labels.*.name, 'pr/do not merge') || contains(github.event.*.labels.*.name, 'DO NOT MERGE') }} diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml index d102b8c96e41..4613569074b3 100644 --- a/.github/workflows/label.yml +++ b/.github/workflows/label.yml @@ -17,6 +17,6 @@ jobs: pull-requests: write steps: - - uses: actions/labeler@v3.0.2 + - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index 7a75989bf673..000000000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Package & Smoke Test - -on: # yamllint disable-line rule:truthy - pull_request: - push: - branches: - - master - - next/* - - release/* - -env: - DOCKER_REPOSITORY: kong/kong-build-tools - -jobs: - package-and-test: - if: github.event_name == 'pull_request' - name: Build & Smoke Test Packages - runs-on: ubuntu-22.04 - - steps: - - name: Swap git with https - run: git config --global url."https://github".insteadOf git://github - - - name: Setup some environment variables - run: | - echo "KONG_SOURCE_LOCATION=$GITHUB_WORKSPACE/kong-src" >> $GITHUB_ENV - echo "KONG_BUILD_TOOLS_LOCATION=$GITHUB_WORKSPACE/kong-build-tools" >> $GITHUB_ENV - - - name: Checkout Kong source code - uses: actions/checkout@v3 - with: - path: ${{ env.KONG_SOURCE_LOCATION }} - submodules: recursive - token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} - - - name: Setup kong-build-tools - run: | - pushd ${{ env.KONG_SOURCE_LOCATION }} - make setup-kong-build-tools - - - name: Setup package naming environment variables - run: | - grep -v '^#' ${{ env.KONG_SOURCE_LOCATION}}/.requirements >> $GITHUB_ENV - - - name: Package & Test - env: - GITHUB_TOKEN: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} - run: | - pushd ${{ env.KONG_SOURCE_LOCATION }} - make package/test/deb - - package-test-and-unofficial-release: - if: github.event_name == 'push' - name: Build & Smoke & Unofficial Release Packages - runs-on: ubuntu-22.04 - strategy: - matrix: - package_type: [deb, rpm, apk] - - steps: - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} - password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} - - - name: Swap git with https - run: git config --global url."https://github".insteadOf git://github - - - name: Setup directory environment variables - run: | - echo "KONG_SOURCE_LOCATION=$GITHUB_WORKSPACE/kong-src" >> $GITHUB_ENV - echo "KONG_BUILD_TOOLS_LOCATION=$GITHUB_WORKSPACE/kong-build-tools" >> $GITHUB_ENV - - - name: Checkout Kong source code - uses: actions/checkout@v3 - with: - path: ${{ env.KONG_SOURCE_LOCATION }} - submodules: recursive - token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} - - - name: Setup kong-build-tools - run: | - pushd ${{ env.KONG_SOURCE_LOCATION }} - make setup-kong-build-tools - - - name: Setup package naming environment variables - run: | - grep -v '^#' ${{ env.KONG_SOURCE_LOCATION}}/.requirements >> $GITHUB_ENV - echo "DOCKER_RELEASE_REPOSITORY=kong/kong" >> $GITHUB_ENV - echo "KONG_TEST_CONTAINER_TAG=${GITHUB_REF_NAME##*/}-${{ matrix.package_type }}" >> $GITHUB_ENV - if [[ ${{matrix.package_type }} == "apk" ]]; then - echo "ADDITIONAL_TAG_LIST=${GITHUB_REF_NAME##*/}-alpine" >> $GITHUB_ENV - fi - if [[ ${{matrix.package_type }} == "deb" ]]; then - echo "ADDITIONAL_TAG_LIST=${GITHUB_REF_NAME##*/}-debian ${GITHUB_REF_NAME##*/} $GITHUB_SHA" >> $GITHUB_ENV - fi - - - name: Package & Test - env: - GITHUB_TOKEN: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} - run: | - pushd ${{ env.KONG_SOURCE_LOCATION }} - make package/test/${{ matrix.package_type }} - - - name: Push Docker Image - continue-on-error: true - env: - SKIP_TESTS: true - run: | - pushd ${{ env.KONG_SOURCE_LOCATION }} - make release/docker/${{ matrix.package_type }} - - - name: Store the package artifacts - continue-on-error: true - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.package_type }} - path: ${{ env.KONG_BUILD_TOOLS_LOCATION }}/output/* - - - name: Comment on commit - continue-on-error: true - uses: peter-evans/commit-comment@v2 - with: - token: ${{ secrets.GHA_COMMENT_TOKEN }} - body: | - Docker image avaialble ${{ env.DOCKER_RELEASE_REPOSITORY }}:${{ env.KONG_TEST_CONTAINER_TAG }} - Artifacts availabe https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..5f4b7bebfc6e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,712 @@ +name: Package & Release + +# The workflow to build and release official Kong packages and images. + +on: # yamllint disable-line rule:truthy + schedule: + - cron: '0 0 * * *' + pull_request: + paths: + - '**/*.rockspec' + - 'build/**' + - 'BUILD.bazel' + - '.requirements' + - '.github/workflows/release.yml' + - '.github/matrix-*.yml' + push: + tags: + - '**' + branches: + - master + - next/* + - release/* + workflow_dispatch: + inputs: + official: + description: 'Official release?' + required: true + type: boolean + default: false + version: + description: 'Release version, e.g. `3.0.0.0-beta.2`' + required: true + type: string + +# `commit-ly` is a flag that indicates whether the build should be run per commit. + +env: + # official release repo + DOCKER_REPOSITORY: kong/kong-gateway + PRERELEASE_DOCKER_REPOSITORY: kong/kong-gateway-dev + LEGACY_PRERELEASE_DOCKER_REPOSITORY: kong/kong-gateway-internal + FULL_RELEASE: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + + # only for pr + GHA_CACHE: ${{ github.event_name == 'pull_request' }} + + HAS_ACCESS_TO_GITHUB_TOKEN: ${{ github.repository_owner == 'Kong' }} + +jobs: + metadata: + name: Metadata + runs-on: ubuntu-latest-kong + outputs: + kong-version: ${{ steps.build-info.outputs.kong-version }} + prerelease-docker-repository: ${{ env.PRERELEASE_DOCKER_REPOSITORY }} + docker-repository: ${{ steps.build-info.outputs.docker-repository }} + release-desc: ${{ steps.build-info.outputs.release-desc }} + release-label: ${{ steps.build-info.outputs.release-label || '' }} + deploy-environment: ${{ steps.build-info.outputs.deploy-environment }} + matrix: ${{ steps.build-info.outputs.matrix }} + gh-release-name: ${{ steps.build-info.outputs.gh-release-name }} + gh-release-tag-name: ${{ steps.build-info.outputs.gh-release-tag-name }} + + steps: + - uses: actions/checkout@v3 + - name: Build Info + id: build-info + run: | + KONG_VERSION=$(bash scripts/grep-kong-version.sh) + echo "kong-version=$KONG_VERSION" >> $GITHUB_OUTPUT + + if [ "${{ github.event_name == 'schedule' }}" == "true" ]; then + echo "release-label=$(date -u +'%Y%m%d')" >> $GITHUB_OUTPUT + fi + + matrix_file=".github/matrix-commitly.yml" + if [ "$FULL_RELEASE" == "true" ]; then + matrix_file=".github/matrix-full.yml" + fi + + # for official releases, the GH release title is "Kong Gateway + # tag name is + echo "gh-release-name=Kong Enterprise ${{ github.ref_name }}" >> $GITHUB_OUTPUT + echo "gh-release-tag-name=${{ github.ref_name }}" >> $GITHUB_OUTPUT + + # nightly releases also get a tag (single `nightly` tag is deleted and pushed daily) + if [ "${{ github.event.inputs.version }}" == "nightly" ]; then + echo "gh-release-name=Kong Enterprise Nightly" >> $GITHUB_OUTPUT + echo "gh-release-tag-name=nightly" >> $GITHUB_OUTPUT + echo "release-label=$(date -u +'%Y%m%d')" >> $GITHUB_OUTPUT + fi + + if [ "${{ github.event.inputs.official }}" == "true" ]; then + release_desc="$KONG_VERSION (official)" + echo "docker-repository=$DOCKER_REPOSITORY" >> $GITHUB_OUTPUT + echo "deploy-environment=release" >> $GITHUB_OUTPUT + else + release_desc="$KONG_VERSION (pre-release)" + echo "docker-repository=$PRERELEASE_DOCKER_REPOSITORY" >> $GITHUB_OUTPUT + fi + + echo "release-desc=$release_desc" >> $GITHUB_OUTPUT + + if [ "${{ env.FULL_RELEASE }}" == "true" ]; then + echo "deploy-environment=release" >> $GITHUB_OUTPUT + fi + + echo "matrix=$(yq e -I=0 -o=json $matrix_file)" >> $GITHUB_OUTPUT + + cat $GITHUB_OUTPUT + + echo "### :package: Building and packaging for $release_desc" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- event_name: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "- ref_name: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY + echo "- inputs.version: ${{ github.event.inputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + cat $GITHUB_OUTPUT >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + + build-packages: + needs: metadata + name: Build & Package - ${{ matrix.label }} + environment: ${{ needs.metadata.outputs.deploy-environment }} + + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-packages'] }}" + + runs-on: ubuntu-22.04 + container: + image: ${{ matrix.image }} + options: --privileged + + steps: + - name: Early Rpm Setup + if: matrix.package == 'rpm' && matrix.image != '' + run: | + # tar/gzip is needed to restore git cache (if available) + yum install -y tar gzip which file zlib-devel + + - name: Early Deb in Container Setup + if: matrix.package == 'deb' && matrix.image != '' + run: | + # tar/gzip is needed to restore git cache (if available) + apt-get update + apt-get install -y git tar gzip file sudo + + - name: Cache Git + id: cache-git + if: (matrix.package == 'rpm' || matrix.image == 'debian:10') && matrix.image != '' + uses: actions/cache@v3 + with: + path: /usr/local/git + key: ${{ matrix.label }}-git-2.41.0 + + # el-7,8, amazonlinux-2,2023, debian-10 doesn't have git 2.18+, so we need to install it manually + - name: Install newer Git + if: (matrix.package == 'rpm' || matrix.image == 'debian:10') && matrix.image != '' && steps.cache-git.outputs.cache-hit != 'true' + run: | + if which apt 2>/dev/null; then + apt update + apt install -y wget libz-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext make gcc autoconf sudo + else + yum update -y + yum groupinstall -y 'Development Tools' + yum install -y wget zlib-devel openssl-devel curl-devel expat-devel gettext-devel perl-CPAN perl-devel + fi + wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.41.0.tar.gz + tar xf git-2.41.0.tar.gz + cd git-2.41.0 + + # https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5948/diffs + if [[ ${{ matrix.image }} == "centos:7" ]]; then + echo 'CFLAGS=-std=gnu99' >> config.mak + fi + + make configure + ./configure --prefix=/usr/local/git + make -j$(nproc) + make install + + - name: Add Git to PATH + if: (matrix.package == 'rpm' || matrix.image == 'debian:10') && matrix.image != '' + run: | + echo "/usr/local/git/bin" >> $GITHUB_PATH + + - name: Debian Git dependencies + if: matrix.image == 'debian:10' + run: | + apt update + # dependencies for git + apt install -y wget libz-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev sudo + + - name: Checkout Kong source code + uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + + - name: Swap git with https + run: git config --global url."https://github".insteadOf git://github + + - name: Cache Packages + id: cache-deps + if: env.GHA_CACHE == 'true' + uses: actions/cache@v3 + with: + path: bazel-bin/pkg + key: ${{ matrix.label }}-build-${{ hashFiles('.requirements', 'plugins-ee/**/*.lua', 'kong/**/*.lua', 'distribution/*.sh', 'distribution/*.lua', '**/*.rockspec', '**/*.bzl', '**/*.bazel') }} + + - name: Set .requirements into environment variables + run: | + grep -v '^#' .requirements >> $GITHUB_ENV + + - name: Setup Bazel + if: steps.cache-deps.outputs.cache-hit != 'true' + uses: bazelbuild/setup-bazelisk@987a6fb73f6be98289d7e48654b519e9f3b8d0cb # v2.0.0 + + - name: Install Deb Build Dependencies + if: matrix.package == 'deb' && steps.cache-deps.outputs.cache-hit != 'true' + run: | + sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + automake \ + build-essential \ + curl \ + file \ + libyaml-dev \ + m4 \ + perl \ + pkg-config \ + unzip \ + zlib1g-dev + + - name: Install Deb Cross Build Dependencies (arm64) + if: matrix.package == 'deb' && steps.cache-deps.outputs.cache-hit != 'true' && endsWith(matrix.label, 'arm64') + run: | + sudo apt-get install crossbuild-essential-arm64 -y + + - name: Install Rpm Build Dependencies + if: matrix.package == 'rpm' && matrix.image != '' && steps.cache-deps.outputs.cache-hit != 'true' + run: | + yum groupinstall -y 'Development Tools' + dnf config-manager --set-enabled powertools || true # enable devel packages on rockylinux:8 + yum install -y libyaml-devel zlib-devel + yum install -y cpanminus || (yum install -y perl && curl -L https://raw.githubusercontent.com/miyagawa/cpanminus/master/cpanm | perl - App::cpanminus) # amazonlinux2023 removed cpanminus + # required for openssl 3.x config + cpanm IPC/Cmd.pm + + - name: Build Kong + if: steps.cache-deps.outputs.cache-hit != 'true' + env: + GITHUB_TOKEN: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + run: | + bazel build --config release //build:kong --verbose_failures ${{ matrix.bazel-args }} + + - name: Package Kong - ${{ matrix.package }} + if: matrix.package != 'rpm' && steps.cache-deps.outputs.cache-hit != 'true' + run: | + bazel build --config release :kong_${{ matrix.package }} --verbose_failures ${{ matrix.bazel-args }} + + - name: Package Kong - rpm + if: matrix.package == 'rpm' && steps.cache-deps.outputs.cache-hit != 'true' + env: + RELEASE_SIGNING_GPG_KEY: ${{ secrets.RELEASE_SIGNING_GPG_KEY }} + NFPM_RPM_PASSPHRASE: ${{ secrets.RELEASE_SIGNING_GPG_KEY_PASSPHRASE }} + run: | + if [ -n "${RELEASE_SIGNING_GPG_KEY:-}" ]; then + RPM_SIGNING_KEY_FILE=$(mktemp) + echo "$RELEASE_SIGNING_GPG_KEY" > $RPM_SIGNING_KEY_FILE + export RPM_SIGNING_KEY_FILE=$RPM_SIGNING_KEY_FILE + fi + + if [[ ${{ matrix.package-type }} == "aws2" ]]; then + bazel build //build:copy_libdd_opentracing + fi + bazel build --config release :kong_${{ matrix.package-type }} --action_env=RPM_SIGNING_KEY_FILE --action_env=NFPM_RPM_PASSPHRASE ${{ matrix.bazel-args }} + + - name: Bazel Debug Outputs + if: failure() + run: | + cat bazel-out/_tmp/actions/stderr-* + sudo dmesg + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.label }}-packages + path: bazel-bin/pkg + retention-days: 3 + + verify-manifest-packages: + needs: [metadata, build-packages] + name: Verify Manifest - Package ${{ matrix.label }} + runs-on: ubuntu-22.04 + + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-packages'] }}" + + steps: + - uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.label }}-packages + path: bazel-bin/pkg + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' # caching pip dependencies + + - name: Verify + run: | + cd scripts/explain_manifest + pip install -r requirements.txt + pkg=$(ls ../../bazel-bin/pkg/kong* |head -n1) + + python ./main.py -f filelist.txt -p $pkg -o test.txt -s ${{ matrix.check-manifest-suite }} + + build-images: + name: Build Images - ${{ matrix.label }} + needs: [metadata, build-packages] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-images'] }}" + steps: + - uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.artifact-from }}-packages + path: bazel-bin/pkg + + - name: Download artifact (alt) + if: matrix.artifact-from-alt != '' + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.artifact-from-alt }}-packages + path: bazel-bin/pkg + + - name: Login to Docker Hub + if: ${{ env.HAS_ACCESS_TO_GITHUB_TOKEN }} + uses: docker/login-action@bc135a1993a1d0db3e9debefa0cfcb70443cc94c # v2.1.0 + with: + username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} + password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.PRERELEASE_DOCKER_REPOSITORY }} + tags: | + type=raw,${{ github.sha }}-${{ matrix.label }} + type=raw,enable=${{ matrix.label == 'ubuntu' }},${{ github.sha }} + + - name: Set up QEMU + if: matrix.docker-platforms != '' + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Set platforms + id: docker_platforms_arg + run: | + platforms="${{ matrix.docker-platforms }}" + if [[ -z "$platforms" ]]; then + platforms="linux/amd64" + fi + + echo "platforms=$platforms" + echo "platforms=$platforms" >> $GITHUB_OUTPUT + + - name: Build Docker Image + uses: docker/build-push-action@v3 + with: + file: build/dockerfiles/${{ matrix.package }}.Dockerfile + context: . + push: true + pull: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: ${{ steps.docker_platforms_arg.outputs.platforms }} + build-args: | + KONG_BASE_IMAGE=${{ matrix.base-image }} + KONG_ARTIFACT_PATH=bazel-bin/pkg/ + PACKAGE_DISTRO=${{ matrix.package-distro }} + EE_PORTS=8002 8445 8003 8446 8004 8447 + + - name: Comment on commit + if: github.event_name == 'push' && matrix.label == 'ubuntu' + # peter-evans/commit-comment@v2 + uses: peter-evans/commit-comment@a9352b8603f5dfe736429a2fd438c09ed567dc83 + with: + token: ${{ secrets.GHA_COMMENT_TOKEN }} + body: | + ### Bazel Build + Docker image available `${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ github.sha }}` + Artifacts available https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + + verify-manifest-images: + needs: [metadata, build-images] + name: Verify Manifest - Image ${{ matrix.label }} + runs-on: ubuntu-22.04 + if: github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') + + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-images'] }}" + + steps: + - uses: actions/checkout@v3 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' # caching pip dependencies + + - name: Verify + if: matrix.label != 'alpine' # skip alpine packages as they are lots of differences to unify + run: | + cd scripts/explain_manifest + # docker image verify requires sudo to set correct permissions, so we + # also install deps for root + sudo -E pip install -r requirements.txt + IMAGE=${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ github.sha }}-${{ matrix.label }} + + sudo -E python ./main.py --image $IMAGE -f docker_image_filelist.txt -s docker-image + + if [[ ! -z "${{ matrix.docker-platforms }}" ]]; then + DOCKER_DEFAULT_PLATFORM=linux/arm64 sudo -E python ./main.py --image $IMAGE -f docker_image_filelist.txt -s docker-image + fi + + scan-images: + name: Scan Images - ${{ matrix.label }} + needs: [metadata, build-images] + runs-on: ubuntu-22.04 + if: github.repository_owner == 'Kong' + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-images'] }}" + steps: + - name: Install regctl + uses: regclient/actions/regctl-installer@main + + - name: Login to Docker Hub + if: ${{ env.HAS_ACCESS_TO_GITHUB_TOKEN }} + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.1.0 + with: + username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} + password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} + + - name: Parse Architecture Specific Image Manifest Digests + id: image_manifest_metadata + run: | + IMAGE=${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ github.sha }}-${{ matrix.label }} + + archs="${{ matrix.docker-platforms }}" + if [[ -z "$archs" ]]; then + archs="linux/amd64" + fi + + for arch in $(echo "$archs" | sed -e 's/,/ /g'); do + arch=${arch#*/} + echo "Fetching digest for ${arch}..." + sha="$(regctl image digest "${IMAGE}" --platform linux/${arch})" + echo "${arch}_image_sha=${IMAGE}@${sha}" + echo "${arch}_image_sha=${IMAGE}@${sha}" >> $GITHUB_OUTPUT + done + + - name: Scan AMD64 Image digest + if: ${{ steps.image_manifest_metadata.outputs.amd64_image_sha != '' }} + id: sbom_action_amd64 + uses: Kong/public-shared-actions/security-actions/scan-docker-image@v1.8.0 + with: + asset_prefix: kong-${{ needs.metadata.outputs.gh-release-tag-name }}-${{ matrix.label }}-linux-amd64 + image: ${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ github.sha }}-${{ matrix.label }} + + - name: Scan ARM64 Image digest + if: ${{ steps.image_manifest_metadata.outputs.arm64_image_sha != '' }} + id: sbom_action_arm64 + uses: Kong/public-shared-actions/security-actions/scan-docker-image@v1.8.0 + with: + asset_prefix: kong-${{ needs.metadata.outputs.gh-release-tag-name }}-${{ matrix.label }}-linux-arm64 + image: ${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ github.sha }}-${{ matrix.label }} + + smoke-tests: + name: Smoke Tests - ${{ matrix.label }} + needs: [metadata, build-images] + runs-on: ubuntu-22.04 + if: github.repository_owner == 'Kong' + + # TODO: test packages + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-images'] }}" + + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: kong + POSTGRES_DB: kong + POSTGRES_PASSWORD: kong + ports: + - "5432:5432" + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8 + + env: + KONG_ADMIN_URI: http://localhost:8001 + KONG_ADMIN_HTTP2_URI: https://localhost:8444 + KONG_PROXY_URI: http://localhost:8000 + + steps: + - uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@f75d088332b07a08afadf6ac53c74509b9453f12 + with: + username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} + password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} + + - name: Setup Kong instance + # always pull the latest image to ensure we're testing the latest version. + run: | + docker run \ + -p 8000:8000 -p 8001:8001 -p 8444:8444\ + -e KONG_PG_PASSWORD=kong \ + -e KONG_ADMIN_LISTEN="0.0.0.0:8001, 0.0.0.0:8444 ssl http2" \ + -e KONG_ANONYMOUS_REPORTS=off \ + --name kong \ + --restart always \ + --network=host -d \ + --pull always \ + ${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ github.sha }}-${{ matrix.label }} \ + sh -c "kong migrations bootstrap && kong start" + sleep 3 + docker logs kong + + - name: Smoke Tests - Version Test + run: | + workflow_version="$( + echo '${{ needs.metadata.outputs.kong-version }}' \ + | sed -e 's@\.@\\\.@g' + )" + + # confirm workflow's version and built container version match with + # dots escaped, and end-line delimited + if ! docker exec kong kong version | grep -E "${workflow_version}$"; then + echo "Built container's 'kong version' didn't match workflow's." + echo "Ensure that versions in the meta.lua files are as expected." + exit 1 + fi + + - name: Smoke Tests - Base Tests + env: + VERBOSE: ${{ runner.debug == '1' && '1' || '' }} + run: build/tests/01-base.sh + + - name: Smoke Tests - Admin API + env: + VERBOSE: ${{ runner.debug == '1' && '1' || '' }} + run: build/tests/02-admin-api.sh + + - name: Smoke Tests - Enterprise + env: + PULP_USERNAME: admin + PULP_PASSWORD: ${{ secrets.PULP_PASSWORD }} + run: build/tests/03-enterprise.sh + + - name: Smoke Tests - HTTP2 Admin API + env: + VERBOSE: ${{ runner.debug == '1' && '1' || '' }} + run: build/tests/03-http2-admin-api.sh + + run_api_e2e_tests: + name: Run API E2E Tests + runs-on: ${{ vars.RELEASE_RUNS_ON }} + needs: [metadata, build-images] + steps: + - name: Trigger API E2E Tests + uses: aurelien-baudet/workflow-dispatch@65f835d60a79935c945a293a8688d7bbc301aeca + with: + workflow: gateway.yml + token: ${{ secrets.TEST_API_GH_TOKEN }} + repo: 'Kong/kong-api-tests' + inputs: '{"control_plane_image": "${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ github.sha }}"}' + ref: backup/2.8.4.1-tests + wait-for-completion: true + + release-packages: + name: Release Packages - ${{ matrix.label }} - ${{ needs.metadata.outputs.release-desc }} + needs: [metadata, build-packages, build-images, smoke-tests] + runs-on: ubuntu-22.04 + if: github.repository_owner == 'Kong' && fromJSON(needs.metadata.outputs.matrix)['release-packages'] != '' + timeout-minutes: 5 # PULP takes a while to publish + environment: ${{ needs.metadata.outputs.deploy-environment }} + + strategy: + # limit to 3 jobs at a time + max-parallel: 3 + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['release-packages'] }}" + + steps: + - uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.artifact-from }}-packages + path: bazel-bin/pkg + + - name: Upload Packages + env: + OFFICIAL_RELEASE: ${{ github.event.inputs.official }} + PULP_HOST: https://api.download.konghq.com + PULP_USERNAME: admin + # PULP_PASSWORD: ${{ secrets.PULP_DEV_PASSWORD }} + PULP_PASSWORD: ${{ secrets.PULP_PASSWORD }} + ARTIFACT_VERSION: ${{ matrix.artifact-version }} + ARTIFACT_TYPE: ${{ matrix.artifact-type }} + ARTIFACT: ${{ matrix.artifact }} + PACKAGE_TYPE: ${{ matrix.package }} + KONG_RELEASE_LABEL: ${{ needs.metadata.outputs.release-label }} + VERBOSE: ${{ runner.debug == '1' && '1' || '' }} + CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} + CLOUDSMITH_DRY_RUN: '' + IGNORE_CLOUDSMITH_FAILURES: ${{ vars.IGNORE_CLOUDSMITH_FAILURES }} + USE_CLOUDSMITH: ${{ vars.USE_CLOUDSMITH }} + USE_PULP: ${{ vars.USE_PULP }} + run: | + sha256sum bazel-bin/pkg/* + + if [[ ${{ matrix.label }} == *-fips ]]; then + # this is required for rpm and apk packages, but not for deb + export KONG_PACKAGE_NAME_OVERRIDE=kong-enterprise-edition-fips + fi + scripts/release-kong.sh + + release-images: + name: Release Images - ${{ matrix.label }} - ${{ needs.metadata.outputs.release-desc }} + needs: [metadata, build-images, smoke-tests] + runs-on: ubuntu-22.04 + if: github.repository_owner == 'Kong' && fromJSON(needs.metadata.outputs.matrix)['release-images'] != '' + + strategy: + # limit to 3 jobs at a time + max-parallel: 3 + fail-fast: false + matrix: + include: "${{ fromJSON(needs.metadata.outputs.matrix)['release-images'] }}" + + steps: + - name: Login to Docker Hub + uses: docker/login-action@f75d088332b07a08afadf6ac53c74509b9453f12 + with: + username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} + password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ needs.metadata.outputs.docker-repository }} + ${{ env.LEGACY_PRERELEASE_DOCKER_REPOSITORY }} + sep-tags: " " + tags: | + type=match,enable=${{ github.event_name == 'workflow_dispatch' }},pattern=\d.\d,value=${{ github.event.inputs.version }} + type=match,enable=${{ github.event_name == 'workflow_dispatch' && matrix.label == 'alpine' }},pattern=\d.\d,value=${{ github.event.inputs.version }},suffix= + type=raw,enable=${{ github.event_name == 'workflow_dispatch' }},${{ github.event.inputs.version }} + type=raw,enable=${{ github.event_name == 'workflow_dispatch' && matrix.label == 'alpine' }},${{ github.event.inputs.version }},suffix= + type=ref,enable=${{ github.event_name != 'workflow_dispatch' }},event=branch + type=ref,enable=${{ github.event_name != 'workflow_dispatch' && matrix.label == 'alpine' }},event=branch,suffix= + type=ref,event=tag + type=ref,enable=${{ matrix.label == 'alpine' }},event=tag,suffix= + type=ref,event=pr + type=schedule,pattern=nightly + type=schedule,enable=${{ matrix.label == 'alpine' }},pattern=nightly,suffix= + type=schedule,pattern={{date 'YYYYMMDD'}} + type=schedule,enable=${{ matrix.label == 'alpine' }},pattern={{date 'YYYYMMDD'}},suffix= + flavor: | + latest=false + suffix=-${{ matrix.label }} + + - name: Push Images + env: + TAGS: "${{ steps.meta.outputs.tags }}" + run: | + PRERELEASE_IMAGE=${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ github.sha }}-${{ matrix.label }} + docker pull $PRERELEASE_IMAGE + for tag in $TAGS; do + docker tag $PRERELEASE_IMAGE $tag + docker push $tag + done diff --git a/.github/workflows/upgrade-tests.yml b/.github/workflows/upgrade-tests.yml new file mode 100644 index 000000000000..b22d3fc8bdb2 --- /dev/null +++ b/.github/workflows/upgrade-tests.yml @@ -0,0 +1,62 @@ +name: Upgrade Tests + +on: + pull_request: + paths: + - 'kong/db/migrations/**' + - 'spec/05-migration/**' + push: + paths-ignore: + # ignore markdown files (CHANGELOG.md, README.md, etc.) + - '**/*.md' + branches: + - master + - release/* + - test-please/* +# cancel previous runs if new commits are pushed to the PR, but run for each commit on master +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + upgrade-test: + name: Run migration tests + runs-on: ubuntu-22.04 + + steps: + - name: Install Docker + run: | + sudo apt-get -y update + sudo apt-get -y install ca-certificates curl gnupg lsb-release + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin + + - name: Install prerequisites + run: | + sudo apt-get -y install jq + + - name: Install gojira + run: | + cd $RUNNER_WORKSPACE + git clone https://github.com/Kong/gojira + mkdir -p $HOME/.local/bin + ln -s $(pwd)/gojira/gojira.sh $HOME/.local/bin/gojira + + - name: Clone Kong source code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run upgrade tests (Postgres) + run: | + export GOJIRA_KONG_REPO_URL=$GITHUB_WORKSPACE + bash -x ./scripts/test-upgrade-path.sh -d postgres 2.8.0 $GITHUB_SHA + + - name: Run upgrade tests (Cassandra) + run: | + export GOJIRA_KONG_REPO_URL=$GITHUB_WORKSPACE + gojira nuke + bash -x ./scripts/test-upgrade-path.sh -d cassandra 2.8.0 $GITHUB_SHA diff --git a/.gitignore b/.gitignore index 8ecd58061979..70bdc5441fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .buildpath .project .idea +.docker .env servroot* @@ -21,8 +22,18 @@ autodoc/output # ldoc spec/docs +# migration output +migration-reports/ + kong-build-tools bin/grpcurl *.so *.bak +*.rock + +bazel-* + +# pongo shell history +.ash_history +.bash_history diff --git a/.luacheckrc b/.luacheckrc index 00cae9e83f4c..29fbcbeec7d5 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -8,6 +8,8 @@ globals = { "_KONG", "kong", "ngx.IS_CLI", + "kprof", + "ngx.worker.pids", } @@ -25,6 +27,10 @@ ignore = { exclude_files = { "spec/fixtures/invalid-module.lua", "spec-old-api/fixtures/invalid-module.lua", + "pgmoon/", + "bazel-bin", + "bazel-out", + "bazel-kong-ee", } files["kong/tools/kong-lua-sandbox.lua"] = { @@ -60,6 +66,14 @@ files["kong/plugins/ldap-auth/*.lua"] = { } +files["kong/tracing/init.lua"] = { + read_globals = { + "table.pack", + "table.unpack", + } +} + + files["spec/**/*.lua"] = { std = "ngx_lua+busted", } @@ -67,3 +81,29 @@ files["spec/**/*.lua"] = { files["spec-old-api/**/*.lua"] = { std = "ngx_lua+busted", } + +files["spec-ee/**/*.lua"] = { + std = "ngx_lua+busted", +} + +files["kong/keyring/init.lua"] = { + read_globals = { + "table.pack", + "table.unpack", + } +} + + +files["kong/hooks.lua"] = { + read_globals = { + "table.pack", + "table.unpack", + } +} + + +files["spec-ee/01-unit/07-keyring/01-init_spec.lua"] = { + read_globals = { + "table.pack", + } +} diff --git a/.requirements b/.requirements index bef9f855a60f..442aacc8205b 100644 --- a/.requirements +++ b/.requirements @@ -2,11 +2,10 @@ KONG_PACKAGE_NAME=kong KONG_CONFLICTS=kong-enterprise-edition KONG_LICENSE="ASL 2.0" -RESTY_VERSION=1.19.9.1 -RESTY_LUAROCKS_VERSION=3.8.0 -RESTY_OPENSSL_VERSION=1.1.1o -RESTY_PCRE_VERSION=8.45 -LIBYAML_VERSION=0.2.5 -KONG_GO_PLUGINSERVER_VERSION=v0.6.1 -KONG_BUILD_TOOLS_VERSION=4.40.1 -KONG_NGINX_MODULE_BRANCH=0.2.2 +OPENRESTY=1.19.9.1 +LUAROCKS=3.9.2 +OPENSSL=3.1.4 +PCRE=8.45 +LIBEXPAT=2.5.0 + +LUA_KONG_NGINX_MODULE=f52a34c17af4543245ff79e3227b9d514b8cfa5c # 0.2.2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d5a0c1afe65f..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,74 +0,0 @@ -dist: xenial - -language: go - -go: - - "1.13.x" - -notifications: - email: false - slack: - if: branch = master OR branch = next - secure: WfMWtoj88BvoWG1AqNQMZ0c9OqjseUaMIwgSpaP8SJ+H33NTMI7vMhbbMBN2ENRD8r2haV0A2D0fnHy/3GQi6ryzKsqR1Jw/c+J0QZXQgmuj51i9iKSOM42n59t9V/QuD5b2BmHt5T+1Q6jLxtgtqBKoaJk3+WE5Iz9l+f+v0Hs= - -services: - - redis-server - - docker - -addons: - postgresql: "9.5" - apt: - packages: - - net-tools - - libpcre3-dev - - valgrind - - build-essential - hosts: - - grpcs_1.test - - grpcs_2.test - - grpcbin - -env: - global: - - INSTALL_CACHE=$HOME/install-cache - - DOWNLOAD_ROOT=$HOME/download-root - - KONG_TEST_PG_DATABASE=travis - - KONG_TEST_PG_USER=travis - - KONG_TEST_PG_RO_USER=travis_ro - - KONG_TEST_PG_HOST=localhost - - JOBS=2 - - DOCKER_MACHINE_ARM64_NAME=travis-ci-kong-${TRAVIS_JOB_ID} - matrix: - - KONG_TEST_DATABASE=postgres TEST_SUITE=integration - - KONG_TEST_DATABASE=cassandra CASSANDRA=3.9 TEST_SUITE=integration - - KONG_TEST_DATABASE=off TEST_SUITE=dbless - - KONG_TEST_DATABASE=postgres TEST_SUITE=plugins - - KONG_TEST_DATABASE=cassandra CASSANDRA=3.9 TEST_SUITE=plugins - - TEST_SUITE=pdk - -install: - - source .ci/setup_env.sh - - make dev - -cache: - apt: true - directories: - - $INSTALL_CACHE - - $HOME/.ccm/repository - -stages: - - lint and unit - - test - -jobs: - include: - - stage: lint and unit - script: - - make lint - - scripts/autodoc - - bin/busted -v -o htest spec/01-unit - env: - - KONG_DATABASE=none - -script: - - .ci/run_tests.sh diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 000000000000..f0dcd8b016bb --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,238 @@ +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("//build/nfpm:rules.bzl", "nfpm_pkg") +load("@bazel_skylib//lib:selects.bzl", "selects") + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) + +filegroup( + name = "distribution_srcs", + srcs = glob(["distribution/**"]), + visibility = ["//visibility:public"], +) + +filegroup( + name = "rockspec_srcs", + srcs = glob(["*.rockspec"]), + visibility = ["//visibility:public"], +) + +filegroup( + name = "plugins_ee_rockspec_srcs", + srcs = glob(["plugins-ee/**/*.rockspec"]), + visibility = ["//visibility:public"], +) + +nfpm_env = { + "KONG_NAME": "kong", + "KONG_REPLACES_1": "kong-community-edition", + "KONG_REPLACES_2": "kong-enterprise-edition-fips", + "KONG_CONFLICTS_1": "kong-community-edition", + "KONG_CONFLICTS_2": "kong-enterprise-edition-fips", +} + +nfpm_pkg( + name = "kong_deb", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + packager = "deb", + pkg_name = "kong", + visibility = ["//visibility:public"], +) + +nfpm_pkg( + name = "kong_apk", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + packager = "apk", + pkg_name = "kong", + visibility = ["//visibility:public"], +) + +nfpm_pkg( + name = "kong_el8", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + packager = "rpm", + pkg_name = "kong.el8", + visibility = ["//visibility:public"], +) + +nfpm_pkg( + name = "kong_el7", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + extra_env = { + "RPM_EXTRA_DEPS": "hostname", + }, + packager = "rpm", + pkg_name = "kong.el7", + visibility = ["//visibility:public"], +) + +nfpm_pkg( + name = "kong_aws2", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + extra_env = { + "RPM_EXTRA_DEPS": "/usr/sbin/useradd", + "RPM_EXTRA_DEPS_2": "/usr/sbin/groupadd", + }, + packager = "rpm", + pkg_name = "kong.aws2", + visibility = ["//visibility:public"], +) + +nfpm_pkg( + name = "kong_aws2022", + config = "//build:package/nfpm.yaml", + env = nfpm_env, + extra_env = { + "RPM_EXTRA_DEPS": "/usr/sbin/useradd", + "RPM_EXTRA_DEPS_2": "/usr/sbin/groupadd", + "RPM_EXTRA_DEPS_3": "libxcrypt-compat", + }, + packager = "rpm", + pkg_name = "kong.aws2022", + visibility = ["//visibility:public"], +) + +###### flags + +# --//:debug=true +bool_flag( + name = "debug", + build_setting_default = True, +) + +config_setting( + name = "debug_flag", + flag_values = { + ":debug": "true", + }, + visibility = ["//visibility:public"], +) + +config_setting( + name = "debug_linux_flag", + constraint_values = [ + "@platforms//os:linux", + ], + flag_values = { + ":debug": "true", + }, + visibility = ["//visibility:public"], +) + +# --//:licensing=false +bool_flag( + name = "licensing", + build_setting_default = False, +) + +config_setting( + name = "licensing_flag", + flag_values = { + ":licensing": "true", + }, + visibility = ["//visibility:public"], +) + +# --//:fips=false +bool_flag( + name = "fips", + build_setting_default = False, +) + +config_setting( + name = "fips_flag", + flag_values = { + ":fips": "true", + }, + visibility = ["//visibility:public"], +) + +##### constraints, platforms and config_settings for cross-compile + +constraint_setting(name = "libc_version") + +constraint_value( + name = "glibc_2_35", + constraint_setting = ":libc_version", +) + +constraint_value( + name = "musl", + constraint_setting = ":libc_version", + visibility = ["//visibility:public"], +) + +# platform sets the constraint values based on user input (--platform=//:PLATFOTM) +platform( + name = "ubuntu-22.04-x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ":glibc_2_35", + ], +) + +platform( + name = "ubuntu-22.04-arm64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:arm64", + ":glibc_2_35", + ], +) + +platform( + name = "alpine-x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ":musl", + ], +) + +platform( + name = "macos-arm64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:arm64", + ], +) + +# config_settings define a select() condition based on user-set constraint_values +# see https://bazel.build/docs/configurable-attributes +config_setting( + name = "arm64-linux-gnu-cross", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:arm64", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "x86_64-linux-musl-cross", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ":musl", + ], + visibility = ["//visibility:public"], +) + +selects.config_setting_group( + # matches all cross build platforms + name = "any-cross", + match_any = [ + ":arm64-linux-gnu-cross", + ":x86_64-linux-musl-cross", + ], + visibility = ["//visibility:public"], +) diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 6faca2171045..000000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,313 +0,0 @@ -pipeline { - agent none - options { - retry(1) - timeout(time: 3, unit: 'HOURS') - } - environment { - UPDATE_CACHE = "true" - DOCKER_CREDENTIALS = credentials('dockerhub') - DOCKER_USERNAME = "${env.DOCKER_CREDENTIALS_USR}" - DOCKER_PASSWORD = "${env.DOCKER_CREDENTIALS_PSW}" - DOCKER_CLI_EXPERIMENTAL = "enabled" - // PULP_PROD and PULP_STAGE are used to do releases - PULP_HOST_PROD = "https://api.pulp.konnect-prod.konghq.com" - PULP_PROD = credentials('PULP') - PULP_HOST_STAGE = "https://api.pulp.konnect-stage.konghq.com" - PULP_STAGE = credentials('PULP_STAGE') - DEBUG = 0 - } - stages { - stage('Release -- Release Branch Release to Unofficial Asset Stores') { - when { - beforeAgent true - anyOf { - branch 'master'; - branch 'release/*'; - } - } - parallel { - stage('RPM') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - PACKAGE_TYPE = "rpm" - PRIVATE_KEY_FILE = credentials('kong.private.gpg-key.asc') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'cp $PRIVATE_KEY_FILE ../kong-build-tools/kong.private.gpg-key.asc' - sh 'make RESTY_IMAGE_BASE=amazonlinux KONG_TEST_CONTAINER_TAG="${GIT_BRANCH##*/}-amazonlinux" RESTY_IMAGE_TAG=2 release-docker-images' - } - } - stage('DEB') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - PACKAGE_TYPE = "deb" - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'make RESTY_IMAGE_BASE=debian KONG_TEST_CONTAINER_TAG="${GIT_BRANCH##*/}-debian" ADDITIONAL_TAG_LIST="${GIT_BRANCH##*/} ${GIT_COMMIT}" RESTY_IMAGE_TAG=11 release-docker-images' - sh 'make RESTY_IMAGE_BASE=ubuntu KONG_TEST_CONTAINER_TAG="${GIT_BRANCH##*/}-ubuntu" RESTY_IMAGE_TAG=20.04 release-docker-images' - } - } - stage('Alpine') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - AWS_ACCESS_KEY = "instanceprofile" - CACHE = false - PACKAGE_TYPE = "apk" - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - } - options { - retry(2) - timeout(time: 2, unit: 'HOURS') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'curl https://raw.githubusercontent.com/Kong/kong/master/scripts/setup-ci.sh | bash' - sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 KONG_TEST_CONTAINER_TAG="${GIT_BRANCH##*/}-alpine" DOCKER_MACHINE_ARM64_NAME="kong-"`cat /proc/sys/kernel/random/uuid` release-docker-images' - } - } - } - } - stage('Release -- Tag Release to Official Asset Stores') { - when { - beforeAgent true - allOf { - buildingTag() - not { triggeredBy 'TimerTrigger' } - } - } - parallel { - stage('RPM') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - PACKAGE_TYPE = "rpm" - PRIVATE_KEY_FILE = credentials('kong.private.gpg-key.asc') - PRIVATE_KEY_PASSPHRASE = credentials('kong.private.gpg-key.asc.password') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - } - options { - retry(2) - timeout(time: 2, unit: 'HOURS') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'cp $PRIVATE_KEY_FILE ../kong-build-tools/kong.private.gpg-key.asc' - sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2 release' - sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2022 release' - sh 'make RESTY_IMAGE_BASE=centos RESTY_IMAGE_TAG=7 release' - sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=7.9 release' - sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 RELEASE_DOCKER=true release' - } - } - stage('DEB') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - PACKAGE_TYPE = "deb" - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - } - options { - retry(2) - timeout(time: 2, unit: 'HOURS') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=10 release' - sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=11 RELEASE_DOCKER=true release' - sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=18.04 release' - sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=20.04 RELEASE_DOCKER=true release' - sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 RELEASE_DOCKER=true release' - } - } - stage('SRC & Alpine') { - agent { - node { - label 'bionic' - } - } - environment { - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - PACKAGE_TYPE = "rpm" - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - AWS_ACCESS_KEY = "instanceprofile" - CACHE = false - } - options { - retry(2) - timeout(time: 2, unit: 'HOURS') - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'curl https://raw.githubusercontent.com/Kong/kong/master/scripts/setup-ci.sh | bash' - sh 'make RESTY_IMAGE_BASE=src RESTY_IMAGE_TAG=src PACKAGE_TYPE=src release' - sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 PACKAGE_TYPE=apk DOCKER_MACHINE_ARM64_NAME="kong-"`cat /proc/sys/kernel/random/uuid` RELEASE_DOCKER=true release' - } - } - } - } - stage('Post Release Steps') { - when { - beforeAgent true - allOf { - buildingTag() - not { triggeredBy 'TimerTrigger' } - expression { env.TAG_NAME ==~ /^\d+\.\d+\.\d+$/ } - } - } - parallel { - stage('PR Docker') { - agent { - node { - label 'bionic' - } - } - environment { - GITHUB_TOKEN = credentials('github_bot_access_token') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - SLACK_WEBHOOK = credentials('core_team_slack_webhook') - GITHUB_USER = "mashapedeployment" - } - steps { - sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-release $TAG_NAME update_docker' - } - post { - failure { - script { - sh 'SLACK_MESSAGE="updating docker-kong failed" ./scripts/send-slack-message.sh' - } - } - success { - script { - sh 'SLACK_MESSAGE="updating docker-kong succeeded. Please review, approve and continue with the kong release script" ./scripts/send-slack-message.sh' - } - } - } - } - stage('PR Homebrew') { - agent { - node { - label 'bionic' - } - } - environment { - GITHUB_TOKEN = credentials('github_bot_access_token') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - SLACK_WEBHOOK = credentials('core_team_slack_webhook') - GITHUB_USER = "mashapedeployment" - } - steps { - sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-release $TAG_NAME homebrew' - } - post { - failure { - script { - sh 'SLACK_MESSAGE="updating homebrew-kong failed" ./scripts/send-slack-message.sh' - } - } - success { - script { - sh 'SLACK_MESSAGE="updating homebrew-kong succeeded. Please review, approve and merge the PR" ./scripts/send-slack-message.sh' - } - } - } - } - stage('PR Vagrant') { - agent { - node { - label 'bionic' - } - } - environment { - GITHUB_TOKEN = credentials('github_bot_access_token') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - SLACK_WEBHOOK = credentials('core_team_slack_webhook') - GITHUB_USER = "mashapedeployment" - } - steps { - sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-release $TAG_NAME vagrant' - } - post { - failure { - script { - sh 'SLACK_MESSAGE="updating kong-vagrant failed" ./scripts/send-slack-message.sh' - } - } - success { - script { - sh 'SLACK_MESSAGE="updating kong-vagrant succeeded. Please review, approve and merge the PR" ./scripts/send-slack-message.sh' - } - } - } - } - stage('PR Pongo') { - agent { - node { - label 'bionic' - } - } - environment { - GITHUB_TOKEN = credentials('github_bot_access_token') - GITHUB_SSH_KEY = credentials('github_bot_ssh_key') - SLACK_WEBHOOK = credentials('core_team_slack_webhook') - GITHUB_USER = "mashapedeployment" - } - steps { - sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-release $TAG_NAME pongo' - } - post { - always { - script { - sh 'SLACK_MESSAGE="pongo branch is pushed go open the PR at https://github.com/Kong/kong-pongo/branches" ./scripts/send-slack-message.sh' - } - } - } - } - } - } - } -} - diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 000000000000..24b272f535b9 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,53 @@ +workspace(name = "kong") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "bazel_skylib", + sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + ], +) + +load("//build:kong_bindings.bzl", "load_bindings") + +load_bindings(name = "kong_bindings") + +http_archive( + name = "rules_foreign_cc", + sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51", + strip_prefix = "rules_foreign_cc-0.9.0", + url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz", +) + +load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") + +# This sets up some common toolchains for building targets. For more details, please see +# https://bazelbuild.github.io/rules_foreign_cc/0.9.0/flatten.html#rules_foreign_cc_dependencies +# TODO: select system `make` if installed, otherwise automatically build +rules_foreign_cc_dependencies( + register_built_tools = False, + register_default_tools = False, +) + +load("//build/openresty:repositories.bzl", "openresty_repositories") + +openresty_repositories() + +load("//build/nfpm:repositories.bzl", "nfpm_repositories") + +nfpm_repositories() + +load("//build:repositories.bzl", "build_repositories") + +build_repositories() + +load("//build/toolchain:repositories.bzl", "toolchain_repositories") + +toolchain_repositories() + +register_toolchains("//build/toolchain:gcc_cross_arm64_toolchain") + +register_toolchains("//build/toolchain:gcc_cross_musl_x86_64_toolchain") diff --git a/build/BUILD.bazel b/build/BUILD.bazel new file mode 100644 index 000000000000..28827a8eb055 --- /dev/null +++ b/build/BUILD.bazel @@ -0,0 +1,187 @@ +load("@kong_bindings//:variables.bzl", "KONG_VAR") +load("//build:build_system.bzl", "kong_directory_genrule") + +exports_files([ + "package/nfpm.yaml", +]) + +lib_deps = [ + "@openssl", #TODO: select over fips (but select doesn't work in list comprehension) +] + +install_lib_deps_cmd = "\n".join([ + """ + DEP=${WORKSPACE_PATH}/$(echo $(locations %s) | awk '{print $1}') + # use tar magic to exclude files and create with correct permission + copy_with_filter ${DEP} ${BUILD_DESTDIR}/kong +""" % dep + for dep in lib_deps +]) + +lualib_deps = [ + "@lua-kong-nginx-module//:all_srcs", + "@lua-resty-lmdb//:all_srcs", + "@lua-resty-events//:all_srcs", + "@lua-resty-websocket//:all_srcs", + "@atc_router", +] + +install_lualib_deps_cmd = "\n".join([ + """ + DEP=$(pwd)/external/%s + # TODO: fix atc_router makefile so that we can choose to only install lualib + sed -i -e '/libatc_router.so/d' ${DEP}/Makefile + make --silent -C ${DEP} LUA_LIB_DIR=${BUILD_DESTDIR}/openresty/lualib install +""" % dep.lstrip("@").split("/")[0] + for dep in lualib_deps +]) + +kong_directory_genrule( + name = "kong", + srcs = [ + "@openresty//:openresty", + "@openresty//:luajit", + "@luarocks//:luarocks_make", + "@luarocks//:luarocks_target", + "@protoc//:all_srcs", + ] + lib_deps + lualib_deps, + cmd = + """ set -e + function copy_with_filter { + mkdir -p $2 + tar -cC $1 --exclude="*.a" --exclude="*.la" \ + --exclude="*/share/*" --exclude="*/bin/*" \ + --exclude="*.log" . | tar -xC $2/. + chmod -R "+rw" $2 + } + function LN { + if [[ "$OSTYPE" == "darwin"* ]]; then + # TODO: support relative path links once we start to cross compile on macOS + ln -sf $@ + else + ln -srf $@ + fi + } + rm -rf ${BUILD_DESTDIR} + mkdir -p ${BUILD_DESTDIR}/kong/lib ${BUILD_DESTDIR}/openresty ${BUILD_DESTDIR}/bin + + if [[ "$OSTYPE" == "darwin"* ]]; then + libext="dylib" + else # assume linux + libext="so" + fi + + OPENRESTY=${WORKSPACE_PATH}/$(echo '$(locations @openresty//:openresty)' | awk '{print $1}') + cp -r ${OPENRESTY}/. ${BUILD_DESTDIR}/openresty/. + LN ${BUILD_DESTDIR}/openresty/bin/resty ${BUILD_DESTDIR}/bin/resty + chmod -R "+rw" ${BUILD_DESTDIR}/openresty + + LUAJIT=${WORKSPACE_PATH}/$(echo '$(locations @openresty//:luajit)' | awk '{print $1}') + copy_with_filter ${LUAJIT} ${BUILD_DESTDIR}/openresty/luajit + cp ${LUAJIT}/bin/luajit ${BUILD_DESTDIR}/openresty/luajit/bin/luajit + tar -cC ${LUAJIT}/share . | tar -xC ${BUILD_DESTDIR}/openresty/luajit/share + chmod -R "+rw" ${BUILD_DESTDIR}/openresty/luajit + + LUAROCKS=${WORKSPACE_PATH}/$(dirname '$(location @luarocks//:luarocks_make)')/luarocks_tree + cp -r ${LUAROCKS}/. ${BUILD_DESTDIR}/. + rm ${BUILD_DESTDIR}/bin/lapis ${BUILD_DESTDIR}/bin/luarocks-admin + + ATC_ROUTER=${WORKSPACE_PATH}/$(location @atc_router) + cp $ATC_ROUTER ${BUILD_DESTDIR}/openresty/lualib/. + + cp -r $(locations @protoc//:all_srcs) ${BUILD_DESTDIR}/kong/. + + """ + install_lib_deps_cmd + install_lualib_deps_cmd + + """ + mkdir -p ${BUILD_DESTDIR}/etc/kong + cp kong.conf.default ${BUILD_DESTDIR}/etc/kong/kong.conf.default + + # housecleaning + mv ${BUILD_DESTDIR}/kong/*.${libext}* ${BUILD_DESTDIR}/kong/lib 2>/dev/null || true + if [[ -d ${BUILD_DESTDIR}/kong/lib64 ]]; then + copy_with_filter ${BUILD_DESTDIR}/kong/lib64 ${BUILD_DESTDIR}/kong/lib + rm -rf ${BUILD_DESTDIR}/kong/lib64 + fi + + # remove pkgconfig since they are invalid anyway + find ${BUILD_DESTDIR} -name "*.pc" -delete + + # clean empty directory + find ${BUILD_DESTDIR} -empty -type d -delete + + # foreign_cc rule dereferences symlink, we will dedup them here + # TODO: patch https://github.com/bazelbuild/rules_foreign_cc/blob/main/foreign_cc/private/framework.bzl#L450 to not remove symlink + for f in $(find ${BUILD_DESTDIR}/kong/lib ${BUILD_DESTDIR}/openresty/luajit/lib -type f -name "*.${libext}*" ); do + if [[ -L "$f" ]]; then continue; fi # already a symlink + target=$(ls -r1 $f.* 2>/dev/null | head -n1) + if [[ ! -z "$target" && "$f" != "$target" ]]; then + LN "$target" "$f" + fi + done + LN ${BUILD_DESTDIR}/openresty/nginx/sbin/nginx ${BUILD_DESTDIR}/openresty/bin/openresty + """, + # XXX: bazel forces 0555 as artifact permission, which is not correct for packagin + # here we deliberately use a different directory so file permission is preserved + # see also https://github.com/bazelbuild/bazel/issues/5588 + output_dir = KONG_VAR["BUILD_NAME"] + ".nop", + visibility = ["//visibility:public"], +) + +genrule( + name = "venv", + srcs = [ + ":kong", + ], + outs = [ + "%s-venv.sh" % KONG_VAR["BUILD_NAME"], + ], + cmd = """ + workspace_path={workspace_path} + build_name={build_name} + """.format( + build_name = KONG_VAR["BUILD_NAME"], + workspace_path = KONG_VAR["WORKSPACE_PATH"], + ) + """ + + cat << EOF > $@ +#!/bin/bash + +INSTALL_ROOT=$${workspace_path}/bazel-bin/build/$${build_name} +ROCKS_CONFIG="\\$$INSTALL_ROOT/rocks_config" +ROCKS_ROOT="\\$$INSTALL_ROOT" + +chmod -R a+rw "\\$$INSTALL_ROOT" + +export LD_LIBRARY_PATH=\\$$INSTALL_ROOT/kong/lib +mkdir -p \\$$INSTALL_ROOT/venv/bin + +echo '#!/bin/bash +'\\$$INSTALL_ROOT/openresty/bin/resty -I \\$$INSTALL_ROOT/openresty/site/lualib -I \\$$INSTALL_ROOT/openresty/lualib --nginx \\$$INSTALL_ROOT/openresty/nginx/sbin/nginx' "\\$$@" +' > \\$$INSTALL_ROOT/venv/bin/resty +chmod +x \\$$INSTALL_ROOT/venv/bin/resty + +export PATH="\\$$INSTALL_ROOT/venv/bin:\\$$INSTALL_ROOT/openresty/bin:\\$$INSTALL_ROOT/openresty/nginx/sbin:\\$$INSTALL_ROOT/openresty/luajit/bin:\\$$INSTALL_ROOT/luarocks/bin:\\$$INSTALL_ROOT/bin:$${workspace_path}/bin:\\$$PATH" + +echo " +rocks_trees = { +{ name = [[system]], root = [[\\$$ROCKS_ROOT]] } +} + +" > \\$$ROCKS_CONFIG + +export LUAROCKS_CONFIG=\\$$ROCKS_CONFIG + +# duplicate package.[c]path even though we have set in resty-cli, so luajit and kong can consume +export LUA_PATH="./?.lua;./?/init.lua;\\$$INSTALL_ROOT/openresty/site/lualib/?.ljbc;\\$$INSTALL_ROOT/openresty/site/lualib/?/init.ljbc;\\$$INSTALL_ROOT/openresty/lualib/?.ljbc;\\$$INSTALL_ROOT/openresty/lualib/?/init.ljbc;\\$$INSTALL_ROOT/openresty/site/lualib/?.lua;\\$$INSTALL_ROOT/openresty/site/lualib/?/init.lua;\\$$INSTALL_ROOT/openresty/lualib/?.lua;\\$$INSTALL_ROOT/openresty/lualib/?/init.lua;\\$$INSTALL_ROOT/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;\\$$ROCKS_ROOT/share/lua/5.1/?.ljbc;\\$$ROCKS_ROOT/share/lua/5.1/?/init.ljbc;\\$$ROCKS_ROOT/share/lua/5.1/?.lua;\\$$ROCKS_ROOT/share/lua/5.1/?/init.lua;;" + +export LUA_CPATH="\\$$INSTALL_ROOT/openresty/site/lualib/?.so;\\$$INSTALL_ROOT/openresty/lualib/?.so;./?.so;\\$$INSTALL_ROOT/lib/lua/5.1/?.so;\\$$INSTALL_ROOT/openresty/luajit/lib/lua/5.1/?.so;\\$$ROCKS_ROOT/lib/lua/5.1/?.so;;" +export KONG_PREFIX="\\$$INSTALL_ROOT/kong/servroot" +export LIBRARY_PREFIX="\\$$INSTALL_ROOT/kong" # let "make dev" happy + +EOF + + echo \\* Please run ". $@" to activate the "$${build_name}" environment + """, + executable = True, + visibility = ["//visibility:public"], +) diff --git a/build/Makefile b/build/Makefile new file mode 120000 index 000000000000..d0b0e8e0086f --- /dev/null +++ b/build/Makefile @@ -0,0 +1 @@ +../Makefile \ No newline at end of file diff --git a/build/README.md b/build/README.md new file mode 100644 index 000000000000..1d4faaee8488 --- /dev/null +++ b/build/README.md @@ -0,0 +1,135 @@ +# Build + +This directory contains the build system for the project. +The build system is designed to be used with the [Bazel](https://bazel.build/). +It is designed to be running on Linux without root privileges, and no virtualization technology is required. + +The build system is tested on Linux (Ubuntu/Debian) and macOS (M1, Ventura). + +## Prerequisites + +The build system requires the following tools to be installed: + +- [Bazel/Bazelisk](https://bazel.build/install/bazelisk), Bazelisk is recommended to ensure the correct version of Bazel is used. +- [Build Dependencies](https://github.com/Kong/kong/blob/master/DEVELOPER.md#prerequisites), the build system requires the same dependencies as Kong itself. + +## Building + +To build Kong and all its dependencies, run the following command: + +Bash/Zsh: + +```bash +bazel build //build:kong --verbose_failures +``` + +The build output is in `bazel-bin/build/kong-dev`. + +To use the build as a virtual development environment, run: + +```bash +bazel build //build:venv --verbose_failures +. ./bazel-bin/build/kong-dev-venv.sh +``` + +Some other targets one might find useful for debugging are: + +- `@openresty//:openresty`: builds openresty +- `@luarocks//:luarocks_make`: builds luarocks for Kong dependencies + +### Build Options + +Following build options can be used to set specific features: + +- **--//:debug=true** turn on debug opitons for OpenResty and LuaJIT, default to true. +- **--action_env=BUILD_NAME=** set the `build_name`, multiple build can exist at same time to allow you +switch between different Kong versions or branches. Default to `kong-dev`; don't set this when you are +building a building an binary package. +- **--action_env=INSTALL_DESTDIR=** set the directory when the build is intended to be installed. Bazel won't +actually install files into this directory, but this will make sure certain hard coded paths and RPATH is +correctly set when building a package. Default to `bazel-bin/build/`. + + +### Official build + +`--config release` specifies the build configuration to use for release, it sets following build options: + +``` +build:release --//:debug=false +build:release --action_env=BUILD_NAME=kong-dev +build:release --action_env=INSTALL_DESTDIR=/usr/local +``` + +To build an official release, use: + +```bash +bazel build --config release //build:kong --verbose_failures +``` + +Supported build targets for binary packages: +- `:kong_deb` +- `:kong_el7` +- `:kong_el8` +- `:kong_aws2` +- `:kong_aws2022` +- `:kong_apk` + +For example, to build the deb package: + +```bash +bazel build --verbose_failures --config release :kong_deb + +``` + +Run `bazel clean` to clean the bazel build cache. + +#### GPG Signing + +GPG singing is supported for the rpm packages (`el*` and `aws*`). + +```bash +bazel build //:kong_el8 --action_env=RPM_SIGNING_KEY_FILE --action_env=NFPM_RPM_PASSPHRASE +``` + +## Cross compiling + +Cross compiling is currently only tested on Ubuntu 22.04 x86_64 with following targeting platforms: + +- **//:ubuntu-22.04-arm64** Ubuntu 22.04 ARM64 + - Requires user to manually install `crossbuild-essential-arm64`. +- **//:alpine-x86_64** Alpine Linux x86_64; bazel manages the build toolchain. + +Make sure platforms are selected both in building Kong and packaing kong: + +```bash +bazel build --config release //build:kong --platforms=//:ubuntu-2204-arm64 +azel build --config release :kong_deb --platforms=//:ubuntu-2204-arm64 +``` + +## Troubleshooting + +Run `bazel build` with `--sandbox_debug --verbose_failures` to get more information about the error. + +The `.log` files in `bazel-bin` contain the build logs. + +## FAQ + +### Caching + +Bazel utilizes a cache to speed up the build process. You might want to clear the cache actively +if you recently changed `BUILD_NAME` or `INSTALL_DESTDIR`. + +To completely remove the entire working tree created by a Bazel instance, run: + +```shell +bazel clean --expunge +``` + +### Cleanup + +In some cases where the build fails or the build is interrupted, the build system may leave behind some temporary files. To clean up the build system, run the following command or simply rerun the build: + +```shell +bazel clean +``` + diff --git a/build/build_system.bzl b/build/build_system.bzl new file mode 100644 index 000000000000..a6be92f75940 --- /dev/null +++ b/build/build_system.bzl @@ -0,0 +1,47 @@ +""" +Load this file for all Kong-specific build macros +and rules that you'd like to use in your BUILD files. +""" + +load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +# A genrule variant that can output a directory. +def _kong_directory_genrule_impl(ctx): + tree = ctx.actions.declare_directory(ctx.attr.output_dir) + env = dicts.add(KONG_VAR, ctx.configuration.default_shell_env, { + "GENRULE_OUTPUT_DIR": tree.path, + }) + + # XXX: remove the "env" from KONG_VAR which is a list + env["OPENRESTY_PATCHES"] = "" + + ctx.actions.run_shell( + inputs = ctx.files.srcs, + tools = ctx.files.tools, + outputs = [tree], + command = "mkdir -p " + tree.path + " && " + ctx.expand_location(ctx.attr.cmd), + env = env, + ) + return [DefaultInfo(files = depset([tree]))] + +kong_directory_genrule = rule( + implementation = _kong_directory_genrule_impl, + attrs = { + "srcs": attr.label_list(), + "cmd": attr.string(), + "tools": attr.label_list(), + "output_dir": attr.string(), + }, +) + +# A rule that can be used as a meta rule that propagates multiple other rules +def _kong_rules_group_impl(ctx): + return [DefaultInfo(files = depset(ctx.files.propagates))] + +kong_rules_group = rule( + implementation = _kong_rules_group_impl, + attrs = { + "propagates": attr.label_list(), + }, +) diff --git a/build/cross_deps/BUILD.bazel b/build/cross_deps/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/cross_deps/libxcrypt/BUILD.bazel b/build/cross_deps/libxcrypt/BUILD.bazel new file mode 100644 index 000000000000..d7862e9d016e --- /dev/null +++ b/build/cross_deps/libxcrypt/BUILD.bazel @@ -0,0 +1,6 @@ +exports_files( + [ + "BUILD.libxcrypt.bazel", + ], + visibility = ["//visibility:public"], +) diff --git a/build/cross_deps/libxcrypt/BUILD.libxcrypt.bazel b/build/cross_deps/libxcrypt/BUILD.libxcrypt.bazel new file mode 100644 index 000000000000..d0cf8e8909a7 --- /dev/null +++ b/build/cross_deps/libxcrypt/BUILD.libxcrypt.bazel @@ -0,0 +1,40 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +configure_make( + name = "libxcrypt", + configure_command = "configure", + configure_in_place = True, + configure_options = select({ + "@kong//:arm64-linux-gnu-cross": [ + "--host=aarch64-linux", + ], + "@kong//:x86_64-linux-musl-cross": [ + "--host=x86_64-linux-musl", + ], + "//conditions:default": [], + }), + lib_source = ":all_srcs", + # out_lib_dir = "lib", + out_shared_libs = select({ + "@platforms//os:macos": [ + "libcrypt.1.dylib", + ], + "//conditions:default": [ + "libcrypt.so.1", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + "install -j" + KONG_VAR["NPROC"], + ], + visibility = ["//visibility:public"], +) diff --git a/build/cross_deps/libxcrypt/repositories.bzl b/build/cross_deps/libxcrypt/repositories.bzl new file mode 100644 index 000000000000..f6c28d022445 --- /dev/null +++ b/build/cross_deps/libxcrypt/repositories.bzl @@ -0,0 +1,18 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def libxcrypt_repositories(): + """Defines the libcrypt repository""" + + # many distros starts replace glibc/libcrypt with libxcrypt + # thus crypt.h and libcrypt.so.1 are missing from cross tool chain + # ubuntu2004: 4.4.10 + # ubuntu2204: 4.4.27 + http_archive( + name = "cross_deps_libxcrypt", + url = "https://github.com/besser82/libxcrypt/releases/download/v4.4.27/libxcrypt-4.4.27.tar.xz", + sha256 = "500898e80dc0d027ddaadb5637fa2bf1baffb9ccd73cd3ab51d92ef5b8a1f420", + strip_prefix = "libxcrypt-4.4.27", + build_file = "//build/cross_deps/libxcrypt:BUILD.libxcrypt.bazel", + ) diff --git a/build/cross_deps/libyaml/BUILD.bazel b/build/cross_deps/libyaml/BUILD.bazel new file mode 100644 index 000000000000..588b8759be7f --- /dev/null +++ b/build/cross_deps/libyaml/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +exports_files( + [ + "BUILD.libyaml.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@cross_deps_libyaml//:libyaml", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/cross_deps/libyaml/BUILD.libyaml.bazel b/build/cross_deps/libyaml/BUILD.libyaml.bazel new file mode 100644 index 000000000000..1a0214757704 --- /dev/null +++ b/build/cross_deps/libyaml/BUILD.libyaml.bazel @@ -0,0 +1,40 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +configure_make( + name = "libyaml", + configure_command = "configure", + configure_in_place = True, + configure_options = select({ + "@kong//:arm64-linux-gnu-cross": [ + "--host=aarch64-linux", + ], + "@kong//:x86_64-linux-musl-cross": [ + "--host=x86_64-linux-musl", + ], + "//conditions:default": [], + }), + lib_source = ":all_srcs", + # out_lib_dir = "lib", + out_shared_libs = select({ + "@platforms//os:macos": [ + "libyaml-0.2.dylib", + ], + "//conditions:default": [ + "libyaml-0.so.2", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + "install -j" + KONG_VAR["NPROC"], + ], + visibility = ["//visibility:public"], +) diff --git a/build/cross_deps/libyaml/repositories.bzl b/build/cross_deps/libyaml/repositories.bzl new file mode 100644 index 000000000000..b7b2800cf965 --- /dev/null +++ b/build/cross_deps/libyaml/repositories.bzl @@ -0,0 +1,15 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def libyaml_repositories(): + """Defines the libyaml repository""" + + http_archive( + name = "cross_deps_libyaml", + url = "https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz", + sha256 = "c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4", + strip_prefix = "yaml-0.2.5", + build_file = "//build/cross_deps/libyaml:BUILD.libyaml.bazel", + ) diff --git a/build/cross_deps/repositories.bzl b/build/cross_deps/repositories.bzl new file mode 100644 index 000000000000..a2afddfc9e9d --- /dev/null +++ b/build/cross_deps/repositories.bzl @@ -0,0 +1,8 @@ +load("//build/cross_deps/zlib:repositories.bzl", "zlib_repositories") +load("//build/cross_deps/libyaml:repositories.bzl", "libyaml_repositories") +load("//build/cross_deps/libxcrypt:repositories.bzl", "libxcrypt_repositories") + +def cross_deps_repositories(): + zlib_repositories() + libyaml_repositories() + libxcrypt_repositories() diff --git a/build/cross_deps/zlib/BUILD.bazel b/build/cross_deps/zlib/BUILD.bazel new file mode 100644 index 000000000000..d650c675249b --- /dev/null +++ b/build/cross_deps/zlib/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +exports_files( + [ + "BUILD.zlib.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@cross_deps_zlib//:zlib", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/cross_deps/zlib/BUILD.zlib.bazel b/build/cross_deps/zlib/BUILD.zlib.bazel new file mode 100644 index 000000000000..a82ac6977817 --- /dev/null +++ b/build/cross_deps/zlib/BUILD.zlib.bazel @@ -0,0 +1,49 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +cmake( + name = "zlib", + build_args = [ + "--", # <- Pass remaining options to the native tool. + "-j" + KONG_VAR["NPROC"], + ], + # partially from https://github.com/envoyproxy/envoy/blob/main/bazel/foreign_cc/BUILD#L546 + cache_entries = { + "CMAKE_CXX_COMPILER_FORCED": "on", + "CMAKE_C_COMPILER_FORCED": "on", + "SKIP_BUILD_EXAMPLES": "on", + "BUILD_SHARED_LIBS": "ON", + + # The following entries are for zlib-ng. Since zlib and zlib-ng are compatible source + # codes and CMake ignores unknown cache entries, it is fine to combine it into one + # dictionary. + # + # Reference: https://github.com/zlib-ng/zlib-ng#build-options. + "ZLIB_COMPAT": "on", + "ZLIB_ENABLE_TESTS": "off", + + # Warning: Turning WITH_OPTIM to "on" doesn't pass ZlibCompressorImplTest.CallingChecksum. + "WITH_OPTIM": "on", + # However turning off SSE4 fixes it. + "WITH_SSE4": "off", + + # Warning: Turning WITH_NEW_STRATEGIES to "on" doesn't pass gzip compressor fuzz test. + # Turning this off means falling into NO_QUICK_STRATEGY route. + "WITH_NEW_STRATEGIES": "off", + + # Only allow aligned address. + # Reference: https://github.com/zlib-ng/zlib-ng#advanced-build-options. + "UNALIGNED_OK": "off", + }, + lib_source = ":all_srcs", + out_shared_libs = ["libz.so.1"], + visibility = ["//visibility:public"], +) diff --git a/build/cross_deps/zlib/repositories.bzl b/build/cross_deps/zlib/repositories.bzl new file mode 100644 index 000000000000..cd6e78ec2621 --- /dev/null +++ b/build/cross_deps/zlib/repositories.bzl @@ -0,0 +1,15 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def zlib_repositories(): + """Defines the zlib repository""" + + http_archive( + name = "cross_deps_zlib", + url = "https://zlib.net/zlib-1.2.13.tar.gz", + sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + strip_prefix = "zlib-1.2.13", + build_file = "//build/cross_deps/zlib:BUILD.zlib.bazel", + ) diff --git a/build/dockerfiles/apk.Dockerfile b/build/dockerfiles/apk.Dockerfile new file mode 100644 index 000000000000..d1fed242ee5c --- /dev/null +++ b/build/dockerfiles/apk.Dockerfile @@ -0,0 +1,49 @@ +ARG KONG_BASE_IMAGE=alpine:3.18 +FROM $KONG_BASE_IMAGE + +LABEL maintainer="Kong Docker Maintainers (@team-gateway-bot)" + +ARG KONG_VERSION +ENV KONG_VERSION $KONG_VERSION + +ARG KONG_PREFIX=/usr/local/kong +ENV KONG_PREFIX $KONG_PREFIX + +ARG EE_PORTS + +ARG TARGETARCH + +ARG KONG_ARTIFACT=kong.${TARGETARCH}.apk.tar.gz +ARG KONG_ARTIFACT_PATH= +COPY ${KONG_ARTIFACT_PATH}${KONG_ARTIFACT} /tmp/kong.apk.tar.gz + +RUN apk add --virtual .build-deps tar gzip \ + && tar -C / -xzf /tmp/kong.apk.tar.gz \ + && apk add --no-cache libstdc++ libgcc pcre perl tzdata libcap zlib zlib-dev bash yaml \ + && adduser -S kong \ + && addgroup -S kong \ + && mkdir -p "${KONG_PREFIX}" \ + && chown -R kong:0 ${KONG_PREFIX} \ + && chown kong:0 /usr/local/bin/kong \ + && chmod -R g=u ${KONG_PREFIX} \ + && rm -rf /tmp/kong.apk.tar.gz \ + && ln -sf /usr/local/openresty/bin/resty /usr/local/bin/resty \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/luajit \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/lua \ + && ln -sf /usr/local/openresty/nginx/sbin/nginx /usr/local/bin/nginx \ + && apk del .build-deps \ + && kong version + +COPY build/dockerfiles/entrypoint.sh /entrypoint.sh + +USER kong + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 8000 8443 8001 8444 $EE_PORTS + +STOPSIGNAL SIGQUIT + +HEALTHCHECK --interval=60s --timeout=10s --retries=10 CMD kong-health + +CMD ["kong", "docker-start"] diff --git a/build/dockerfiles/deb.Dockerfile b/build/dockerfiles/deb.Dockerfile new file mode 100644 index 000000000000..0a568272f8f0 --- /dev/null +++ b/build/dockerfiles/deb.Dockerfile @@ -0,0 +1,44 @@ +ARG KONG_BASE_IMAGE=debian:bullseye-slim +FROM $KONG_BASE_IMAGE + +LABEL maintainer="Kong Docker Maintainers (@team-gateway-bot)" + +ARG KONG_VERSION +ENV KONG_VERSION $KONG_VERSION + +ARG KONG_PREFIX=/usr/local/kong +ENV KONG_PREFIX $KONG_PREFIX + +ARG EE_PORTS + +ARG TARGETARCH + +ARG KONG_ARTIFACT=kong.${TARGETARCH}.deb +ARG KONG_ARTIFACT_PATH= +COPY ${KONG_ARTIFACT_PATH}${KONG_ARTIFACT} /tmp/kong.deb + +RUN apt-get update \ + && apt-get install -y --no-install-recommends /tmp/kong.deb \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/kong.deb \ + && chown kong:0 /usr/local/bin/kong \ + && chown -R kong:0 ${KONG_PREFIX} \ + && ln -sf /usr/local/openresty/bin/resty /usr/local/bin/resty \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/luajit \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/lua \ + && ln -sf /usr/local/openresty/nginx/sbin/nginx /usr/local/bin/nginx \ + && kong version + +COPY build/dockerfiles/entrypoint.sh /entrypoint.sh + +USER kong + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 8000 8443 8001 8444 $EE_PORTS + +STOPSIGNAL SIGQUIT + +HEALTHCHECK --interval=60s --timeout=10s --retries=10 CMD kong-health + +CMD ["kong", "docker-start"] diff --git a/build/dockerfiles/entrypoint.sh b/build/dockerfiles/entrypoint.sh new file mode 100755 index 000000000000..f4f2a499b777 --- /dev/null +++ b/build/dockerfiles/entrypoint.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -Eeo pipefail + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + # Do not continue if _FILE env is not set + if ! [ "${!fileVar:-}" ]; then + return + elif [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +export KONG_NGINX_DAEMON=${KONG_NGINX_DAEMON:=off} + +if [[ "$1" == "kong" ]]; then + + all_kong_options="/usr/local/share/lua/5.1/kong/templates/kong_defaults.lua" + set +Eeo pipefail + while IFS='' read -r LINE || [ -n "${LINE}" ]; do + opt=$(echo "$LINE" | grep "=" | sed "s/=.*$//" | sed "s/ //" | tr '[:lower:]' '[:upper:]') + file_env "KONG_$opt" + done < $all_kong_options + set -Eeo pipefail + + file_env KONG_PASSWORD + PREFIX=${KONG_PREFIX:=/usr/local/kong} + + if [[ "$2" == "docker-start" ]]; then + kong prepare -p "$PREFIX" "$@" + + ln -sfn /dev/stdout $PREFIX/logs/access.log + ln -sfn /dev/stdout $PREFIX/logs/admin_access.log + ln -sfn /dev/stderr $PREFIX/logs/error.log + + exec /usr/local/openresty/nginx/sbin/nginx \ + -p "$PREFIX" \ + -c nginx.conf + fi +fi + +exec "$@" diff --git a/build/dockerfiles/rpm.Dockerfile b/build/dockerfiles/rpm.Dockerfile new file mode 100644 index 000000000000..b58d96c400f7 --- /dev/null +++ b/build/dockerfiles/rpm.Dockerfile @@ -0,0 +1,55 @@ +ARG KONG_BASE_IMAGE=redhat/ubi8 +FROM $KONG_BASE_IMAGE + +LABEL maintainer="Kong Docker Maintainers (@team-gateway-bot)" + +ARG KONG_VERSION +ENV KONG_VERSION $KONG_VERSION + +# RedHat required labels +LABEL name="Kong" \ + vendor="Kong" \ + version="$KONG_VERSION" \ + release="1" \ + url="https://konghq.com" \ + summary="Next-Generation API Platform for Modern Architectures" \ + description="Next-Generation API Platform for Modern Architectures" + +# RedHat required LICENSE file approved path +COPY LICENSE /licenses/ + +ARG KONG_PREFIX=/usr/local/kong +ENV KONG_PREFIX $KONG_PREFIX + +ARG EE_PORTS + +ARG TARGETARCH + +ARG KONG_ARTIFACT=kong.el8.${TARGETARCH}.rpm +ARG KONG_ARTIFACT_PATH= +COPY ${KONG_ARTIFACT_PATH}${KONG_ARTIFACT} /tmp/kong.rpm + +# hadolint ignore=DL3015 +RUN yum install -y /tmp/kong.rpm \ + && rm /tmp/kong.rpm \ + && chown kong:0 /usr/local/bin/kong \ + && chown -R kong:0 /usr/local/kong \ + && ln -sf /usr/local/openresty/bin/resty /usr/local/bin/resty \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/luajit \ + && ln -sf /usr/local/openresty/luajit/bin/luajit /usr/local/bin/lua \ + && ln -sf /usr/local/openresty/nginx/sbin/nginx /usr/local/bin/nginx \ + && kong version + +COPY build/dockerfiles/entrypoint.sh /entrypoint.sh + +USER kong + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 8000 8443 8001 8444 $EE_PORTS + +STOPSIGNAL SIGQUIT + +HEALTHCHECK --interval=60s --timeout=10s --retries=10 CMD kong-health + +CMD ["kong", "docker-start"] diff --git a/build/kong_bindings.bzl b/build/kong_bindings.bzl new file mode 100644 index 000000000000..f44ddd2ccfde --- /dev/null +++ b/build/kong_bindings.bzl @@ -0,0 +1,80 @@ +""" +Global varibles +""" + +def _load_vars(ctx): + # Read env from .requirements + requirements = ctx.read(Label("@kong//:.requirements")) + content = ctx.execute(["bash", "-c", "echo '%s' | " % requirements + + """grep -E '^(\\w*)=(.+)$' | sed -E 's/^(.*)=(.*)$/"\\1": "\\2",/'"""]).stdout + content = content.replace('""', '"') + + # Workspace path + workspace_path = "%s" % ctx.path(Label("@//:WORKSPACE")).dirname + content += '"WORKSPACE_PATH": "%s",\n' % workspace_path + + # Local env + # Temporarily fix for https://github.com/bazelbuild/bazel/issues/14693#issuecomment-1079006291 + for key in [ + "PATH", + # above should not be needed + "GITHUB_TOKEN", + "RPM_SIGNING_KEY_FILE", + "NFPM_RPM_PASSPHRASE", + ]: + value = ctx.os.environ.get(key, "") + if value: + content += '"%s": "%s",\n' % (key, value) + + build_name = ctx.os.environ.get("BUILD_NAME", "") + content += '"BUILD_NAME": "%s",\n' % build_name + + build_destdir = workspace_path + "/bazel-bin/build/" + build_name + content += '"BUILD_DESTDIR": "%s",\n' % build_destdir + + install_destdir = ctx.os.environ.get("INSTALL_DESTDIR", "MANAGED") + if install_destdir == "MANAGED": + install_destdir = build_destdir + content += '"INSTALL_DESTDIR": "%s",\n' % install_destdir + + # Kong Version + # TODO: this may not change after a bazel clean if cache exists + kong_version = ctx.execute(["bash", "scripts/grep-kong-version.sh"], working_directory = workspace_path).stdout + content += '"KONG_VERSION": "%s",' % kong_version.strip() + + nproc = ctx.execute(["nproc"]).stdout.strip() + content += '"%s": "%s",' % ("NPROC", nproc) + + macos_target = "" + if ctx.os.name == "mac os x": + macos_target = ctx.execute(["sw_vers", "-productVersion"]).stdout.strip() + content += '"MACOSX_DEPLOYMENT_TARGET": "%s",' % macos_target + + # convert them into a list of labels relative to the workspace root + # TODO: this may not change after a bazel clean if cache exists + patches = [ + '"@kong//:%s"' % str(p).replace(workspace_path, "").lstrip("/") + for p in ctx.path(workspace_path + "/build/openresty/patches").readdir() + ] + + content += '"OPENRESTY_PATCHES": [%s],' % (", ".join(patches)) + + ctx.file("BUILD.bazel", "") + ctx.file("variables.bzl", "KONG_VAR = {\n" + content + "\n}") + +def _load_bindings_impl(ctx): + _load_vars(ctx) + +load_bindings = repository_rule( + implementation = _load_bindings_impl, + # force "fetch"/invalidation of this repository every time it runs + # so that environ vars, patches and kong version is up to date + # see https://blog.bazel.build/2017/02/22/repository-invalidation.html + local = True, + environ = [ + "BUILD_NAME", + "INSTALL_DESTDIR", + "RPM_SIGNING_KEY_FILE", + "NFPM_RPM_PASSPHRASE", + ], +) diff --git a/build/libexpat/BUILD.bazel b/build/libexpat/BUILD.bazel new file mode 100644 index 000000000000..b4330b09f4c2 --- /dev/null +++ b/build/libexpat/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +exports_files( + [ + "BUILD.libexpat.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@libexpat//:libexpat", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/libexpat/BUILD.libexpat.bazel b/build/libexpat/BUILD.libexpat.bazel new file mode 100644 index 000000000000..4db19caed6fb --- /dev/null +++ b/build/libexpat/BUILD.libexpat.bazel @@ -0,0 +1,58 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +configure_make( + name = "libexpat", + configure_command = "configure", + configure_in_place = True, + configure_options = [ + # configure a miminal feature set at first so that we don't + # end up depend to a lot of dependencies; do not when turning + # on any of the feature below, we need to add it o kong package's + # dependencies, and compile it (under build/cross_deps) for + # cross build platforms + "--enable-static=no", + "--without-xmlwf", + "--without-examples", + "--without-docbook", + ] + select({ + "@kong//:aarch64-linux-anylibc-cross": [ + "--host=aarch64-linux", + ], + "@kong//:x86_64-linux-musl-cross": [ + "--host=x86_64-linux-musl", + ], + "//conditions:default": [], + }), + env = select({ + "@platforms//os:macos": { + # don't use rule_foreign_cc's libtool as archiver as it seems to be a bug + # see https://github.com/bazelbuild/rules_foreign_cc/issues/947 + "AR": "/usr/bin/ar", + }, + "//conditions:default": {}, + }), + lib_source = ":all_srcs", + # out_lib_dir = "lib", + out_shared_libs = select({ + "@platforms//os:macos": [ + "libexpat.1.dylib", + ], + "//conditions:default": [ + "libexpat.so.1", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + "install -j" + KONG_VAR["NPROC"], + ], + visibility = ["//visibility:public"], +) diff --git a/build/libexpat/repositories.bzl b/build/libexpat/repositories.bzl new file mode 100644 index 000000000000..3662761ca78b --- /dev/null +++ b/build/libexpat/repositories.bzl @@ -0,0 +1,20 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def libexpat_repositories(): + """Defines the libexpat repository""" + + version = KONG_VAR["LIBEXPAT"] + tag = "R_" + version.replace(".", "_") + + maybe( + http_archive, + name = "libexpat", + url = "https://github.com/libexpat/libexpat/releases/download/" + tag + "/expat-" + version + ".tar.gz", + sha256 = "6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033", + strip_prefix = "expat-" + version, + build_file = "//build/libexpat:BUILD.libexpat.bazel", + ) diff --git a/build/luarocks/BUILD.bazel b/build/luarocks/BUILD.bazel new file mode 100644 index 000000000000..8f332e3aa1ff --- /dev/null +++ b/build/luarocks/BUILD.bazel @@ -0,0 +1,18 @@ +load("//build:build_system.bzl", "kong_rules_group") + +exports_files( + [ + "BUILD.luarocks.bazel", + "luarocks_wrap_script.lua", + ], + visibility = ["//visibility:public"], +) + +kong_rules_group( + name = "luarocks", + propagates = [ + "@luarocks//:luarocks_make", + "@luarocks//:luarocks_target", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/luarocks/BUILD.luarocks.bazel b/build/luarocks/BUILD.luarocks.bazel new file mode 100644 index 000000000000..9a57517779ed --- /dev/null +++ b/build/luarocks/BUILD.luarocks.bazel @@ -0,0 +1,111 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong//build:build_system.bzl", "kong_template_genrule") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# This rules is used to bootstrap luarocks to install rocks dependencies +# A different rule is used to install luarocks in the release artifact +# so that we got correct interpreter path, lua paths, etc. +configure_make( + name = "luarocks_host", + configure_command = "configure", + configure_in_place = True, + configure_options = [ + "--lua-suffix=jit", + "--with-lua=$$EXT_BUILD_DEPS$$/luajit", + "--with-lua-include=$$EXT_BUILD_DEPS$$/luajit/include/luajit-2.1", + ], + lib_source = ":all_srcs", + out_bin_dir = "", + out_binaries = ["bin/luarocks"], # fake binary + out_data_dirs = ["luarocks"], # mark all files as data + targets = [ + "build", + "install", + ], + visibility = ["//visibility:public"], + deps = [ + "@openresty//:luajit", + ], +) + +kong_template_genrule( + name = "luarocks_exec", + srcs = [ + "@openssl//:openssl", + "@libexpat//:libexpat", + ] + select({ + "@kong//:any-cross": ["@cross_deps_libyaml//:libyaml"], + "//conditions:default": [ + "@luarocks//:luarocks_host", + "@openresty//:luajit", + ], + }), + is_executable = True, + output = "luarocks_exec.sh", + substitutions = { + "{{lib_rpath}}": "%s/kong/lib" % KONG_VAR["INSTALL_DESTDIR"], + }, + template = "@//build/luarocks:templates/luarocks_exec.sh", + tools = select({ + "@kong//:any-cross": [ + "@luarocks//:luarocks_host", + "@openresty//:luajit", + ], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) + +kong_template_genrule( + name = "luarocks_make", + srcs = [ + "@kong//:rockspec_srcs", + "@luarocks//:luarocks_exec", + "@luarocks//:luarocks_target", # to avoid concurrency issue, run this after luarocks_target + ], + is_executable = True, + output = "luarocks_make.log", + progress_message = "Luarocks: Install Kong rocks dependencies", + template = "@//build/luarocks:templates/luarocks_make.sh", + visibility = ["//visibility:public"], +) + +# install luarocks itself in target configuration +kong_template_genrule( + name = "luarocks_target", + srcs = [":luarocks_exec"] + select({ + "@kong//:any-cross": [], + "//conditions:default": [ + "@luarocks//:luarocks_host", + "@openresty//:luajit", + ], + }), + is_executable = True, + output = "luarocks_target.log", + progress_message = "Luarocks: Install luarocks on target system", + substitutions = { + "{{build_destdir}}": KONG_VAR["BUILD_DESTDIR"], + "{{install_destdir}}": KONG_VAR["INSTALL_DESTDIR"], + "{{luarocks_version}}": KONG_VAR["LUAROCKS"], + "{{workspace_path}}": KONG_VAR["WORKSPACE_PATH"], + }, + template = "@//build/luarocks:templates/luarocks_target.sh", + tools = [ + "@//build/luarocks:luarocks_wrap_script.lua", + ] + select({ + "@//:any-cross": [ + "@luarocks//:luarocks_host", + "@openresty//:luajit", + ], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) diff --git a/build/luarocks/luarocks_repositories.bzl b/build/luarocks/luarocks_repositories.bzl new file mode 100644 index 000000000000..de37ea9ee072 --- /dev/null +++ b/build/luarocks/luarocks_repositories.bzl @@ -0,0 +1,19 @@ +"""A module defining the third party dependency luarocks""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def luarocks_repositories(): + version = KONG_VAR["LUAROCKS"] + + maybe( + http_archive, + name = "luarocks", + build_file = "//build/luarocks:BUILD.luarocks.bazel", + strip_prefix = "luarocks-" + version, + sha256 = "bca6e4ecc02c203e070acdb5f586045d45c078896f6236eb46aa33ccd9b94edb", + urls = [ + "https://luarocks.org/releases/luarocks-" + version + ".tar.gz", + ], + ) diff --git a/build/luarocks/luarocks_wrap_script.lua b/build/luarocks/luarocks_wrap_script.lua new file mode 100644 index 000000000000..44e03cbaceb5 --- /dev/null +++ b/build/luarocks/luarocks_wrap_script.lua @@ -0,0 +1,33 @@ +local cfg = require("luarocks.core.cfg") +assert(cfg.init()) +-- print(require("inspect")(cfg)) + +local fs = require "luarocks.fs" +fs.init() + +local queries = require("luarocks.queries") +local search = require("luarocks.search") + +local name = arg[1] +local tree = arg[2] +local install_dest = arg[3] + +local query = queries.new(name, nil, nil, true) + +local _, ver = assert(search.pick_installed_rock(query)) + +if install_dest:sub(-1) ~= "/" then + install_dest = install_dest .. "/" +end +-- HACK +cfg.lua_interpreter = "luajit" +cfg.sysconfdir = install_dest .. "etc/luarocks" +cfg.variables["LUA_DIR"] = install_dest .. "openresty/luajit" +cfg.variables["LUA_INCDIR"] = install_dest .. "openresty/luajit/include/luajit-2.1" +cfg.variables["LUA_BINDIR"] = install_dest .. "openresty/luajit/bin" + +local wrap = fs.wrap_script + +wrap( + string.format("%s/lib/luarocks/rocks-5.1/luarocks/%s/bin/%s", tree, ver, name), + string.format("%s/bin/%s", tree, name), "one", name, ver) diff --git a/build/luarocks/templates/luarocks_exec.sh b/build/luarocks/templates/luarocks_exec.sh new file mode 100644 index 000000000000..145531a20fed --- /dev/null +++ b/build/luarocks/templates/luarocks_exec.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +# template variables starts +libexpat_path="{{@libexpat//:libexpat}}" +libxml2_path="{{@libxml2//:libxml2}}" +openssl_path="{{@openssl//:openssl}}" +luarocks_host_path="{{@luarocks//:luarocks_host}}" +luajit_path="{{@openresty//:luajit}}" +kongrocks_path="{{@kongrocks//:all_srcs}}" +cross_deps_libyaml_path="{{@cross_deps_libyaml//:libyaml}}" +CC={{CC}} +LD={{LD}} +LIB_RPATH={{lib_rpath}} +# template variables ends + +root_path=$(pwd) + +ROCKS_DIR=$root_path/$(dirname $@)/luarocks_tree +if [ ! -d $ROCKS_DIR ]; then + mkdir -p $ROCKS_DIR +fi +# pre create the dir and file so bsd readlink is happy +mkdir -p "$ROCKS_DIR/../cache" +CACHE_DIR=$(readlink -f "$ROCKS_DIR/../cache") +touch "$ROCKS_DIR/../luarocks_config.lua" +ROCKS_CONFIG=$(readlink -f "$ROCKS_DIR/../luarocks_config.lua") + +EXPAT_DIR=$root_path/$libexpat_path +LIBXML2_DIR=$root_path/$libxml2_path +OPENSSL_DIR=$root_path/$openssl_path + +# we use system libyaml on macos +if [[ "$OSTYPE" == "darwin"* ]]; then + YAML_DIR=$(HOME=~$(whoami) PATH=/opt/homebrew/bin:$PATH brew --prefix)/opt/libyaml +elif [[ -d $cross_deps_libyaml_path ]]; then + # TODO: is there a good way to use locations but doesn't break non-cross builds? + YAML_DIR=$root_path/$cross_deps_libyaml_path +else + YAML_DIR=/usr +fi + +if [[ $CC != /* ]]; then + # point to our relative path of managed toolchain + CC=$root_path/$CC + LD=$root_path/$LD +fi + +echo " +rocks_trees = { + { name = [[system]], root = [[$ROCKS_DIR]] } +} +local_cache = '$CACHE_DIR' +show_downloads = true +gcc_rpath = false -- disable default rpath, add our own +variables = { + CC = '$CC', + LD = '$LD', + LDFLAGS = '-Wl,-rpath,$LIB_RPATH', +} +" > $ROCKS_CONFIG + +LUAROCKS_HOST=$luarocks_host_path + +host_luajit=$root_path/$luajit_path/bin/luajit + +cat << EOF > $@ +LIB_RPATH=$LIB_RPATH +LUAROCKS_HOST=$LUAROCKS_HOST +ROCKS_DIR=$ROCKS_DIR +CACHE_DIR=$CACHE_DIR +ROCKS_CONFIG=$ROCKS_CONFIG + +export LUAROCKS_CONFIG=$ROCKS_CONFIG +export CC=$CC +export LD=$LD +export EXT_BUILD_ROOT=$root_path # for musl + +# no idea why PATH is not preserved in ctx.actions.run_shell +export PATH=$PATH + +if [[ $kongrocks_path == external* ]]; then + p=$root_path/external/kongrocks/rocks + echo "Using bundled rocks from \$p" + echo "If errors like 'No results matching query were found for Lua 5.1.' are shown, submit a PR to https://github.com/kong/kongrocks" + private_rocks_args="--only-server \$p" +fi + +# force the interpreter here instead of invoking luarocks directly, +# some distros has BINPRM_BUF_SIZE smaller than the shebang generated, +# which is usually more than 160 bytes +$host_luajit $root_path/$LUAROCKS_HOST/bin/luarocks \$private_rocks_args \$@ \\ + OPENSSL_DIR=$OPENSSL_DIR \\ + CRYPTO_DIR=$OPENSSL_DIR \\ + EXPAT_DIR=$EXPAT_DIR \\ + LIBXML2_DIR=$LIBXML2_DIR \\ + YAML_DIR=$YAML_DIR +EOF \ No newline at end of file diff --git a/build/luarocks/templates/luarocks_make.sh b/build/luarocks/templates/luarocks_make.sh new file mode 100644 index 000000000000..dc5d6105f3c2 --- /dev/null +++ b/build/luarocks/templates/luarocks_make.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e + +# template variables starts +luarocks_exec="{{@luarocks//:luarocks_exec}}" +# template variables ends + +if [[ "$OSTYPE" == "darwin"* ]]; then + export DEVELOPER_DIR=$(xcode-select -p) + export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) +fi +mkdir -p $(dirname $@) +# lyaml needs this and doesn't honor --no-doc +# the alternate will populate a non-existent HOME +# env var just to let ldoc happy +# alias LDOC command to true(1) command +export LDOC=true + +$luarocks_exec make --no-doc 2>&1 >$@.tmp + +# only generate the output when the command succeeds +mv $@.tmp $@ \ No newline at end of file diff --git a/build/luarocks/templates/luarocks_target.sh b/build/luarocks/templates/luarocks_target.sh new file mode 100644 index 000000000000..f84d52dcb4c3 --- /dev/null +++ b/build/luarocks/templates/luarocks_target.sh @@ -0,0 +1,59 @@ +#!/bin/bash -e + +# template variables starts +workspace_path="{{workspace_path}}" +luarocks_version="{{luarocks_version}}" +install_destdir="{{install_destdir}}" +build_destdir="{{build_destdir}}" + +luarocks_exec="{{@luarocks//:luarocks_exec}}" +luajit_path="{{@openresty//:luajit}}" +luarocks_host_path="{{@luarocks//:luarocks_host}}" +luarocks_wrap_script="{{@//build/luarocks:luarocks_wrap_script.lua}}" +# template variables ends + +mkdir -p $(dirname $@) + + +# install luarocks +$luarocks_exec install "luarocks $luarocks_version" + +# use host configuration to invoke luarocks API to wrap a correct bin/luarocks script +rocks_tree=$workspace_path/$(dirname $luarocks_exec)/luarocks_tree +host_luajit=$workspace_path/$luajit_path/bin/luajit + +host_luarocks_tree=$luarocks_host_path +export LUA_PATH="$build_destdir/share/lua/5.1/?.lua;$build_destdir/share/lua/5.1/?/init.lua;$host_luarocks_tree/share/lua/5.1/?.lua;$host_luarocks_tree/share/lua/5.1/?/init.lua;;" + +ROCKS_CONFIG="luarocks_make_config.lua" +cat << EOF > $ROCKS_CONFIG +rocks_trees = { + { name = [[system]], root = [[$rocks_tree]] } +} +EOF +export LUAROCKS_CONFIG=$ROCKS_CONFIG + +$host_luajit $luarocks_wrap_script \ + luarocks $rocks_tree $install_destdir 2>&1 > $@.tmp + +# write the luarocks config with host configuration +mkdir -p $rocks_tree/etc/luarocks +cat << EOF > $rocks_tree/etc/luarocks/config-5.1.lua +-- LuaRocks configuration +rocks_trees = { + { name = "user", root = home .. "/.luarocks" }; + { name = "system", root = "$install_destdir" }; + } + lua_interpreter = "luajit"; + variables = { + LUA_DIR = "$install_destdir/openresty/luajit"; + LUA_INCDIR = "$install_destdir/openresty/luajit/include/luajit-2.1"; + LUA_BINDIR = "$install_destdir/openresty/luajit/bin"; +} +EOF + +# TODO: this still doesn't work +sed -i -e "s|$rocks_tree|$install_destdir|g" $rocks_tree/bin/luarocks + +# only generate the output when the command succeeds +mv $@.tmp $@ \ No newline at end of file diff --git a/build/nfpm/BUILD.bazel b/build/nfpm/BUILD.bazel new file mode 100644 index 000000000000..d70ebc0efe10 --- /dev/null +++ b/build/nfpm/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "all_srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/build/nfpm/BUILD.nfpm.bazel b/build/nfpm/BUILD.nfpm.bazel new file mode 100644 index 000000000000..70ff958ed431 --- /dev/null +++ b/build/nfpm/BUILD.nfpm.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/build/nfpm/repositories.bzl b/build/nfpm/repositories.bzl new file mode 100644 index 000000000000..3f4f1a4e974d --- /dev/null +++ b/build/nfpm/repositories.bzl @@ -0,0 +1,55 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _nfpm_release_select_impl(ctx): + if ctx.attr.build_file: + ctx.file("BUILD.bazel", ctx.read(ctx.attr.build_file)) + elif ctx.attr.build_file_content: + ctx.file("BUILD.bazel", ctx.attr.build_file_content) + + os_name = ctx.os.name + os_arch = ctx.os.arch + + if os_arch == "aarch64": + os_arch = "arm64" + elif os_arch == "amd64": + os_arch = "x86_64" + else: + fail("Unsupported arch %s" % os_arch) + + if os_name == "mac os x": + os_name = "Darwin" + elif os_name != "linux": + fail("Unsupported OS %s" % os_name) + + nfpm_bin = "%s" % ctx.path(Label("@nfpm_%s_%s//:nfpm" % (os_name, os_arch))) + ctx.symlink(nfpm_bin, "nfpm") + +nfpm_release_select = repository_rule( + implementation = _nfpm_release_select_impl, + attrs = { + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + }, +) + +def nfpm_repositories(): + gh_matrix = [ + ["linux", "x86_64", "4c63031ddbef198e21c8561c438dde4c93c3457ffdc868d7d28fa670e0cc14e5"], + ["linux", "arm64", "2af1717cc9d5dcad5a7e42301dabc538acf5d12ce9ee39956c66f30215311069"], + ["Darwin", "x86_64", "fb3b8ab5595117f621c69cc51db71d481fbe733fa3c35500e1b64319dc8fd5b4"], + ["Darwin", "arm64", "9ca3ac6e0c4139a9de214f78040d1d11dd221496471696cc8ab5d357850ccc54"], + ] + for name, arch, sha in gh_matrix: + http_archive( + name = "nfpm_%s_%s" % (name, arch), + url = "https://github.com/goreleaser/nfpm/releases/download/v2.23.0/nfpm_2.23.0_%s_%s.tar.gz" % (name, arch), + sha256 = sha, + build_file = "//build/nfpm:BUILD.bazel", + ) + + nfpm_release_select( + name = "nfpm", + build_file = "//build/nfpm:BUILD.bazel", + ) diff --git a/build/nfpm/rules.bzl b/build/nfpm/rules.bzl new file mode 100644 index 000000000000..d6f5bb94f46a --- /dev/null +++ b/build/nfpm/rules.bzl @@ -0,0 +1,88 @@ +""" +NFPM package rule. +""" + +load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def _nfpm_pkg_impl(ctx): + env = dicts.add(ctx.attr.env, ctx.attr.extra_env, KONG_VAR, ctx.configuration.default_shell_env) + + target_cpu = ctx.attr._cc_toolchain[cc_common.CcToolchainInfo].cpu + if target_cpu == "k8" or target_cpu == "x86_64" or target_cpu == "amd64": + target_arch = "amd64" + elif target_cpu == "aarch64" or target_cpu == "arm64": + target_arch = "arm64" + else: + fail("Unsupported platform cpu: %s" % target_cpu) + env["ARCH"] = target_arch + + # XXX: remove the "env" from KONG_VAR which is a list + env["OPENRESTY_PATCHES"] = "" + + pkg_ext = ctx.attr.packager + if pkg_ext == "apk": + pkg_ext = "apk.tar.gz" + + # create like kong.amd64.deb + out = ctx.actions.declare_file("%s/%s.%s.%s" % ( + ctx.attr.out_dir, + ctx.attr.pkg_name, + target_arch, + pkg_ext, + )) + + nfpm_args = ctx.actions.args() + nfpm_args.add("pkg") + nfpm_args.add("-f", ctx.file.config.path) + nfpm_args.add("-p", ctx.attr.packager) + nfpm_args.add("-t", out.path) + + ctx.actions.run_shell( + inputs = ctx.files._nfpm_bin, + mnemonic = "nFPM", + command = "ln -sf %s nfpm-prefix; external/nfpm/nfpm $@" % KONG_VAR["BUILD_DESTDIR"], + arguments = [nfpm_args], + outputs = [out], + env = env, + ) + + # TODO: fix runfiles so that it can used as a dep + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = ctx.files.config))] + +nfpm_pkg = rule( + _nfpm_pkg_impl, + attrs = { + "config": attr.label( + mandatory = True, + allow_single_file = True, + doc = "nFPM configuration file.", + ), + "packager": attr.string( + mandatory = True, + doc = "Packager name.", + ), + "env": attr.string_dict( + doc = "Environment variables to set when running nFPM.", + ), + "extra_env": attr.string_dict( + # https://github.com/bazelbuild/bazel/issues/12457 + doc = "Additional environment variables to set when running nFPM. This is a workaround since Bazel doesn't support union operator for select yet.", + ), + "pkg_name": attr.string( + mandatory = True, + doc = "Output package name.", + ), + "out_dir": attr.string( + doc = "Output directory name.", + default = "pkg", + ), + # hidden attributes + "_nfpm_bin": attr.label( + default = "@nfpm//:all_srcs", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + }, +) diff --git a/build/openresty/BUILD.bazel b/build/openresty/BUILD.bazel new file mode 100644 index 000000000000..c527359e1f08 --- /dev/null +++ b/build/openresty/BUILD.bazel @@ -0,0 +1,6 @@ +exports_files( + [ + "BUILD.openresty.bazel", + ], + visibility = ["//visibility:public"], +) diff --git a/build/openresty/BUILD.kong-build-tools.bazel b/build/openresty/BUILD.kong-build-tools.bazel new file mode 100644 index 000000000000..70ff958ed431 --- /dev/null +++ b/build/openresty/BUILD.kong-build-tools.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/build/openresty/BUILD.openresty.bazel b/build/openresty/BUILD.openresty.bazel new file mode 100644 index 000000000000..9dbbbf7f31d0 --- /dev/null +++ b/build/openresty/BUILD.openresty.bazel @@ -0,0 +1,231 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make", "make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") +load("@openresty_binding//:variables.bzl", "LUAJIT_VERSION") + +filegroup( + name = "luajit_srcs", + srcs = glob( + include = ["bundle/LuaJIT*/**"], + ), +) + +genrule( + name = "luajit_xcflags", + outs = ["luajit_xcflags.txt"], + cmd = "macos=" + select({ + "@platforms//os:macos": "1", + "//conditions:default": "0", + }) + "\n" + + "aarch64=" + select({ + "@platforms//cpu:arm64": "1", + "//conditions:default": "0", + }) + "\n" + + "debug=" + select({ + "@kong//:debug_flag": "1", + "//conditions:default": "0", + }) + "\n" + + "cross=" + select({ + "@kong//:any-cross": "1", + "//conditions:default": "0", + }) + + """ + flags="-DLUAJIT_ENABLE_LUA52COMPAT -DLUAJIT_VERSION=\\\\\\"{luajit_version}\\\\\\"" + if [[ $$debug -eq 1 ]]; then + flags="$$flags -DLUA_USE_ASSERT -DLUA_USE_APICHECK" + if [[ $$macos -ne 1 ]]; then + if [[ $$cross -ne 1 ]]; then + flags="$$flags -DLUA_USE_VALGRIND" + fi + if [[ $$aarch64 -ne 1 ]]; then + flags="$$flags -DLUAJIT_USE_SYSMALLOC" + fi + fi + fi + + if [[ $$macos -eq 1 ]]; then + flags="$$flags -fno-stack-check" + fi + + echo "$$flags" >$@ + + """.format(luajit_version = LUAJIT_VERSION), +) + +make( + name = "luajit", + args = [ + "LDFLAGS=\"-Wl,-rpath,%s/kong/lib\"" % KONG_VAR["INSTALL_DESTDIR"], # make ffi.load happy, even when it's invoked without nginx + "XCFLAGS=\"$(cat $$EXT_BUILD_ROOT$$/$(execpath :luajit_xcflags))\"", + "LUA_ROOT=%s/openresty/luajit" % KONG_VAR["INSTALL_DESTDIR"].rstrip("/"), + "MACOSX_DEPLOYMENT_TARGET=" + KONG_VAR["MACOSX_DEPLOYMENT_TARGET"], + ] + select({ + "@kong//:any-cross": [ + "HOST_CC=cc", + ], + "//conditions:default": [ + ], + }), + build_data = [ + ":luajit_xcflags", + ], + lib_source = ":luajit_srcs", + out_binaries = [ + "luajit", + ], + out_shared_libs = select({ + "@platforms//os:macos": [ + "libluajit-5.1.2.dylib", + ], + "//conditions:default": [ + "libluajit-5.1.so.2", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + "install", + ], + visibility = ["//visibility:public"], +) + +CONFIGURE_OPTIONS = [ + "--with-pcre-jit", + "--with-http_ssl_module", + "--with-http_sub_module", + "--with-http_realip_module", + "--with-http_stub_status_module", + "--with-http_v2_module", + "--with-stream_realip_module", # >= 1.11.4 + "--with-stream_ssl_preread_module", # >= 1.11.5 + "--without-http_encrypted_session_module", + "--with-luajit=$$EXT_BUILD_DEPS$$/luajit", + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/pcre/include\"", + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/openssl/include\"", + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/luajit/include\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/pcre/lib\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/openssl/lib\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/luajit/lib\"", + # here let's try not having --disable-new-dtags; --disable-new-dtags creates runpath instead of rpath + # note runpath can't handle indirect dependency (nginx -> luajit -> dlopen("other")), so each indirect + # dependency should have its rpath set (luajit, libxslt etc); on the other side, rpath is not + # overridable by LD_LIBRARY_PATH and it may cause trouble debugging, so we _should_ prefer runpath. + # if it doesn't work, then add --disable-new-dtags back + "--with-ld-opt=\"-Wl,-rpath,%s/kong/lib\"" % KONG_VAR["INSTALL_DESTDIR"], + "-j%s" % KONG_VAR["NPROC"], + + # options from our customed patch + "--with-install-prefix=%s" % KONG_VAR["INSTALL_DESTDIR"], + + # Note $$EXT_BUILD_ROOT$$ is bazel variable not from environment variable + # which points to the directory of current WORKSPACE + + # external modules + "--add-module=$$EXT_BUILD_ROOT$$/external/lua-kong-nginx-module", + "--add-module=$$EXT_BUILD_ROOT$$/external/lua-kong-nginx-module/stream", + "--add-module=$$EXT_BUILD_ROOT$$/external/lua-resty-lmdb", + "--add-module=$$EXT_BUILD_ROOT$$/external/lua-resty-events", +] + select({ + "@kong//:arm64-linux-gnu-cross": [ + "--crossbuild=Linux:aarch64", + "--with-endian=le", + "--with-int=4", + "--with-long=8", + "--with-long-long=8", + "--with-ptr-size=8", + "--with-sig-atomic-t=4", + "--with-size-t=8", + "--with-off-t=8", + "--with-time-t=8", + "--with-sys-nerr=132", + ], + "@kong//:x86_64-linux-musl-cross": [ + "--crossbuild=Linux:x86_64", + "--with-endian=le", + "--with-int=4", + "--with-long=8", + "--with-long-long=8", + "--with-ptr-size=8", + "--with-sig-atomic-t=4", + "--with-size-t=8", + "--with-off-t=8", + "--with-time-t=8", + "--with-sys-nerr=132", + ], + "//conditions:default": [], +}) + select({ + "@kong//:any-cross": [ + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/zlib/include\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/zlib/lib\"", + ], + "//conditions:default": [], +}) + select({ + # any cross build that migrated to use libxcrypt needs those flags + # alpine uses different libc so doesn't need it + "@kong//:arm64-linux-gnu-cross": [ + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/libxcrypt/include\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/libxcrypt/lib\"", + ], + "//conditions:default": [], +}) + select({ + "@kong//:debug_flag": [ + "--with-debug", + "--with-no-pool-patch", + "--with-cc-opt=\"-DNGX_LUA_USE_ASSERT -DNGX_LUA_ABORT_AT_PANIC\"", + ], + "//conditions:default": [], +}) + select({ + "@kong//:fips_flag": [ + "--with-cc-opt=\"-I$$EXT_BUILD_DEPS$$/include\"", + "--with-ld-opt=\"-L$$EXT_BUILD_DEPS$$/lib -Wl,-Bsymbolic-functions -Wl,-z,relro\"", + ], + "//conditions:default": [], +}) + +# TODO: set prefix to populate pid_path, conf_path, log_path etc + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +configure_make( + name = "openresty", + build_data = [ + "@lua-kong-nginx-module//:all_srcs", + "@lua-resty-lmdb//:all_srcs", + "@lua-resty-events//:all_srcs", + "@openresty_binding//:all_srcs", + ], + configure_command = "configure", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS, + lib_source = ":all_srcs", + out_bin_dir = "", + out_binaries = [ + "nginx/sbin/nginx", + ], + targets = [ + "-j" + KONG_VAR["NPROC"], + "install -j" + KONG_VAR["NPROC"], + ], + visibility = ["//visibility:public"], + deps = [ + "@pcre", + "@openresty//:luajit", + "@openssl//:openssl", + ] + select({ + "@kong//:any-cross": [ + "@cross_deps_zlib//:zlib", + ], + "//conditions:default": [], + }) + select({ + # any cross build that migrated to use libxcrypt needs those flags + # alpine uses different libc so doesn't need it + "@kong//:arm64-linux-gnu-cross": [ + "@cross_deps_libxcrypt//:libxcrypt", + ], + "//conditions:default": [], + }), +) diff --git a/build/openresty/atc_router/BUILD.bazel b/build/openresty/atc_router/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/openresty/atc_router/atc_router_repositories.bzl b/build/openresty/atc_router/atc_router_repositories.bzl new file mode 100644 index 000000000000..2daf5879f835 --- /dev/null +++ b/build/openresty/atc_router/atc_router_repositories.bzl @@ -0,0 +1,14 @@ +"""A module defining the dependency atc-router""" + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def atc_router_repositories(): + maybe( + git_repository, + name = "atc_router", + branch = KONG_VAR["ATC_ROUTER"], + remote = "https://github.com/Kong/atc-router", + visibility = ["//visibility:public"], # let this to be referenced by openresty build + ) diff --git a/build/openresty/lua-resty-lmdb-cross.patch b/build/openresty/lua-resty-lmdb-cross.patch new file mode 100644 index 000000000000..d1bf0820f57f --- /dev/null +++ b/build/openresty/lua-resty-lmdb-cross.patch @@ -0,0 +1,51 @@ +lua-resty-lmdb is an external repository, previous artifact may carry +thus we always clean here + +diff --git a/config b/config +index 126c78c..1f0b2aa 100644 +--- a/config ++++ b/config +@@ -5,6 +5,8 @@ ngx_module_incs="$ngx_addon_dir/lmdb/libraries/liblmdb $ngx_addon_dir/src" + + . auto/module + ++rm -f $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a ++ + LINK_DEPS="$LINK_DEPS $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a" + CORE_LIBS="$CORE_LIBS $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a" + +diff --git a/config.make b/config.make +index 14d8cc2..cf17251 100644 +--- a/config.make ++++ b/config.make +@@ -3,7 +3,7 @@ cat <>$NGX_MAKEFILE + + $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a: + echo "Building liblmdb"; \\ +- \$(MAKE) -C $ngx_addon_dir/lmdb/libraries/liblmdb; \\ ++ \$(MAKE) -C $ngx_addon_dir/lmdb/libraries/liblmdb CC=\$(CC) AR=\$(AR); \\ + echo "Finished building liblmdb" + + EOF +diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile +index c252b50..1054432 100644 +--- a/lmdb/libraries/liblmdb/Makefile ++++ b/lmdb/libraries/liblmdb/Makefile +@@ -18,13 +18,13 @@ + # There may be other macros in mdb.c of interest. You should + # read mdb.c before changing any of them. + # +-CC = gcc +-AR = ar ++CC ?= gcc ++AR ?= ar + W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized + THREADS = -pthread + OPT = -O2 -g +-CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) +-LDFLAGS = $(THREADS) ++CFLAGS += $(THREADS) $(OPT) $(W) $(XCFLAGS) ++LDFLAGS += $(THREADS) + LDLIBS = + SOLIBS = + SOEXT = .so diff --git a/build/openresty/msgpack_c/BUILD.bazel b/build/openresty/msgpack_c/BUILD.bazel new file mode 100644 index 000000000000..b1082eee35a2 --- /dev/null +++ b/build/openresty/msgpack_c/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +exports_files( + [ + "BUILD.msgpack_c.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@msgpack_c//:msgpack_c", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/openresty/msgpack_c/BUILD.msgpack_c.bazel b/build/openresty/msgpack_c/BUILD.msgpack_c.bazel new file mode 100644 index 000000000000..b496d1569bb4 --- /dev/null +++ b/build/openresty/msgpack_c/BUILD.msgpack_c.bazel @@ -0,0 +1,27 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# msgpack_c cmake detects cross compile automatically +cmake( + name = "msgpack_c", + build_args = [ + "--", # <- Pass remaining options to the native tool. + "-j" + KONG_VAR["NPROC"], + ], + cache_entries = { + "BUILD_SHARED_LIBS": "OFF", + }, + lib_source = ":all_srcs", + out_static_libs = [ + "libmsgpackc.a", + ], + visibility = ["//visibility:public"], +) diff --git a/build/openresty/msgpack_c/README.md b/build/openresty/msgpack_c/README.md new file mode 100644 index 000000000000..667545c0bd32 --- /dev/null +++ b/build/openresty/msgpack_c/README.md @@ -0,0 +1,5 @@ +This target is modified from https://github.com/bazelbuild/rules_foreign_cc/tree/main/examples/third_party +with following chnages: + +- Read version from requirements.txt +- Updated `build_file` to new path under //build/openresty \ No newline at end of file diff --git a/build/openresty/msgpack_c/msgpack_c_repositories.bzl b/build/openresty/msgpack_c/msgpack_c_repositories.bzl new file mode 100644 index 000000000000..75521c8b9b0c --- /dev/null +++ b/build/openresty/msgpack_c/msgpack_c_repositories.bzl @@ -0,0 +1,17 @@ +"""A module defining the third party dependency msgpack-c""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def msgpack_c_repositories(): + version = KONG_VAR["MSGPACK_C"] + + maybe( + http_archive, + name = "msgpack_c", + build_file = "//build/openresty/msgpack_c:BUILD.msgpack_c.bazel", + strip_prefix = "msgpack-" + version, + sha256 = "420fe35e7572f2a168d17e660ef981a589c9cbe77faa25eb34a520e1fcc032c8", + url = "https://github.com/msgpack/msgpack-c/releases/download/" + version + "/msgpack-" + version + ".tar.gz", + ) diff --git a/build/openresty/openssl/BUILD.bazel b/build/openresty/openssl/BUILD.bazel new file mode 100644 index 000000000000..85e0e35912a8 --- /dev/null +++ b/build/openresty/openssl/BUILD.bazel @@ -0,0 +1,18 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +# load("@rules_cc//cc:defs.bzl", "cc_test") + +exports_files( + [ + "BUILD.openssl.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@openssl//:openssl", + # "@openssl//:runnable_openssl", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/openresty/openssl/BUILD.openssl.bazel b/build/openresty/openssl/BUILD.openssl.bazel new file mode 100644 index 000000000000..5fd4a4585680 --- /dev/null +++ b/build/openresty/openssl/BUILD.openssl.bazel @@ -0,0 +1,78 @@ +"""An openssl build file based on a snippet found in the github issue: +https://github.com/bazelbuild/rules_foreign_cc/issues/337 + +Note that the $(PERL) "make variable" (https://docs.bazel.build/versions/main/be/make-variables.html) +is populated by the perl toolchain provided by rules_perl. +""" + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +# Read https://wiki.openssl.org/index.php/Compilation_and_Installation + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +CONFIGURE_OPTIONS = [ + "-g", + "shared", + "-DPURIFY", + "no-threads", + "no-unit-test", + "--openssldir=%s/kong" % KONG_VAR["INSTALL_DESTDIR"], + "-Wl,-rpath,%s/kong/lib" % KONG_VAR["INSTALL_DESTDIR"], +] + select({ + "@kong//:debug_flag": ["-d"], + "//conditions:default": [], +}) + +configure_make( + name = "openssl", + configure_command = "config", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS, + env = select({ + "@platforms//os:macos": { + "AR": "", + }, + "@kong//:arm64-linux-gnu-cross": { + "MACHINE": "aarch64", + "SYSTEM": "linux2", + }, + # no extra args needed for "@kong//:x86_64-linux-musl-cross" + "//conditions:default": {}, + }), + lib_source = ":all_srcs", + out_binaries = ["openssl"], + # Note that for Linux builds, libssl must come before libcrypto on the linker command-line. + # As such, libssl must be listed before libcrypto + out_shared_libs = select({ + "@platforms//os:macos": [ + "libssl.1.1.dylib", + "libcrypto.1.1.dylib", + ], + "//conditions:default": [ + "libssl.so.1.1", + "libcrypto.so.1.1", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + "install_sw", + ], + # TODO: uncomment this to allow bazel build a perl if not installed on system + # toolchains = ["@rules_perl//:current_toolchain"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "gen_dir", + srcs = [":openssl"], + output_group = "gen_dir", + visibility = ["//visibility:public"], +) diff --git a/build/openresty/openssl/README.md b/build/openresty/openssl/README.md new file mode 100644 index 000000000000..82a1c5fa9c2c --- /dev/null +++ b/build/openresty/openssl/README.md @@ -0,0 +1,9 @@ +This target is modified from https://github.com/bazelbuild/rules_foreign_cc/tree/main/examples/third_party +with following changes: + +- Read version from requirements.txt +- Updated `build_file` to new path under //build/openresty +- Remove Windows build support +- Removed the bazel mirror as it's missing latest versions +- Remove runnable test for now until cross compile has been sorted out +- Use system Perl for now \ No newline at end of file diff --git a/build/openresty/openssl/openssl.bzl b/build/openresty/openssl/openssl.bzl new file mode 100644 index 000000000000..efd50ef7c70c --- /dev/null +++ b/build/openresty/openssl/openssl.bzl @@ -0,0 +1,91 @@ +"""An openssl build file based on a snippet found in the github issue: +https://github.com/bazelbuild/rules_foreign_cc/issues/337 + +Note that the $(PERL) "make variable" (https://docs.bazel.build/versions/main/be/make-variables.html) +is populated by the perl toolchain provided by rules_perl. +""" + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +# Read https://wiki.openssl.org/index.php/Compilation_and_Installation + +CONFIGURE_OPTIONS = select({ + "@kong//:aarch64-linux-anylibc-cross": [ + "linux-aarch64", + ], + # no extra args needed for "@kong//:x86_64-linux-musl-cross" or non-cross builds + "//conditions:default": [], +}) + [ + "-g", + "-O3", # force -O3 even we are using --debug (for example on CI) + "shared", + "-DPURIFY", + "no-threads", + "no-tests", + "--prefix=%s/kong" % KONG_VAR["INSTALL_DESTDIR"], + "--openssldir=%s/kong" % KONG_VAR["INSTALL_DESTDIR"], + "--libdir=lib", # force lib instead of lib64 (multilib postfix) + "-Wl,-rpath,%s/kong/lib" % KONG_VAR["INSTALL_DESTDIR"], +] + select({ + "@kong//:debug_flag": ["--debug"], + "//conditions:default": [], +}) + +def build_openssl( + name = "openssl", + fips = False): + if fips: + extra_make_targets = [ + "install_fips INSTALLTOP=$BUILD_TMPDIR/$INSTALL_PREFIX/fipsmodule OPENSSLDIR=$BUILD_TMPDIR/$INSTALL_PREFIX/fipsmodule", + ] + extra_configure_options = [ + "enable-fips", + ] + else: + extra_make_targets = [] + extra_configure_options = [] + + native.filegroup( + name = name + "-all_srcs", + srcs = native.glob( + include = ["**"], + exclude = ["*.bazel"], + ), + ) + + configure_make( + name = name, + configure_command = "config", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS + extra_configure_options, + env = select({ + "@platforms//os:macos": { + "AR": "/usr/bin/ar", + }, + "//conditions:default": {}, + }), + lib_source = ":%s-all_srcs" % name, + out_binaries = ["openssl"], + # Note that for Linux builds, libssl must come before libcrypto on the linker command-line. + # As such, libssl must be listed before libcrypto + out_shared_libs = select({ + "@platforms//os:macos": [ + "libssl.3.dylib", + "libcrypto.3.dylib", + ], + "//conditions:default": [ + "libssl.so.3", + "libcrypto.so.3", + ], + }), + targets = [ + "-j" + KONG_VAR["NPROC"], + # don't set the prefix by --prefix switch, but only override the install destdir using INSTALLTOP + # while install. this makes both bazel and openssl (build time generated) paths happy. + "install_sw INSTALLTOP=$BUILD_TMPDIR/$INSTALL_PREFIX", + ] + extra_make_targets, + # TODO: uncomment this to allow bazel build a perl if not installed on system + # toolchains = ["@rules_perl//:current_toolchain"], + visibility = ["//visibility:public"], + ) diff --git a/build/openresty/openssl/openssl_repositories.bzl b/build/openresty/openssl/openssl_repositories.bzl new file mode 100644 index 000000000000..cab43702d1dd --- /dev/null +++ b/build/openresty/openssl/openssl_repositories.bzl @@ -0,0 +1,20 @@ +"""A module defining the third party dependency OpenSSL""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def openssl_repositories(): + version = KONG_VAR["OPENSSL"] + + maybe( + http_archive, + name = "openssl", + build_file = "//build/openresty/openssl:BUILD.bazel", + sha256 = "840af5366ab9b522bde525826be3ef0fb0af81c6a9ebd84caa600fea1731eee3", + strip_prefix = "openssl-" + version, + urls = [ + "https://www.openssl.org/source/openssl-" + version + ".tar.gz", + "https://github.com/openssl/openssl/releases/download/openssl-" + version + "/openssl-" + version + ".tar.gz", + ], + ) diff --git a/build/openresty/openssl/openssl_test.cc b/build/openresty/openssl/openssl_test.cc new file mode 100644 index 000000000000..611f7b132e8c --- /dev/null +++ b/build/openresty/openssl/openssl_test.cc @@ -0,0 +1,53 @@ +#include + +#include +#include +#include +#include + +// Use (void) to silent unused warnings. +#define assertm(exp, msg) assert(((void)msg, exp)) + +// From https://stackoverflow.com/a/2262447/7768383 +bool simpleSHA256(const void* input, unsigned long length, unsigned char* md) +{ + SHA256_CTX context; + if (!SHA256_Init(&context)) + return false; + + if (!SHA256_Update(&context, (unsigned char*)input, length)) + return false; + + if (!SHA256_Final(md, &context)) + return false; + + return true; +} + +// Convert an byte array into a string +std::string fromByteArray(const unsigned char* data, unsigned long length) +{ + std::stringstream shastr; + shastr << std::hex << std::setfill('0'); + for (unsigned long i = 0; i < length; ++i) + { + shastr << std::setw(2) << static_cast(data[i]); + } + + return shastr.str(); +} + +std::string MESSAGE = "hello world"; +std::string MESSAGE_HASH = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; + +int main(int argc, char* argv[]) +{ + unsigned char md[SHA256_DIGEST_LENGTH] = {}; + + assertm(simpleSHA256(static_cast(MESSAGE.data()), MESSAGE.size(), md), "Failed to generate hash"); + std::string hash = fromByteArray(md, SHA256_DIGEST_LENGTH); + + assertm(hash == MESSAGE_HASH, "Unexpected message hash"); + + return 0; +} diff --git a/build/openresty/openssl/openssl_test.sh b/build/openresty/openssl/openssl_test.sh new file mode 100755 index 000000000000..ae3c91b4e69a --- /dev/null +++ b/build/openresty/openssl/openssl_test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [[ ! -e "$OPENSSL" ]]; then + echo "openssl does not exist" + exit 1 +fi + +exec $OPENSSL help diff --git a/build/openresty/patches/LuaJIT-2.1-20210510_01-ffi-arm64-macos-fix-vararg-call-handling.patch b/build/openresty/patches/LuaJIT-2.1-20210510_01-ffi-arm64-macos-fix-vararg-call-handling.patch new file mode 100644 index 000000000000..9047d7c86d1a --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20210510_01-ffi-arm64-macos-fix-vararg-call-handling.patch @@ -0,0 +1,62 @@ +From 521b367567dc5d91d7f9ae29c257998953e24e53 Mon Sep 17 00:00:00 2001 +From: Mike Pall +Date: Sun, 2 May 2021 22:11:05 +0200 +Subject: [PATCH] FFI/ARM64/OSX: Fix vararg call handling. + +Thanks to Igor Munkin. +--- + LuaJIT-2.1-20210510/src/lj_ccall.c | 8 ++++---- + LuaJIT-2.1-20210510/src/lj_ccallback.c | 2 +- + 2 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/LuaJIT-2.1-20210510/src/lj_ccall.c b/LuaJIT-2.1-20210510/src/lj_ccall.c +index a91ffc7e..3c029823 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_ccall.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_ccall.c +@@ -334,7 +334,7 @@ + isfp = sz == 2*sizeof(float) ? 2 : 1; + + #define CCALL_HANDLE_REGARG \ +- if (LJ_TARGET_IOS && isva) { \ ++ if (LJ_TARGET_OSX && isva) { \ + /* IOS: All variadic arguments are on the stack. */ \ + } else if (isfp) { /* Try to pass argument in FPRs. */ \ + int n2 = ctype_isvector(d->info) ? 1 : \ +@@ -345,10 +345,10 @@ + goto done; \ + } else { \ + nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \ +- if (LJ_TARGET_IOS && d->size < 8) goto err_nyi; \ ++ if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \ + } \ + } else { /* Try to pass argument in GPRs. */ \ +- if (!LJ_TARGET_IOS && (d->info & CTF_ALIGN) > CTALIGN_PTR) \ ++ if (!LJ_TARGET_OSX && (d->info & CTF_ALIGN) > CTALIGN_PTR) \ + ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ + if (ngpr + n <= maxgpr) { \ + dp = &cc->gpr[ngpr]; \ +@@ -356,7 +356,7 @@ + goto done; \ + } else { \ + ngpr = maxgpr; /* Prevent reordering. */ \ +- if (LJ_TARGET_IOS && d->size < 8) goto err_nyi; \ ++ if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \ + } \ + } + +diff --git a/LuaJIT-2.1-20210510/src/lj_ccallback.c b/LuaJIT-2.1-20210510/src/lj_ccallback.c +index 8d6cb737..80d738c6 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_ccallback.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_ccallback.c +@@ -460,7 +460,7 @@ void lj_ccallback_mcode_free(CTState *cts) + nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \ + } \ + } else { \ +- if (!LJ_TARGET_IOS && n > 1) \ ++ if (!LJ_TARGET_OSX && n > 1) \ + ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ + if (ngpr + n <= maxgpr) { \ + sp = &cts->cb.gpr[ngpr]; \ +-- +2.34.1 + diff --git a/build/openresty/patches/LuaJIT-2.1-20210510_02-arm64-fix-pcall-error-case.patch b/build/openresty/patches/LuaJIT-2.1-20210510_02-arm64-fix-pcall-error-case.patch new file mode 100644 index 000000000000..37934cee30d6 --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20210510_02-arm64-fix-pcall-error-case.patch @@ -0,0 +1,29 @@ +From b4b2dce9fc3ffaaaede39b36d06415311e2aa516 Mon Sep 17 00:00:00 2001 +From: Mike Pall +Date: Wed, 27 Oct 2021 21:56:07 +0200 +Subject: [PATCH] ARM64: Fix pcall() error case. + +Reported by Alex Orlenko. +--- + src/vm_arm64.dasc | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/LuaJIT-2.1-20210510/src/vm_arm64.dasc b/LuaJIT-2.1-20210510/src/vm_arm64.dasc +index c7090ca3..eb87857f 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/vm_arm64.dasc ++++ b/bundle/LuaJIT-2.1-20210510/src/vm_arm64.dasc +@@ -1163,9 +1163,10 @@ static void build_subroutines(BuildCtx *ctx) + |//-- Base library: catch errors ---------------------------------------- + | + |.ffunc pcall ++ | cmp NARGS8:RC, #8 + | ldrb TMP0w, GL->hookmask +- | subs NARGS8:RC, NARGS8:RC, #8 + | blo ->fff_fallback ++ | sub NARGS8:RC, NARGS8:RC, #8 + | mov RB, BASE + | add BASE, BASE, #16 + | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1 +-- +2.34.1 + diff --git a/build/openresty/patches/LuaJIT-2.1-20210510_03_patch_macro_luajit_version.patch b/build/openresty/patches/LuaJIT-2.1-20210510_03_patch_macro_luajit_version.patch new file mode 100644 index 000000000000..5e7869bcf3ea --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20210510_03_patch_macro_luajit_version.patch @@ -0,0 +1,14 @@ +diff --git a/bundle/LuaJIT-2.1-20210510/src/luajit.h b/bundle/LuaJIT-2.1-20210510/src/luajit.h +index 1dddaec..651de67 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/luajit.h ++++ b/bundle/LuaJIT-2.1-20210510/src/luajit.h +@@ -32,7 +32,9 @@ + + #define OPENRESTY_LUAJIT + ++#ifndef LUAJIT_VERSION + #define LUAJIT_VERSION "LuaJIT 2.1.0-beta3" ++#endif + #define LUAJIT_VERSION_NUM 20100 /* Version 2.1.0 = 02.01.00. */ + #define LUAJIT_VERSION_SYM luaJIT_version_2_1_0_beta3 + #define LUAJIT_COPYRIGHT "Copyright (C) 2005-2021 Mike Pall" diff --git a/build/openresty/patches/LuaJIT-2.1-20210510_04_kprof.patch b/build/openresty/patches/LuaJIT-2.1-20210510_04_kprof.patch new file mode 100644 index 000000000000..fd6a054c131e --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20210510_04_kprof.patch @@ -0,0 +1,3117 @@ +From 5fcb6215dc6c4832f0e70eca58be13845d8bc567 Mon Sep 17 00:00:00 2001 +From: Qi +Date: Sun, 27 Nov 2022 08:08:30 +0000 +Subject: [PATCH] feat: new buit-in function `gcsnapshot()` + +syntax: ok, err = gcsnapshot(filename[, timeout=120]) +context: any +subsystem: any + +This function will dump all LuaJIT GC objects to a file specified by the `filename`. +During the dumping, +the worker will stop processing requests until this function is done. + +chore(ci): disable Travis + +feat(profiling): support memory tracing on the fly (#1) + +* feat: new built-in module `kprof` + +The `kprof` is the Kong Profiling extension library + +--- + +The old built-in function `gcsnapshot` was moved to `kprof.mem.gcsnapshot`. + +--- + +syntax: ok, err = kprof.mem.start(path, block_size, max_stack_depth[, timeout=120]) +context: any +subsystem: any + +Start the memory tracing for all Lua VM instances. + +This function will start the memory tracing to collect the following information: + +* memory allocation +* memory free +* memory reallocation +* stack traceback + +The memory tracing will be stopped automatically after the timeout (in seconds), +but you also have to call kprof.mem.stop() to stop the memory tracing manually, +if you don't do it, some tracing data will be corrupted. + +--- + +syntax: ok, err = kprof.mem.stop() +context: any +subsystem: any + +Stop the memory tracing, and flush all tracing data to the file system. + +--- + +syntax: tbl = kprof.mem.status() +context: any +subsystem: any + +Return a table like: +{ + enabled: bool, + filename: string, + block_size: number, + max_stack_depth: number, + timeout_at: number, + error: string or nil, +} + +--------- + +Co-authored-by: Chrono + +fix(tracing): fix compiler error on legacy compilers on rhel7 (#3) + +lj_mem_tracing.c: In function '_write_trace_event': +lj_mem_tracing.c:283:3: error: 'for' loop initial declarations are only allowed in C99 mode + for (int i = 0; i < g_max_stack_depth; i++) { + ^ +lj_mem_tracing.c:283:3: note: use option -std=c99 or -std=gnu99 to compile your code +lj_mem_tracing.c: In function 'lj_mem_tracing_stop': +lj_mem_tracing.c:528:3: error: 'for' loop initial declarations are only allowed in C99 mode + for (int i = 0; i < sizeof(g_mbufs) / sizeof(g_mbufs[0]); i++) { + +tests: disable tests that are not for luajit2 v2.1-20210510 + +docs(README): add Kong's changes into the `README.md` (#5) + +Co-authored-by: Chrono + +fix(profiling): fix tracer doesn't cover the alloc/free of function object + +The memory tracer should cover the alloc/free of the Lua function object, which means that we should insert probes where the function object alloc/free. + +feat(tracing): covers almost all memory operations + +Co-authored-by: Chrono + +docs(README): add a guide on how to generate patch + +fix(profiling): fix assertion that doesn't cover the case that free a NULL pointer + +style(profiling): remove unused `#include "lj_mem_tracing.h"` +--- + src/Makefile | 12 +- + src/Makefile.dep | 22 +- + src/lib_base.c | 2 + + src/lib_init.c | 5 + + src/lib_kprof.c | 116 ++++++ + src/lj_bcread.c | 5 + + src/lj_cdata.c | 1 + + src/lj_cdata.h | 1 + + src/lj_debug.c | 5 + + src/lj_debug.h | 1 + + src/lj_dispatch.c | 23 +- + src/lj_gc.c | 7 + + src/lj_gc.h | 2 + + src/lj_gcsnapshot.c | 947 +++++++++++++++++++++++++++++++++++++++++++ + src/lj_gcsnapshot.h | 11 + + src/lj_ir.c | 15 +- + src/lj_mbuf.h | 170 ++++++++ + src/lj_mcode.c | 5 + + src/lj_mem_tracing.c | 888 ++++++++++++++++++++++++++++++++++++++++ + src/lj_mem_tracing.h | 88 ++++ + src/lj_parse.c | 4 + + src/lj_profile.c | 18 +- + src/lj_snap.c | 10 + + src/lj_str.c | 1 + + src/lj_tab.c | 24 +- + src/lj_trace.c | 22 +- + src/lj_udata.c | 1 + + src/ljamalg.c | 1 + + src/lualib.h | 7 + + 29 files changed, 2382 insertions(+), 32 deletions(-) + create mode 100644 src/lib_kprof.c + create mode 100644 src/lj_gcsnapshot.c + create mode 100644 src/lj_gcsnapshot.h + create mode 100644 src/lj_mbuf.h + create mode 100644 src/lj_mem_tracing.c + create mode 100644 src/lj_mem_tracing.h + +diff --git a/bundle/LuaJIT-2.1-20210510/src/Makefile b/bundle/LuaJIT-2.1-20210510/src/Makefile +index 47a21c93..bf563363 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/Makefile ++++ b/bundle/LuaJIT-2.1-20210510/src/Makefile +@@ -112,6 +112,9 @@ XCFLAGS= + # + # Disable LJ_GC64 mode for x64. + #XCFLAGS+= -DLUAJIT_DISABLE_GC64 ++ ++# Kong profiling extension library ++#XCFLAGS+= -DLUAJIT_DISABLE_KPROF + # + ############################################################################## + +@@ -226,7 +229,8 @@ TARGET_DYNXLDOPTS= + TARGET_LFSFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + TARGET_XCFLAGS= $(TARGET_LFSFLAGS) -U_FORTIFY_SOURCE + TARGET_XLDFLAGS= +-TARGET_XLIBS= -lm ++TARGET_XLIBS= -lm -lmsgpackc # We don't generate the dynamic lib during the building process, ++ # so the linker will use the static lib instead. + TARGET_TCFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS) + TARGET_ACFLAGS= $(CCOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS) + TARGET_ASFLAGS= $(ASOPTIONS) $(TARGET_XCFLAGS) $(TARGET_FLAGS) $(TARGET_CFLAGS) +@@ -492,13 +496,13 @@ LJVM_BOUT= $(LJVM_S) + LJVM_MODE= elfasm + + LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \ +- lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o ++ lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o lib_kprof.o + LJLIB_C= $(LJLIB_O:.o=.c) + +-LJCORE_O= lj_assert.o lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \ ++LJCORE_O= lj_assert.o lj_gc.o lj_gcsnapshot.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \ + lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \ + lj_prng.o lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o \ +- lj_strscan.o lj_strfmt.o lj_strfmt_num.o lj_api.o lj_profile.o \ ++ lj_strscan.o lj_strfmt.o lj_strfmt_num.o lj_api.o lj_profile.o lj_mem_tracing.o \ + lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \ + lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \ + lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \ +diff --git a/bundle/LuaJIT-2.1-20210510/src/Makefile.dep b/bundle/LuaJIT-2.1-20210510/src/Makefile.dep +index 3f26599e..f7c667e4 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/Makefile.dep ++++ b/bundle/LuaJIT-2.1-20210510/src/Makefile.dep +@@ -41,6 +41,8 @@ lib_string.o: lib_string.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ + lib_table.o: lib_table.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ + lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \ + lj_tab.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h ++lib_kprof.o: lj_libdef.h lj_mem_tracing.h lj_gcsnapshot.h lauxlib.h lualib.h \ ++ lj_gc.h lj_lib.h lj_err.h + lj_alloc.o: lj_alloc.c lj_def.h lua.h luaconf.h lj_arch.h lj_alloc.h \ + lj_prng.h + lj_api.o: lj_api.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ +@@ -80,7 +82,8 @@ lj_cconv.o: lj_cconv.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_gc.h lj_cdata.h lj_cconv.h \ + lj_ccallback.h + lj_cdata.o: lj_cdata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ +- lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h ++ lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h \ ++ lj_mem_tracing.h + lj_char.o: lj_char.c lj_char.h lj_def.h lua.h luaconf.h + lj_clib.o: lj_clib.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ + lj_err.h lj_errmsg.h lj_tab.h lj_str.h lj_udata.h lj_ctype.h lj_cconv.h \ +@@ -121,6 +124,13 @@ lj_gc.o: lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ + lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h lj_udata.h \ + lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h \ + lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h ++lj_gcsnapshot.o: lj_gcsnapshot.c lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ ++ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h lj_udata.h \ ++ lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h \ ++ lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h ++lj_mem_tracing.o: lj_mem_tracing.c lj_profile.h lj_def.h lj_obj.h \ ++ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h lj_udata.h \ ++ lj_cdata.h lj_vmevent.h lj_mbuf.h + lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_buf.h \ + lj_str.h lj_strfmt.h lj_jit.h lj_ir.h lj_dispatch.h +@@ -189,7 +199,7 @@ lj_state.o: lj_state.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_prng.h lj_lex.h \ + lj_alloc.h luajit.h + lj_str.o: lj_str.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ +- lj_err.h lj_errmsg.h lj_str.h lj_char.h ++ lj_err.h lj_errmsg.h lj_str.h lj_char.h lj_mem_tracing.h + lj_strfmt.o: lj_strfmt.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_buf.h lj_gc.h lj_str.h lj_state.h lj_char.h lj_strfmt.h + lj_strfmt_num.o: lj_strfmt_num.c lj_obj.h lua.h luaconf.h lj_def.h \ +@@ -197,17 +207,17 @@ lj_strfmt_num.o: lj_strfmt_num.c lj_obj.h lua.h luaconf.h lj_def.h \ + lj_strscan.o: lj_strscan.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_char.h lj_strscan.h + lj_tab.o: lj_tab.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ +- lj_err.h lj_errmsg.h lj_tab.h ++ lj_err.h lj_errmsg.h lj_tab.h lj_mem_tracing.h + lj_trace.o: lj_trace.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_frame.h lj_bc.h \ + lj_state.h lj_ir.h lj_jit.h lj_iropt.h lj_mcode.h lj_trace.h \ + lj_dispatch.h lj_traceerr.h lj_snap.h lj_gdbjit.h lj_record.h lj_asm.h \ + lj_vm.h lj_vmevent.h lj_target.h lj_target_*.h lj_prng.h + lj_udata.o: lj_udata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ +- lj_gc.h lj_udata.h ++ lj_gc.h lj_udata.h lj_mem_tracing.h + lj_vmevent.o: lj_vmevent.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_str.h lj_tab.h lj_state.h lj_dispatch.h lj_bc.h lj_jit.h lj_ir.h \ +- lj_vm.h lj_vmevent.h ++ lj_vm.h lj_vmevent.h lj_mem_tracing.h + lj_vmmath.o: lj_vmmath.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ + lj_ir.h lj_vm.h + ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_assert.c lj_obj.h \ +@@ -232,7 +242,7 @@ ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_assert.c lj_obj.h \ + lj_asm_*.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c lib_aux.c \ + lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c lib_io.c \ + lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c lib_ffi.c \ +- lib_init.c ++ lib_init.c lib_kprof.c + luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h + host/buildvm.o: host/buildvm.c host/buildvm.h lj_def.h lua.h luaconf.h \ + lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_gc.h lj_obj.h lj_bc.h lj_ir.h \ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lib_base.c b/bundle/LuaJIT-2.1-20210510/src/lib_base.c +index 4e4f686e..7de25376 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lib_base.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lib_base.c +@@ -36,6 +36,8 @@ + #include "lj_strfmt.h" + #include "lj_lib.h" + #include "lj_cdata.h" ++#include "lj_gcsnapshot.h" ++ + + /* -- Base library: checks ------------------------------------------------ */ + +diff --git a/bundle/LuaJIT-2.1-20210510/src/lib_init.c b/bundle/LuaJIT-2.1-20210510/src/lib_init.c +index 56e0619a..fddb31a0 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lib_init.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lib_init.c +@@ -26,6 +26,11 @@ static const luaL_Reg lj_lib_load[] = { + { LUA_DBLIBNAME, luaopen_debug }, + { LUA_BITLIBNAME, luaopen_bit }, + { LUA_JITLIBNAME, luaopen_jit }, ++ ++#ifndef LUAJIT_DISABLE_KPROF ++ { LUA_KPROFNAME, luaopen_kprof }, ++#endif ++ + { NULL, NULL } + }; + +diff --git a/bundle/LuaJIT-2.1-20210510/src/lib_kprof.c b/bundle/LuaJIT-2.1-20210510/src/lib_kprof.c +new file mode 100644 +index 00000000..4b241b88 +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lib_kprof.c +@@ -0,0 +1,116 @@ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++/* ++** Kong profiling extension library. ++*/ ++ ++#define lib_kprof_c ++#define LUA_LIB ++ ++#include "lua.h" ++#include "lualib.h" ++#include "lj_gc.h" ++#include "lj_lib.h" ++#include "lj_err.h" ++#include "lauxlib.h" ++#include "lj_gcsnapshot.h" ++#include "lj_mem_tracing.h" ++ ++ ++#define DEFAULT_GCSNAPSHOT_TIMEOUT (120) /* seconds */ ++#define DEFAULT_MEMTRACE_TIMEOUT (120) /* seconds */ ++ ++#define MIN_BLOCK_SIZE (1024 * 1024 * 128) /* 128MiB */ ++#define MIN_STACK_DEPTH (1) ++#define MAX_STACK_DEPTH (64) ++ ++/* ------------------------------------------------------------------------ */ ++ ++#define LJLIB_MODULE_kprof ++#define LJLIB_MODULE_kprof_mem ++ ++LJLIB_CF(kprof_mem_gcsnapshot) ++{ ++ const char* filename = luaL_checkstring(L, 1); ++ ++ if (filename == NULL) { ++ lj_err_argtype(L, 1, "string"); ++ return 0; ++ } ++ ++ unsigned int timeout = luaL_optint(L, 2, (lua_Integer)DEFAULT_GCSNAPSHOT_TIMEOUT); ++ ++ /* force a full GC before we snapshot */ ++ lj_gc_fullgc(L); ++ ++ char* err = NULL; ++ ++ if (gc_snapshot(L, filename, (const char**)(&err), timeout)) { ++ lua_pushnil(L); ++ lua_pushstring(L, err); ++ return 2; ++ } ++ ++ lua_pushboolean(L, 1); ++ lua_pushnil(L); ++ return 2; ++} ++ ++LJLIB_CF(kprof_mem_start) ++{ ++ const char* filename = luaL_checkstring(L, 1); ++ size_t block_size = (size_t)luaL_checkinteger(L, 2); ++ uint32_t stack_depth = (uint32_t)luaL_checkinteger(L, 3); ++ unsigned int timeout = luaL_optint(L, 4, (lua_Integer)DEFAULT_MEMTRACE_TIMEOUT); ++ ++ if (block_size < MIN_BLOCK_SIZE) { ++ lua_pushnil(L); ++ lua_pushstring(L, "arg #2 must be at least 128MiB"); ++ return 2; ++ } ++ ++ if (stack_depth < MIN_STACK_DEPTH || stack_depth > MAX_STACK_DEPTH) { ++ lua_pushnil(L); ++ lua_pushstring(L, "arg #3 must be between 1 and 64"); ++ return 2; ++ } ++ ++ if (lj_mem_tracing_start(L, filename, block_size, stack_depth, timeout)) { ++ lua_pushnil(L); ++ lua_insert(L, -2); ++ return 2; ++ } ++ ++ lua_pushboolean(L, 1); ++ lua_pushnil(L); ++ return 2; ++} ++ ++LJLIB_CF(kprof_mem_stop) ++{ ++ if (lj_mem_tracing_stop(L)) { ++ lua_pushnil(L); ++ lua_insert(L, -2); ++ return 2; ++ } ++ ++ lua_pushboolean(L, 1); ++ lua_pushnil(L); ++ return 2; ++} ++ ++LJLIB_CF(kprof_mem_status) ++{ ++ return lj_mem_tracing_status(L); ++} ++ ++#include "lj_libdef.h" ++ ++LUALIB_API int luaopen_kprof(lua_State *L) ++{ ++ LJ_LIB_REG(L, LUA_KPROFNAME, kprof); ++ LJ_LIB_REG(L, LUA_KPROFNAME ".mem", kprof_mem); ++ return 1; ++} ++ ++#endif /* LUAJIT_DISABLE_KPROF */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_bcread.c b/bundle/LuaJIT-2.1-20210510/src/lj_bcread.c +index 047f98df..3930a68f 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_bcread.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_bcread.c +@@ -22,6 +22,8 @@ + #include "lj_bcdump.h" + #include "lj_state.h" + #include "lj_strfmt.h" ++#include "lj_mem_tracing.h" ++ + + /* Reuse some lexer fields for our own purposes. */ + #define bcread_flags(ls) ls->level +@@ -382,6 +384,9 @@ GCproto *lj_bcread_proto(LexState *ls) + setmref(pt->uvinfo, NULL); + setmref(pt->varinfo, NULL); + } ++ ++ lj_mem_tracing_append_proto(G(ls->L), pt); ++ + return pt; + } + +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_cdata.c b/bundle/LuaJIT-2.1-20210510/src/lj_cdata.c +index ffc31078..65ba6028 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_cdata.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_cdata.c +@@ -14,6 +14,7 @@ + #include "lj_cconv.h" + #include "lj_cdata.h" + ++ + /* -- C data allocation --------------------------------------------------- */ + + /* Allocate a new C data object holding a reference to another object. */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_cdata.h b/bundle/LuaJIT-2.1-20210510/src/lj_cdata.h +index b93bec86..0b0efb87 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_cdata.h ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_cdata.h +@@ -10,6 +10,7 @@ + #include "lj_gc.h" + #include "lj_ctype.h" + ++ + #if LJ_HASFFI + + /* Get C data pointer. */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_debug.c b/bundle/LuaJIT-2.1-20210510/src/lj_debug.c +index bef4aa58..f3a1fbbc 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_debug.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_debug.c +@@ -583,6 +583,11 @@ static int debug_putchunkname(SBuf *sb, GCproto *pt, int pathstrip) + return 1; + } + ++int lj_debug_putchunkname(SBuf *sb, GCproto *pt, int pathstrip) ++{ ++ return debug_putchunkname(sb, pt, pathstrip); ++} ++ + /* Put a compact stack dump into a buffer. */ + void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt, int depth) + { +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_debug.h b/bundle/LuaJIT-2.1-20210510/src/lj_debug.h +index 330bec25..91664627 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_debug.h ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_debug.h +@@ -43,6 +43,7 @@ LJ_FUNC void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc); + LJ_FUNC int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, + int ext); + #if LJ_HASPROFILE ++LJ_FUNC int lj_debug_putchunkname(SBuf *sb, GCproto *pt, int pathstrip); + LJ_FUNC void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt, + int depth); + #endif +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_dispatch.c b/bundle/LuaJIT-2.1-20210510/src/lj_dispatch.c +index bf8d8812..04fd981c 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_dispatch.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_dispatch.c +@@ -30,6 +30,9 @@ + #if LJ_HASPROFILE + #include "lj_profile.h" + #endif ++ ++#include "lj_mem_tracing.h" ++ + #include "lj_vm.h" + #include "luajit.h" + +@@ -434,8 +437,15 @@ void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc) + L->top = L->base + slots; /* Fix top again. */ + } + } +- if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1]))) +- callhook(L, LUA_HOOKRET, -1); ++ if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1]))) { ++ if (lj_mem_tracing_is_enabled(g)) { ++ lj_mem_tracing_on_call_return(L); ++ ++ } else { ++ callhook(L, LUA_HOOKRET, -1); ++ } ++ } ++ + ERRNO_RESTORE + } + +@@ -494,7 +504,14 @@ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc) + int i; + for (i = 0; i < missing; i++) /* Add missing parameters. */ + setnilV(L->top++); +- callhook(L, LUA_HOOKCALL, -1); ++ ++ if (lj_mem_tracing_is_enabled(g)) { ++ lj_mem_tracing_on_call_return(L); ++ ++ } else { ++ callhook(L, LUA_HOOKCALL, -1); ++ } ++ + /* Preserve modifications of missing parameters by lua_setlocal(). */ + while (missing-- > 0 && tvisnil(L->top - 1)) + L->top--; +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_gc.c b/bundle/LuaJIT-2.1-20210510/src/lj_gc.c +index cfbce037..e3366737 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_gc.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_gc.c +@@ -854,6 +854,7 @@ void lj_gc_barriertrace(global_State *g, uint32_t traceno) + /* Call pluggable memory allocator to allocate or resize a fragment. */ + void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz) + { ++ void* op = p; + global_State *g = G(L); + lj_assertG((osz == 0) == (p == NULL), "realloc API violation"); + p = g->allocf(g->allocd, p, osz, nsz); +@@ -863,6 +864,9 @@ void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz) + lj_assertG(checkptrGC(p), + "allocated memory address %p outside required range", p); + g->gc.total = (g->gc.total - osz) + nsz; ++ ++ lj_mem_tracing_probe(G(L), op, p, osz, nsz); ++ + return p; + } + +@@ -879,6 +883,9 @@ void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size) + setgcrefr(o->gch.nextgc, g->gc.root); + setgcref(g->gc.root, o); + newwhite(g, o); ++ ++ lj_mem_tracing_probe(G(L), NULL, o, 0, size); ++ + return o; + } + +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_gc.h b/bundle/LuaJIT-2.1-20210510/src/lj_gc.h +index af8c476b..bb90b6b9 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_gc.h ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_gc.h +@@ -7,6 +7,7 @@ + #define _LJ_GC_H + + #include "lj_obj.h" ++#include "lj_mem_tracing.h" + + /* Garbage collector states. Order matters. */ + enum { +@@ -120,6 +121,7 @@ static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize) + { + g->gc.total -= (GCSize)osize; + g->allocf(g->allocd, p, osize, 0); ++ lj_mem_tracing_probe(g, p, NULL, osize, 0); + } + + #define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n)*sizeof(t)))) +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.c b/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.c +new file mode 100644 +index 00000000..d43136bd +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.c +@@ -0,0 +1,947 @@ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++#define lj_gcsnapshot_c ++#define LUA_CORE ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "lj_gcsnapshot.h" ++#include "lj_err.h" ++#include "lj_buf.h" ++#include "lj_str.h" ++#include "lj_tab.h" ++#include "lj_func.h" ++#include "lj_udata.h" ++#include "lj_meta.h" ++#include "lj_state.h" ++#include "lj_frame.h" ++#include "lj_debug.h" ++#if LJ_HASFFI ++#include "lj_ctype.h" ++#include "lj_cdata.h" ++#endif ++#include "lj_trace.h" ++ ++/* -- functions for heapsnapshot ------------------------------------------ */ ++ ++/* encoding functions for Lua Object */ ++ ++/* ++** ++** The GC snapshot is a array of map. ++** The first element is the meta data of this snapshot including the following fields: ++** { ++** version = { ++** major = , ++** minor = , ++** patch = , ++** string = "" ++** }, ++** gcsize = , ++** changes = "", ++** } ++** Versioning is strictly based on Semantic Versioning (https://semver.org/) ++** ++** The remaining elements are the objects in the Lua heap. ++** ++** Every Lua object will be encoded ('messagepack') as a dictionary. ++** ++** For Lua String: ++** { type = "string", value = "", bytes=+ } ++** ++** For Lua Number: ++** { type = "number", value = "", bytes= } ++** ++** For Lua Boolean: ++** { type = "boolean", value = "", bytes= } ++** ++** For Lua Lightuserdata: ++** { type = "lightud", address = uint64_t, bytes= } ++** ++** For Lua Trace: ++** { type = "trace", address = uint64_t, bytes= } ++** ++** For Lua Table: ++** { ++** type = "table", ++** address = uint64_t, ++** overhead=+, ++** hash=[, , ...], ++** array=[, ...] ++** } ++** ++** For Lua Function: ++** { ++** type = "function", ++** address = uint64_t, ++** overhead = , ++** env = , ++** upvalues = [, , ...] ++** } ++** ++** For Lua Thread: ++** { ++** type = "thread", ++** address = uint64_t, ++** overhead = , ++** env = , ++** upvalues = [, ...] ++** } ++** ++** For Lua Proto: ++** { ++** type = "proto", ++** address = uint64_t, ++** bytes = , ++** } ++** ++** For Lua cdata: ++** { ++** type = "cdata", ++** address = uint64_t, ++** bytes = , ++** ctype = ++** } ++** ++** For Lua udata: ++** { ++** type = "udata", ++** address = uint64_t, ++** bytes = ++** } ++** ++** For Lua nil: ++** { type = "nil", bytes= } ++** ++** Sub-objects are used to avoid circular references. ++** ++** If a sub-object is a simple object (such as a string), it will be encoded into its parent object, for example: ++** { type = "table", ++** address = uint64_t, ++** overhead=+, ++** hash=[{ type = "string", value = "", bytes=+ }, , ...], ++** array=[, ...] ++** } ++** ++** If a sub-object is a complex object (such as a table), it will be encoded as a reference, for example: ++** { type = "table", ++** address = uint64_t, ++** overhead=+, ++** hash=[, { type = "ref", address = uint64_t }, ...], ++** array=[, ...] ++** } ++** ++*/ ++ ++#define TIMEOUT_CHECK_STEP (1000) ++#define RETURN_ON_ERROR(v) { if (v) return 1; } ++ ++#define SNAPSHOT_CHANGES \ ++ "2.0.0\n" \ ++ "* Add the `metatable` field to the `table` type.\n" \ ++ "* Add the `position` field to the `function` type" \ ++ " (only for Lua function).\n" \ ++ "* Add the name for each upvalue in the `upvalues`" \ ++ "field of the `function` type" \ ++ " (only for Lua function).\n" ++ ++/* encoding functions for Lua Types */ ++static int mp_TValue(lua_State *L, msgpack_packer* pk, TValue* tv); ++static int mp_boolean(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_string(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_number(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_lightud(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_trace(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_table(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_function(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_thread(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_proto(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_cdata(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_udata(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_nil(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++static int mp_subobjects(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv); ++ ++/* encoding functions for C Type */ ++static int mp_cstring(msgpack_packer* pk, const char* str); ++ ++static int mp_snapshot_header(lua_State *L, msgpack_packer* pk); ++ ++static int is_supported_subobject(GCobj* o, TValue* tv) { ++ if (o) { ++ uint8_t gct = o->gch.gct; ++ ++ switch(gct) { ++ case ~LJ_TSTR: ++ case ~LJ_TTRUE: ++ case ~LJ_TFALSE: ++ case ~LJ_TLIGHTUD: ++ case ~LJ_TNIL: ++ case ~LJ_TTAB: ++ case ~LJ_TTHREAD: ++ case ~LJ_TFUNC: ++ case ~LJ_TPROTO: ++ case ~LJ_TUDATA: ++ case ~LJ_TCDATA: ++ case ~LJ_TTRACE: ++ return 1; ++ ++ default: ++ return 0; ++ } ++ ++ } else if (tv) { ++ if (tvistab(tv) || tvisthread(tv) || tvisfunc(tv) || ++ tvisproto(tv) || tvisudata(tv) || tviscdata(tv) || ++ tvisbool(tv) || tvisnumber(tv) || tvisstr(tv) || ++ tvislightud(tv) || tvisnil(tv)) { ++ return 1; ++ } ++ } ++ ++ return 0; ++} ++ ++ ++static int mp_TValue(lua_State *L, msgpack_packer* pk, TValue* tv) { ++ if (tv == NULL) return 1; ++ ++ if (tvisbool(tv)) { ++ return mp_boolean(L, pk, NULL, tv); ++ ++ } else if (tvisstr(tv)) { ++ return mp_string(L, pk, NULL, tv); ++ ++ } else if (tvisnumber(tv)) { ++ return mp_number(L, pk, NULL, tv); ++ ++ } else if (tvislightud(tv)) { ++ return mp_lightud(L, pk, NULL, tv); ++ ++ } else if (tvisnil(tv)) { ++ return mp_nil(L, pk, NULL, NULL); ++ } ++ ++ return 0; ++} ++ ++static int mp_boolean(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ ++ if (o == NULL && tv == NULL) return 1; ++ ++ int is_true = o ? (o->gch.gct == ~LJ_TTRUE) : tvistrue(tv); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "boolean")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(TValue))); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "value")); ++ if (is_true) { ++ RETURN_ON_ERROR(msgpack_pack_true(pk)); ++ ++ } else { ++ RETURN_ON_ERROR(msgpack_pack_false(pk)); ++ } ++ ++ return 0; ++} ++ ++static int mp_string(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ ++ if (o == NULL && tv == NULL) return 1; ++ ++ GCstr*s = NULL; ++ ++ if (o) { ++ s = gco2str(o); ++ } else { ++ s = strV(tv); ++ } ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "string")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(GCstr) + lj_str_size(s->len))); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "value")); ++ if (s->len > 0) { ++ RETURN_ON_ERROR(msgpack_pack_bin(pk, s->len)); ++ RETURN_ON_ERROR(msgpack_pack_bin_body(pk, strdata(s), s->len)); ++ ++ } else { ++ RETURN_ON_ERROR(msgpack_pack_nil(pk)); ++ } ++ ++ ++ return 0; ++} ++ ++static int mp_number(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ UNUSED(o); ++ ++ if (tv == NULL) return 1; ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "number")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(TValue))); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "value")); ++ ++ if (tvisint(tv)) { ++ RETURN_ON_ERROR(msgpack_pack_int64(pk, intV(tv))); ++ ++ } else { ++ lj_assertL(tvisnum(tv), "unknown number type"); ++ RETURN_ON_ERROR(msgpack_pack_double(pk, numV(tv))); ++ } ++ ++ return 0; ++} ++ ++static int mp_lightud(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ ++ if (o == NULL && tv == NULL) return 1; ++ ++ void* ptr = NULL; ++ ++ if (o) { ++ ptr = o; ++ ++ } else { ++ ptr = tv; ++ } ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "lightud")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(TValue))); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)ptr)); ++ ++ return 0; ++} ++ ++static int mp_trace(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ GCtrace* T = gco2trace(o); ++ ++ if (T->traceno == 0) return 0; ++ ++ MSize size = ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + ++ T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry); ++ ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "trace")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, size)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, (uint64_t)T)); ++ ++ return 0; ++} ++ ++static int mp_table(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ GCtab* t = gco2tab(o); ++ ++ MSize overhead = 0; ++ ++ if (t->hmask > 0) { ++ overhead += (t->hmask + 1) * sizeof(Node); ++ } ++ ++ if (t->asize > 0 && LJ_MAX_COLOSIZE != 0 && t->colo <= 0) { ++ overhead += t->asize * sizeof(TValue); ++ } ++ ++ if (LJ_MAX_COLOSIZE != 0 && t->colo) { ++ overhead += sizetabcolo((uint32_t)t->colo & 0x7f); ++ ++ } else { ++ overhead += sizeof(GCtab); ++ } ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 6)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "table")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, (uint64_t)t)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "overhead")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, overhead)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "metatable")); ++ GCtab *mt = tabref(t->metatable); ++ if (mt && is_supported_subobject((GCobj*)mt, NULL)) { ++ RETURN_ON_ERROR(mp_subobjects(L, pk, (GCobj*)mt, NULL)); ++ ++ } else { ++ RETURN_ON_ERROR(msgpack_pack_nil(pk)); ++ } ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "array")); ++ RETURN_ON_ERROR(msgpack_pack_array(pk, t->asize)); ++ uint64_t i = 0; ++ for (i = 0; i < t->asize; i++) { ++ TValue* tv = arrayslot(t, i); ++ ++ if (is_supported_subobject(NULL, tv)) { ++ RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, tv)); ++ ++ } else { ++ RETURN_ON_ERROR(msgpack_pack_nil(pk)); ++ } ++ } ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "hash")); ++ uint64_t j, count = 0; ++ for (i = 0; i < 2; i++) { ++ Node *node = noderef(t->node); ++ ++ if (i == 1) RETURN_ON_ERROR(msgpack_pack_array(pk, count * 2)); ++ ++ for (j = 0; j < t->hmask + 1; j++) { ++ Node* n = &node[j]; ++ ++ if (!tvisnil(&n->key) && ++ !tvisnil(&n->val) && ++ is_supported_subobject(NULL, &n->key) && ++ is_supported_subobject(NULL, &n->val)) { ++ if (i == 0) ++count; ++ else { ++ RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, &n->key)); ++ RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, &n->val)); ++ } ++ } ++ } ++ ++ } ++ ++ return 0; ++} ++ ++static int mp_function(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ GCfunc* fn = gco2func(o); ++ ++ MSize overhead = isluafunc(fn) ? ++ sizeLfunc((MSize)fn->l.nupvalues) + sizeof(GCupval) * fn->l.nupvalues : ++ sizeCfunc((MSize)fn->c.nupvalues) + sizeof(GCupval) * fn->c.nupvalues; ++ ++ size_t items = isluafunc(fn) ? 6 : 5; ++ RETURN_ON_ERROR(msgpack_pack_map(pk, items)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "function")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, (uint64_t)fn)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "overhead")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, overhead)); ++ ++ if (isluafunc(fn)) { ++ /* ++ ** basically a copy of lj_debug_pushloc() in src/lj_debug.c ++ */ ++ ++ GCproto* pt = funcproto(fn); ++ GCstr *name = proto_chunkname(pt); ++ const char *s = strdata(name); ++ char* pos = NULL; ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "position")); ++ ++ if (pt->firstline == ~(BCLine)0) { ++ /* -1 and + 1 for the null terminator */ ++ size_t size = (sizeof("builtin:") - 1) + (name->len + 1); ++ pos = calloc(size, sizeof(char)); ++ ++ if (pos == NULL) return 1; ++ ++ snprintf(pos, size, "builtin:%s", s); ++ RETURN_ON_ERROR(msgpack_pack_str(pk, size - 1)); ++ RETURN_ON_ERROR(msgpack_pack_str_body(pk, pos, size - 1)); ++ ++ free(pos); ++ ++ } else { ++ size_t len = 0; ++ ++ if (*s == '@' || *s == '=') { ++ /* ++ ** + 32 for the line number ++ ** + 1 for the ':' ++ ** + 1 for the null terminator ++ */ ++ size_t size = (name->len - 1) + 32 + 1 + 1; ++ pos = calloc(size, sizeof(char)); ++ if (pos == NULL) return 1; ++ /* s + 1 to skip the '@' or '=' */ ++ len = sprintf(pos, "%s:%" PRIu32 , s + 1, (uint32_t)(pt->firstline)); ++ ++ } else if (name->len > 40) { ++ /* ++ ** + 64 for the address ++ ** + 32 for the line number ++ ** + 1 for the ':' ++ ** + 1 for the null terminator ++ */ ++ size_t size = 64 + 32 + 1 + 1; ++ pos = calloc(size, sizeof(char)); ++ if (pos == NULL) return 1; ++ len = sprintf(pos, "%" PRIuPTR ":%" PRIu32, (uintptr_t)pt, (uint32_t)(pt->firstline)); ++ ++ } else { ++ /* ++ ** + 32 for the line number ++ ** + 2 for two '\"' ++ ** + 1 for the ':' ++ ** + 1 for the null terminator ++ */ ++ size_t size = name->len + 32 + 2 + 1 + 1; ++ pos = calloc(size, sizeof(char)); ++ if (pos == NULL) return 1; ++ len = sprintf(pos, "\"%s\":%" PRIu32, s, (uint32_t)(pt->firstline)); ++ } ++ ++ RETURN_ON_ERROR(msgpack_pack_str(pk, len)); ++ RETURN_ON_ERROR(msgpack_pack_str_body(pk, pos, len)); ++ free(pos); ++ } ++ } ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "env")); ++ GCtab* env = tabref(fn->c.env); ++ RETURN_ON_ERROR(mp_subobjects(L, pk, (GCobj*)env, NULL)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "upvalues")); ++ uint32_t i, j, count = 0; ++ for (i = 0; i < 2; i++) { ++ if (i == 1) RETURN_ON_ERROR(msgpack_pack_array(pk, count * 2)); ++ ++ if (isluafunc(fn)) { ++ for (j = 0; j < fn->l.nupvalues; j++) { ++ GCupval* uv = &gcref(fn->l.uvptr[j])->uv; ++ TValue* tv = uvval(uv); ++ ++ if (is_supported_subobject(NULL, tv)) { ++ if (i == 0) ++count; ++ else { ++ RETURN_ON_ERROR(mp_cstring(pk, lj_debug_uvname(funcproto(fn), j))); ++ RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, tv)); ++ } ++ } ++ } ++ ++ } else { ++ for (j = 0; j < fn->c.nupvalues; j++) { ++ TValue* tv = &fn->c.upvalue[j]; ++ ++ if (is_supported_subobject(NULL, tv)) { ++ if (i == 0) ++count; ++ else { ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, j)); ++ RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, tv)); ++ } ++ } ++ } ++ } ++ } ++ ++ return 0; ++} ++ ++static int mp_thread(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ lua_State* th = gco2th(o); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 5)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "thread")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, (uint64_t)th)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "overhead")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, sizeof(lua_State) + th->stacksize * sizeof(TValue))); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "env")); ++ RETURN_ON_ERROR(mp_subobjects(L, pk, (GCobj*)tabref(th->env), NULL)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "upvalues")); ++ uint32_t i, count = 0; ++ for (i = 0; i < 2; i++) { ++ GCobj* uv = gcref(th->openupval); ++ ++ if (i == 1) RETURN_ON_ERROR(msgpack_pack_array(pk, count)); ++ ++ while (uv) { ++ if (is_supported_subobject(NULL, uvval(gco2uv(uv)))) { ++ if (i == 0) ++count; ++ else RETURN_ON_ERROR(mp_subobjects(L, pk, NULL, uvval(gco2uv(uv)))); ++ } ++ ++ uv = gcref(uv->gch.nextgc); ++ } ++ } ++ ++ return 0; ++} ++ ++static int mp_proto(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ GCproto* pt = gco2pt(o); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "proto")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)pt)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(GCproto) + pt->sizekgc * sizeof(GCRef))); ++ ++ return 0; ++} ++ ++static int mp_cdata(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ global_State *g = G(L); ++ GCcdata* cd = gco2cd(o); ++ GCstr* name = NULL; ++ ++ MSize cd_size = 0; ++ if (LJ_LIKELY(!cdataisv(cd))) { ++ CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid); ++ CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR; ++ cd_size = sizeof(GCcdata) + sz; ++ ++ if (gcref(ct->name)) { ++ name = gco2str(gcref(ct->name)); ++ } ++ ++ } else { ++ cd_size = sizecdatav(cd); ++ } ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 4)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "cdata")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)cd)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, cd_size)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "ctype")); ++ if (name == NULL) { ++ RETURN_ON_ERROR(mp_cstring(pk, "N/A")); ++ ++ } else { ++ if (name->len > 0) { ++ RETURN_ON_ERROR(msgpack_pack_bin(pk, name->len)); ++ RETURN_ON_ERROR(msgpack_pack_bin_body(pk, strdata(name), name->len)); ++ ++ } else { ++ RETURN_ON_ERROR(msgpack_pack_nil(pk)); ++ } ++ } ++ ++ return 0; ++} ++ ++static int mp_udata(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ UNUSED(tv); ++ ++ if (o == NULL) return 1; ++ ++ GCudata* ud = gco2ud(o); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "udata")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)ud)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeudata(ud))); ++ ++ return 0; ++} ++ ++static int mp_nil(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ UNUSED(L); ++ UNUSED(o); ++ UNUSED(tv); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 2)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "nil")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "bytes")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, sizeof(TValue))); ++ ++ return 0; ++} ++ ++static int mp_subobjects(lua_State *L, msgpack_packer* pk, GCobj* o, TValue* tv) { ++ if (o) { ++ uint8_t gct = o->gch.gct; ++ ++ switch(gct) { ++ case ~LJ_TSTR: ++ return mp_string(L, pk, o, NULL); ++ ++ case ~LJ_TTRUE: ++ case ~LJ_TFALSE: ++ return mp_boolean(L, pk, o, NULL); ++ ++ case ~LJ_TLIGHTUD: ++ return mp_lightud(L, pk, o, NULL); ++ ++ case ~LJ_TNIL: ++ return mp_nil(L, pk, NULL, NULL); ++ ++ case ~LJ_TTAB: ++ case ~LJ_TTHREAD: ++ case ~LJ_TFUNC: ++ case ~LJ_TPROTO: ++ case ~LJ_TUDATA: ++ case ~LJ_TCDATA: ++ case ~LJ_TTRACE: ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 2)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "ref")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)o)); ++ ++ return 0; ++ ++ default: ++ return 0; ++ } ++ ++ } else if (tv) { ++ if (tvistab(tv) || tvisthread(tv) || tvisfunc(tv) || tvisproto(tv) || tvisudata(tv) || tviscdata(tv)) { ++ GCobj* o = gcV(tv); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 2)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "type")); ++ RETURN_ON_ERROR(mp_cstring(pk, "ref")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "address")); ++ RETURN_ON_ERROR(msgpack_pack_fix_uint64(pk, (uint64_t)o)); ++ ++ return 0; ++ ++ } else if (tvisbool(tv) || tvisnumber(tv) || tvisstr(tv) || tvislightud(tv) || tvisnil(tv)) { ++ // TValue should not is a Trace object ++ RETURN_ON_ERROR(mp_TValue(L, pk, tv)); ++ return 0; ++ } ++ } ++ ++ return 0; ++} ++ ++ ++static int mp_snapshot_header(lua_State *L, msgpack_packer* pk) { ++ global_State *g = G(L); ++ ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 3)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "version")); ++ RETURN_ON_ERROR(msgpack_pack_map(pk, 4)); ++ RETURN_ON_ERROR(mp_cstring(pk, "major")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, 2)); ++ RETURN_ON_ERROR(mp_cstring(pk, "minor")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, 0)); ++ RETURN_ON_ERROR(mp_cstring(pk, "patch")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, 0)); ++ RETURN_ON_ERROR(mp_cstring(pk, "string")); ++ RETURN_ON_ERROR(mp_cstring(pk, "2.0.0")); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "gcsize")); ++ RETURN_ON_ERROR(msgpack_pack_uint64(pk, g->gc.total)); ++ ++ RETURN_ON_ERROR(mp_cstring(pk, "changes")); ++ RETURN_ON_ERROR(mp_cstring(pk, SNAPSHOT_CHANGES)); ++ ++ return 0; ++} ++ ++ ++static int mp_cstring(msgpack_packer* pk, const char* str) { ++ lj_assertX(str != NULL, "expected string to be non-null in function mp_cstring()"); ++ ++ size_t len = strlen(str); ++ RETURN_ON_ERROR(msgpack_pack_str(pk, len)); ++ RETURN_ON_ERROR(msgpack_pack_str_body(pk, str, len)); ++ return 0; ++} ++ ++ ++int gc_snapshot(lua_State *L, const char* filename, const char** err, unsigned int timeout) { ++ global_State *g = G(L); ++ ++ time_t timeout_at = time(NULL) + timeout; ++ ++ FILE* fp = fopen(filename, "wb"); ++ ++ if (fp == NULL) { ++ *err = strerror(errno); ++ return 1; ++ } ++ ++ msgpack_packer pk; ++ msgpack_packer_init(&pk, fp, msgpack_fbuffer_write); ++ ++ if (mp_snapshot_header(L, &pk)) { ++ *err = "failed to write snapshot header"; ++ fclose(fp); ++ return 1; ++ } ++ ++ GCobj* o = gcref(g->gc.root); ++ size_t count = 0; ++ ++ while (o != NULL) { ++ uint8_t gct = o->gch.gct; ++ ++ ++count; ++ ++ if (count % TIMEOUT_CHECK_STEP == 0) { ++ if (time(NULL) > timeout_at) { ++ *err = "timeout"; ++ fclose(fp); ++ return 1; ++ } ++ } ++ ++ int rc = 0; ++ ++ switch(gct) { ++ case ~LJ_TSTR: ++ rc = mp_string(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TTRUE: ++ case ~LJ_TFALSE: ++ rc = mp_boolean(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TLIGHTUD: ++ rc = mp_lightud(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TTRACE: ++ rc = mp_trace(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TTAB: ++ rc = mp_table(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TFUNC: ++ rc = mp_function(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TTHREAD: ++ rc = mp_thread(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TPROTO: ++ rc = mp_proto(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TCDATA: ++ rc = mp_cdata(L, &pk, o, NULL); ++ break; ++ ++ case ~LJ_TUDATA: ++ rc = mp_udata(L, &pk, o, NULL); ++ break; ++ ++ default: ++ break; ++ } ++ ++ if (rc) { ++ *err = "unknown error on packing gc objects"; ++ fclose(fp); ++ return 1; ++ } ++ ++ o = gcref(o->gch.nextgc); ++ } ++ ++ if (fclose(fp)) { ++ *err = strerror(errno); ++ return 1; ++ } ++ ++ return 0; ++} ++ ++#endif /* LUAJIT_DISABLE_KPROF */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.h b/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.h +new file mode 100644 +index 00000000..03846eec +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_gcsnapshot.h +@@ -0,0 +1,11 @@ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++#ifndef _LJ_GCSNAPSHOT_H ++#define _LJ_GCSNAPSHOT_H ++ ++#include "lj_def.h" ++ ++LJ_FUNC int gc_snapshot(lua_State *L, const char* filename, const char** err, unsigned int timeout); ++ ++#endif /* _LJ_GCSNAPSHOT_H */ ++#endif /* LUAJIT_DISABLE_KPROF */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_ir.c b/bundle/LuaJIT-2.1-20210510/src/lj_ir.c +index 4be11a61..eeca48e0 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_ir.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_ir.c +@@ -32,6 +32,7 @@ + #include "lj_strscan.h" + #include "lj_strfmt.h" + #include "lj_prng.h" ++#include "lj_mem_tracing.h" + + /* Some local macros to save typing. Undef'd at the end. */ + #define IR(ref) (&J->cur.ir[(ref)]) +@@ -74,11 +75,17 @@ void LJ_FASTCALL lj_ir_growtop(jit_State *J) + IRIns *baseir = J->irbuf + J->irbotlim; + MSize szins = J->irtoplim - J->irbotlim; + if (szins) { ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + baseir = (IRIns *)lj_mem_realloc(J->L, baseir, szins*sizeof(IRIns), + 2*szins*sizeof(IRIns)); ++ lj_mem_tracing_ir_probe(J2G(J), szins*sizeof(IRIns), 2*szins*sizeof(IRIns)); ++ + J->irtoplim = J->irbotlim + 2*szins; + } else { ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + baseir = (IRIns *)lj_mem_realloc(J->L, NULL, 0, LJ_MIN_IRSZ*sizeof(IRIns)); ++ lj_mem_tracing_ir_probe(J2G(J), 0, LJ_MIN_IRSZ*sizeof(IRIns)); ++ + J->irbotlim = REF_BASE - LJ_MIN_IRSZ/4; + J->irtoplim = J->irbotlim + LJ_MIN_IRSZ; + } +@@ -101,11 +108,17 @@ static void lj_ir_growbot(jit_State *J) + J->irtoplim -= ofs; + J->cur.ir = J->irbuf = baseir - J->irbotlim; + } else { ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + /* Double the buffer size, but split the growth amongst top/bottom. */ + IRIns *newbase = lj_mem_newt(J->L, 2*szins*sizeof(IRIns), IRIns); ++ lj_mem_tracing_ir_probe(J2G(J), 0, 2*szins*sizeof(IRIns)); ++ + MSize ofs = szins >= 256 ? 128 : (szins >> 1); /* Limit bottom growth. */ + memcpy(newbase + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns)); +- lj_mem_free(G(J->L), baseir, szins*sizeof(IRIns)); ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); ++ lj_mem_free(J2G(J), baseir, szins*sizeof(IRIns)); ++ lj_mem_tracing_ir_probe(J2G(J), szins*sizeof(IRIns), 0); ++ + J->irbotlim -= ofs; + J->irtoplim = J->irbotlim + 2*szins; + J->cur.ir = J->irbuf = newbase - J->irbotlim; +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_mbuf.h b/bundle/LuaJIT-2.1-20210510/src/lj_mbuf.h +new file mode 100644 +index 00000000..e0317edf +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_mbuf.h +@@ -0,0 +1,170 @@ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++#ifndef _LJ_MBUF_H ++#define _LJ_MBUF_H ++ ++#include ++#include ++#include ++#include ++#include ++#include "lj_def.h" ++ ++ ++/* ++** A buffer based on mmap. ++*/ ++ ++ ++#define MBUF_SYNC_FLUSH (0) ++#define MBUF_ASYNC_FLUSH (1) ++ ++ ++typedef struct MBuf { ++ uint8_t* b; /* Base of buffer. */ ++ uint8_t* p; /* Current position. */ ++ uint8_t* e; /* End of buffer, [b, e) */ ++ uint8_t i; /* Is it initialized? */ ++ char* f; /* Filename. */ ++} MBuf; ++ ++static LJ_AINLINE int lj_mbuf_init(MBuf* b, const char* f, size_t max_size, char** err) { ++ lj_assertX(b != NULL, "lj_mbuf_init: b is NULL"); ++ lj_assertX(f != NULL, "lj_mbuf_init: filename is NULL"); ++ lj_assertX(max_size != 0, "lj_mbuf_init: max_size is 0"); ++ ++ int fd = -1; ++ void* p = MAP_FAILED; ++ b->f = NULL; ++ ++ fd = open(f, O_RDWR | O_CREAT | O_TRUNC, 0644); ++ if (fd == -1) { ++ *err = strerror(errno); ++ goto on_error; ++ } ++ ++ p = mmap(NULL, max_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); ++ if (p == MAP_FAILED) { ++ *err = strerror(errno); ++ goto on_error; ++ } ++ ++ if (ftruncate(fd, max_size) == -1) { ++ *err = strerror(errno); ++ goto on_error; ++ } ++ ++ if (close(fd) == -1) { ++ *err = strerror(errno); ++ goto on_error; ++ } ++ ++ b->b = (uint8_t*)p; ++ b->p = b->b; ++ b->e = b->b + max_size; ++ b->i = 1; ++ ++ size_t len = strlen(f); ++ ++ b->f = malloc(len + 1); ++ if (b->f == NULL) { ++ *err = "no memory"; ++ goto on_error; ++ } ++ ++ memcpy(b->f, f, len); ++ b->f[len] = '\0'; ++ ++ return 0; ++ ++on_error: ++ if (fd != -1) close(fd); ++ if (p != MAP_FAILED) munmap(p, max_size); ++ if (b->f != NULL) { ++ free(b->f); ++ b->f = NULL; ++ } ++ ++ return 1; ++} ++ ++static LJ_AINLINE int lj_mbuf_putmem(MBuf* b, const void* p, size_t n) { ++ lj_assertX(b != NULL, "lj_mbuf_putmem: b is NULL"); ++ lj_assertX(b->i == 1, "lj_mbuf_putmem: b is not initialized"); ++ lj_assertX(p != NULL, "lj_mbuf_putmem: p is NULL"); ++ lj_assertX(n != 0, "lj_mbuf_putmem: n is 0"); ++ ++ if (b->p + n > b->e) return 1; ++ ++ memcpy(b->p, p, n); ++ b->p += n; ++ ++ return 0; ++} ++ ++static LJ_AINLINE int lj_mbuf_flush(MBuf* b, int is_async) { ++ lj_assertX(b != NULL, "lj_mbuf_flush: b is NULL"); ++ lj_assertX(b->i == 1, "lj_mbuf_flush: b is not initialized"); ++ if (msync(b->b, ++ b->p - b->b, ++ is_async ? MS_ASYNC : MS_SYNC) == -1) { ++ return 1; ++ } ++ return 0; ++} ++ ++ ++static LJ_AINLINE size_t lj_mbuf_left(MBuf* b) { ++ lj_assertX(b != NULL, "lj_mbuf_left: b is NULL"); ++ lj_assertX(b->i == 1, "lj_mbuf_left: b is not initialized"); ++ return b->e - b->p; ++} ++ ++static LJ_AINLINE int lj_mbuf_free(MBuf* b, char** err) { ++ lj_assertX(b != NULL, "lj_mbuf_free: b is NULL"); ++ lj_assertX(b->i == 1, "lj_mbuf_free: b is not initialized"); ++ ++ int rc = 0; ++ ++ if (munmap(b->b, b->e - b->b) == -1) { ++ *err = strerror(errno); ++ rc = 1; ++ goto free_filename; ++ } ++ ++ int fd = open(b->f, O_RDWR); ++ if (fd == -1) { ++ *err = strerror(errno); ++ rc = 1; ++ goto free_filename; ++ } ++ ++ if (ftruncate(fd, b->p - b->b) == -1) { ++ *err = strerror(errno); ++ close(fd); ++ rc = 1; ++ goto free_filename; ++ } ++ ++ if (close(fd) == -1) { ++ *err = strerror(errno); ++ rc = 1; ++ goto free_filename; ++ } ++ ++free_filename: ++ free(b->f); ++ b->f = NULL; ++ b->i = 0; ++ ++ return rc; ++} ++ ++static LJ_AINLINE int lj_mbuf_is_initialized(MBuf* b) { ++ lj_assertX(b != NULL, "lj_mbuf_is_initialized: b is NULL"); ++ return b->i; ++} ++ ++ ++#endif /* _LJ_MBUF_H */ ++#endif /* LUAJIT_DISABLE_KPROF */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_mcode.c b/bundle/LuaJIT-2.1-20210510/src/lj_mcode.c +index 9c859d26..bccae944 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_mcode.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_mcode.c +@@ -19,6 +19,7 @@ + #if LJ_HASJIT || LJ_HASFFI + #include "lj_vm.h" + #endif ++#include "lj_mem_tracing.h" + + /* -- OS-specific functions ----------------------------------------------- */ + +@@ -105,6 +106,9 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) + if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL); + p = NULL; + } ++ ++ lj_mem_tracing_mcode_probe(J2G(J), 0, sz); ++ + return p; + } + +@@ -112,6 +116,7 @@ static void mcode_free(jit_State *J, void *p, size_t sz) + { + UNUSED(J); + munmap(p, sz); ++ lj_mem_tracing_mcode_probe(J2G(J), sz, 0); + } + + static int mcode_setprot(void *p, size_t sz, int prot) +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.c b/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.c +new file mode 100644 +index 00000000..48e85182 +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.c +@@ -0,0 +1,888 @@ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++#define lj_mem_tracing_c ++#define LUA_CORE ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "lauxlib.h" ++#include "lj_mem_tracing.h" ++#include "lj_buf.h" ++#include "lj_debug.h" ++#include "lj_dispatch.h" ++#include "lj_trace.h" ++#include "lj_mbuf.h" ++#include "lj_frame.h" ++ ++ ++#define SBUF_INIT_SIZE 4096 ++ ++/* ++** A hard code from the spec of msgpack, ++** this is the type hint for the `NIL` value in msgpack. ++** ++** ``` ++** MessagePack specification ++** Last modified at 2017-08-09 22:42:07 -0700 ++** Sadayuki Furuhashi © 2013-04-21 21:52:33 -0700 ++** ``` ++*/ ++#define MSG_PACK_NIL 0xc0 ++ ++ ++/* -- size of each type of msgpack object --------------------------------- */ ++ ++#define MSGPACK_NIL_SIZE 1 ++#define MSGPACK_UINT8_SIZE 2 ++#define MSGPACK_UINT16_SIZE 3 ++#define MSGPACK_UINT32_SIZE 5 ++#define MSGPACK_UINT64_SIZE 9 ++#define MSGPACK_INT8_SIZE 2 ++#define MSGPACK_INT16_SIZE 3 ++#define MSGPACK_INT32_SIZE 5 ++#define MSGPACK_INT64_SIZE 9 ++#define MSGPACK_STR_HEADER_MAX_SIZE 5 ++ ++/* -- marker of each event ------------------------------------------------ */ ++ ++#define EVENT_ALLOC 0 ++#define EVENT_FREE 1 ++#define EVENT_REALLOC 2 ++#define EVENT_SYMBOL 3 ++#define EVENT_TRACE 4 ++#define EVENT_INIT 5 ++#define EVENT_IR 6 ++#define EVENT_JIT_TRACE 7 ++#define EVENT_SNAPSHOT 8 ++#define EVENT_MCODE 9 ++#define EVENT_LAST EVENT_MCODE ++ ++#if EVENT_LAST > UINT8_MAX ++#error "event type is too large" ++#endif ++ ++/* -- common macros ------------------------------------------------------- */ ++ ++#define PROBE_GUARD_ERROR() do { \ ++ if (LJ_UNLIKELY(g_error)) { \ ++ lj_mem_tracing_stop(NULL); \ ++ return; \ ++ } \ ++} while (0) ++ ++#define PROBE_GUARD_TIMEOUT() do { \ ++ if (LJ_UNLIKELY(time(NULL) > g_timeout_at)) { \ ++ lj_mem_tracing_stop(NULL); \ ++ return; \ ++ } \ ++} while (0) ++ ++#define PROBE_GUARD_SKIP() do { \ ++ if (LJ_UNLIKELY(g_skip_probes > 0)) { \ ++ g_skip_probes--; \ ++ return; \ ++ } \ ++} while (0) ++ ++#define PROBE_GUARD_MEMORY_OPS() do { \ ++ PROBE_GUARD_ERROR(); \ ++ PROBE_GUARD_TIMEOUT(); \ ++ PROBE_GUARD_SKIP(); \ ++} while (0) ++ ++#define PROBE_GUARD_NON_MEMORY_OPS() do { \ ++ PROBE_GUARD_ERROR(); \ ++ PROBE_GUARD_TIMEOUT(); \ ++} while (0) ++ ++ ++/* -- global states ------------------------------------------------------- */ ++ ++int g_lj_mem_tracing_is_enabled = 0; ++global_State* g_lj_mem_tracing_G = NULL; /* the target Lua VM */ ++static time_t g_timeout_at = 0; ++static size_t g_block_size = 0; /* mmap-based buffer size */ ++static char* g_filename = NULL; /* mmap-based buffer filename pattern */ ++static uint32_t g_max_stack_depth = 0; /* max stack depth */ ++static msgpack_packer g_packer; /* msgpack packer */ ++ ++ ++/* ++** g_sbuf is a string buffer used to store the symbol name temporarily, ++** and avoid allocating/freeing memory during the tracing. ++*/ ++static SBuf g_sbuf; ++ ++/* ++** g_L is a new Lua VM outside the memory tracer, ++** which will allow us to use some utility functions of Lua, ++** like lua_pushfstring(). ++*/ ++static lua_State* g_L = NULL; ++ ++/* ++** double-buffering for mmap-based buffer ++** to avoid the overhead of File I/O. ++** g_mbufs[1] is the current buffer, ++** g_mbufs[0] is the buffer to be flushed. ++*/ ++static MBuf g_mbufs[2]; ++ ++/* ++** g_protos is an array of GCproto pointers, ++** the length of the array is `g_max_stack_depth`, ++** it is used to store the protos of the current trace ++** from the top (g_protos[0]) of the stack to the bottom of the stack. ++** it will be updated if call/return events are triggered. ++*/ ++static GCproto** g_protos = NULL; ++ ++/* ++** g_block_id is used to generate the filename of the mmap-based buffer, ++** the filename will be like: -.bin, ++** and the g_block_id will be increased by 1 after each flush. ++** starts from 0. ++*/ ++static int g_block_id = 0; ++ ++/* ++** if g_error is not NULL, ++** it means there is an error when writing to the buffer, ++** and the error string will be pushed into the top of the stack of `g_L`. ++*/ ++static char* g_error = NULL; ++ ++ ++/* ++** g_skip_probes is used to skip some n times of probes. ++** I insert `lj_mem_tracing_probe` into the following functions: ++** - lj_mem_realloc() ++** - lj_mem_free() ++** - lj_mem_newgco() ++** Those functions will not only be called when creating a new Lua object, ++** but also be called for some internal data structures, ++** like: ++** - IRIns ++** - SnapShot ++** - GCTrace ++** - MCode ++** I want to identify those internal memory operations for ++** those data structures, so I need to skip some probes ++** as those memory management functions likely calls the ++** above functions to allocate/free memory for those data structures. ++*/ ++static size_t g_skip_probes = 0; ++ ++ ++/* -- helper functions ---------------------------------------------------- */ ++ ++/* ++** msgpack buffer writer callback for mmap-based buffer ++*/ ++static int _mbuf_writer(void* data, const char* buf, size_t len); ++ ++/* ++** switch the current buffer to the other one ++*/ ++static void _switch_buffer(); ++ ++/* ++** msgpack packer for uint8_t, uint16_t, uint32_t, and uint64_t, ++** which will be packed as fixuint8, fixuint16, fixuint32, and fixuint64 ++** respectively depending on the value of x. ++*/ ++static void _msgpack_pack_uintx(uint64_t x); ++ ++/* ++** msgpack packer for int8_t, int16_t, int32_t, and int64_t, ++** which will be packed as fixint8, fixint16, fixint32, and fixint64 ++** respectively depending on the value of x. ++*/ ++static void _msgpack_pack_intx(int64_t x); ++ ++ ++/* -- event handlers ------------------------------------------------------ */ ++ ++/* ++** write trace event to the buffer, ++** the trace event has some protos, each proto is a pointer to GCproto, ++** which is a uint64_t value. ++*/ ++static void _write_trace_event(); ++ ++/* ++** write symbol event to the buffer, ++** the symbol event has 3 elements, ++** the first element is the EVENT_SYMBOL, ++** the second element is the pointer to the symbol, ++** the third element is the symbol name (like /path/to/init.lua:12), ++*/ ++static void _write_symbol_event(void *p, const char* name, size_t len); ++ ++/* ++** write alloc event to the buffer, ++** the alloc event has 3 elements, ++** the first element is the EVENT_ALLOC, ++** the second element is the pointer to the allocated memory, ++** the third element is the size of the allocated memory, ++*/ ++static void _write_alloc_event(void* p, size_t sz); ++ ++/* ++** write free event to the buffer, ++** the free event has 3 elements, ++** the first element is the EVENT_FREE, ++** the second element is the pointer to the freed memory, ++** the third element is the size of the freed memory, ++*/ ++static void _write_free_event(void* p, size_t sz); ++ ++/* ++** write realloc event to the buffer, ++** the realloc event has 4 elements, ++** the first element is the EVENT_REALLOC, ++** the second element is the pointer to the old memory, ++** the third element is the pointer to the new memory, ++** the fourth element is the size of the old memory, ++** the fifth element is the size of the new memory, ++*/ ++static void _write_realloc_event(void* op, void* np, size_t oz, size_t nz); ++ ++/* ++** write ir event to the buffer, ++** the ir event has 2 elements, ++** the first element is the EVENT_IR, ++** the second element is the delta of the memory usage of IRs. ++*/ ++static void _write_ir_event(int64_t delta); ++ ++/* ++** write jit trace event to the buffer, ++** the jit trace event has 2 elements, ++** the first element is the EVENT_JIT_TRACE, ++** the second element is the delta of the memory usage of JIT traces. ++*/ ++static void _write_jit_trace_event(int64_t delta); ++ ++/* ++** write jit snapshot event to the buffer, ++** the jit snapshot event has 2 elements, ++** the first element is the EVENT_JIT_SNAPSHOT, ++** the second element is the delta of the memory usage of snapshots. ++*/ ++static void _write_snapshot_event(int64_t delta); ++ ++/* ++** write jit mcode event to the buffer, ++** the jit mcode event has 2 elements, ++** the first element is the EVENT_JIT_MCODE, ++** the second element is the delta of the memory usage of machine code. ++*/ ++static void _write_mcode_event(size_t delta); ++ ++/* ++** write init event to the buffer, ++** the init event has 5 elements, ++** the first element is the EVENT_INIT, ++** the second element is the major version of the tracing data, ++** the third element is the minor version of the tracing data, ++** the fourth element is the patch version of the tracing data, ++** the fifth element is the `g_max_stack_depth` ++*/ ++static void _write_init_event(lua_State* L); ++ ++static void _append_proto(GCproto *pt); ++static void _dump_symbol_table(lua_State *L); ++ ++static void _enable_hook_for_call_return(lua_State* L); ++static void _disable_hook_for_call_return(lua_State* L); ++ ++ ++static int _mbuf_writer(void* data, const char* buf, size_t len) { ++ UNUSED(data); ++ ++ if (LJ_UNLIKELY(lj_mbuf_putmem(&g_mbufs[1], buf, len))) { ++ g_error = "failed to write to buffer (even if there is enough space)"; ++ return 0; ++ } ++ ++ return len; ++} ++ ++ ++static void _switch_buffer() { ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < 1)) { ++ g_error = "buffer is unexpectedly full, can't write NIL to indicate the end of buffer"; ++ return; ++ } ++ ++ char tmp = MSG_PACK_NIL; ++ if (LJ_UNLIKELY(lj_mbuf_putmem(&g_mbufs[1], &tmp, 1))) { ++ /* ++ ** if the buffer can't save a complete event, ++ ** we should write a NIL to indicate the end of buffer ++ ** for analysis tools. ++ */ ++ g_error = "failed to write NIL to buffer as the end of buffer"; ++ return; ++ } ++ ++ /* schedule async flush */ ++ if (lj_mbuf_flush(&g_mbufs[1], MBUF_ASYNC_FLUSH)) { ++ g_error = "failed to schedule flush to file system"; ++ } ++ ++ if (lj_mbuf_is_initialized(&g_mbufs[0])) { ++ /* flush to file system synchronously */ ++ lj_mbuf_free(&g_mbufs[0], &g_error); ++ } ++ ++ g_mbufs[0] = g_mbufs[1]; ++ ++ /* ++ ** initialize a new buffer, ++ ** the file name is -.bin, ++ */ ++ lua_pushfstring(g_L, "%s-%d.bin", g_filename, g_block_id++); ++ lj_mbuf_init(&g_mbufs[1], lua_tostring(g_L, -1), g_block_size, &g_error); ++ lua_pop(g_L, 1); ++} ++ ++ ++static void _msgpack_pack_uintx(uint64_t x) { ++ if (x == 0) { ++ msgpack_pack_nil(&g_packer); ++ } else if (x <= UINT8_MAX) { ++ msgpack_pack_fix_uint8(&g_packer, (uint8_t)x); ++ } else if (x <= UINT16_MAX) { ++ msgpack_pack_fix_uint16(&g_packer, (uint16_t)x); ++ } else if (x <= UINT32_MAX) { ++ msgpack_pack_fix_uint32(&g_packer, (uint32_t)x); ++ } else { ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)x); ++ } ++} ++ ++ ++static void _msgpack_pack_intx(int64_t x) { ++ if (x == 0) { ++ msgpack_pack_nil(&g_packer); ++ } else if (x <= INT8_MAX) { ++ msgpack_pack_fix_int8(&g_packer, (int8_t)x); ++ } else if (x <= INT16_MAX) { ++ msgpack_pack_fix_int16(&g_packer, (int16_t)x); ++ } else if (x <= INT32_MAX) { ++ msgpack_pack_fix_int32(&g_packer, (int32_t)x); ++ } else { ++ msgpack_pack_fix_int64(&g_packer, (int64_t)x); ++ } ++} ++ ++ ++static void _write_trace_event() { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ g_max_stack_depth * MSGPACK_UINT64_SIZE; ++ int i; ++ ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_TRACE); ++ ++ for (i = 0; i < g_max_stack_depth; i++) { ++ if (g_protos[i] == NULL) msgpack_pack_nil(&g_packer); ++ else msgpack_pack_fix_uint64(&g_packer, (uint64_t)g_protos[i]); ++ } ++} ++ ++ ++static void _write_symbol_event(void *p, const char* name, size_t len) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_STR_HEADER_MAX_SIZE + ++ len; ++ ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_SYMBOL); ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)p); ++ msgpack_pack_str(&g_packer, len); ++ msgpack_pack_str_body(&g_packer, name, len); ++} ++ ++ ++static void _write_alloc_event(void* p, size_t sz) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE; ++ ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_ALLOC); ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)p); ++ _msgpack_pack_uintx(sz); ++} ++ ++ ++static void _write_free_event(void* p, size_t sz) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE; ++ ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_FREE); ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)p); ++ _msgpack_pack_uintx(sz); ++} ++ ++ ++static void _write_realloc_event(void* op, void* np, size_t oz, size_t nz) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE; ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_REALLOC); ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)op); ++ msgpack_pack_fix_uint64(&g_packer, (uint64_t)np); ++ _msgpack_pack_uintx(oz); ++ _msgpack_pack_uintx(nz); ++} ++ ++ ++static void _write_ir_event(int64_t delta) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_INT64_SIZE; ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_IR); ++ _msgpack_pack_intx(delta); ++} ++ ++ ++static void _write_jit_trace_event(int64_t delta) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_INT64_SIZE; ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_JIT_TRACE); ++ _msgpack_pack_intx(delta); ++} ++ ++ ++void _write_snapshot_event(int64_t delta) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_INT64_SIZE; ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_SNAPSHOT); ++ _msgpack_pack_intx(delta); ++} ++ ++ ++static void _write_mcode_event(size_t delta) { ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_INT64_SIZE; ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ _switch_buffer(); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_MCODE); ++ _msgpack_pack_intx(delta); ++} ++ ++ ++static void _write_init_event(lua_State *L) ++{ ++ size_t event_max_size = MSGPACK_UINT8_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE + ++ MSGPACK_UINT64_SIZE; ++ ++ if (LJ_UNLIKELY(lj_mbuf_left(&g_mbufs[1]) < event_max_size)) { ++ luaL_error(L, "not enough space in the buffer"); ++ } ++ ++ msgpack_pack_fix_uint8(&g_packer, EVENT_INIT); ++ msgpack_pack_fix_uint64(&g_packer, 1); ++ msgpack_pack_fix_uint64(&g_packer, 0); ++ msgpack_pack_fix_uint64(&g_packer, 0); ++ msgpack_pack_fix_uint64(&g_packer, g_max_stack_depth); ++} ++ ++ ++static void _append_proto(GCproto *pt) { ++ /* ++ ** C-style string like the `:16` in `init.lua:16` ++ ** ^^^ ++ */ ++ char firstline[64]; ++ ++ lj_buf_reset(&g_sbuf); ++ ++ if (lj_debug_putchunkname(&g_sbuf, pt, 0)) { ++ int len = snprintf(firstline, ++ sizeof(firstline) / sizeof(char), ++ ":%" PRIu32 ";", ++ (uint32_t)(pt->firstline)); ++ ++ lj_assertX(len > 0, "snprintf failed (no data written)"); ++ lj_assertX(len < sizeof(firstline) / sizeof(char), "snprintf failed (buffer overflow)"); ++ ++ lj_buf_putmem(&g_sbuf, firstline, len); ++ } ++ ++ _write_symbol_event(pt, mref(g_sbuf.b, char), sbuflen(&g_sbuf)); ++} ++ ++ ++static void _dump_symbol_table(lua_State *L) { ++ global_State *g = G(L); ++ GCobj* o = gcref(g->gc.root); ++ ++ _write_symbol_event(0, "[unknown];", sizeof("[unknown];") - 1); ++ ++ while (o != NULL) { ++ if (o->gch.gct == ~LJ_TPROTO) { ++ _append_proto(gco2pt(o)); ++ } ++ ++ o = gcref(o->gch.nextgc); ++ } ++} ++ ++ ++static void _enable_hook_for_call_return(lua_State* L) { ++ G(L)->hookmask = (uint8_t)((G(L)->hookmask & ~HOOK_EVENTMASK) | ++ ((LUA_MASKCALL | LUA_MASKRET) & HOOK_EVENTMASK)); ++ lj_trace_abort(G(L)); ++ lj_dispatch_update(G(L)); ++} ++ ++ ++static void _disable_hook_for_call_return(lua_State* L) { ++ g_lj_mem_tracing_G->hookmask = (uint8_t)((g_lj_mem_tracing_G->hookmask & ~HOOK_EVENTMASK) | 0); ++ lj_trace_abort(g_lj_mem_tracing_G); ++ lj_dispatch_update(g_lj_mem_tracing_G); ++} ++ ++ ++int lj_mem_tracing_start(lua_State *L, ++ const char* filename, ++ size_t block_size, ++ uint32_t max_stack_depth, ++ unsigned int timeout) { ++ if (g_lj_mem_tracing_is_enabled) { ++ lua_pushstring(L, "tracing is already enabled for this Lua VM or another Lua VM"); ++ return 1; ++ } ++ ++ g_filename = NULL; ++ g_protos = NULL; ++ g_L = NULL; ++ ++ size_t len = strlen(filename); ++ g_filename = (char*)malloc(len + 1); ++ if (g_filename == NULL) { ++ lua_pushstring(L, "failed to allocate memory for filename"); ++ goto on_error; ++ } ++ strcpy(g_filename, filename); ++ ++ g_L = luaL_newstate(); ++ if (g_L == NULL) { ++ lua_pushstring(L, "failed to create new state for tracing"); ++ goto on_error; ++ } ++ ++ lua_pushfstring(g_L, "%s-%d.bin", g_filename, 0); ++ char* err = NULL; ++ if (lj_mbuf_init(&g_mbufs[1], lua_tostring(g_L, -1), block_size, &err)) { ++ lua_pushfstring(L, "failed to open file %s: %s", lua_tostring(g_L, -1), err); ++ goto on_error; ++ } ++ lua_pop(g_L, 1); ++ ++ g_protos = calloc(max_stack_depth, sizeof(GCproto*)); ++ if (g_protos == NULL) { ++ lua_pushstring(L, "failed to allocate memory for protos"); ++ goto on_error; ++ } ++ ++ lj_buf_init(g_L, &g_sbuf); ++ lj_buf_reset(&g_sbuf); ++ lj_buf_need(&g_sbuf, SBUF_INIT_SIZE); ++ ++ msgpack_packer_init(&g_packer, NULL, _mbuf_writer); ++ ++ _enable_hook_for_call_return(L); ++ ++ g_lj_mem_tracing_G = G(L); ++ g_timeout_at = time(NULL) + timeout; ++ g_max_stack_depth = max_stack_depth; ++ g_block_size = block_size; ++ g_block_id = 1; ++ g_error = NULL; ++ g_skip_probes = 0; ++ ++ _write_init_event(L); ++ _dump_symbol_table(L); ++ ++ /* ++ ** we must set g_lj_mem_tracing_is_enabled to 1 last, ++ ** because some Lua Objects will be ++ ** allocated during the initialization. ++ */ ++ g_lj_mem_tracing_is_enabled = 1; ++ return 0; ++ ++on_error: ++ if (filename != NULL) free(g_filename); ++ if (g_L != NULL) lua_close(g_L); ++ if (lj_mbuf_is_initialized(&g_mbufs[1])) lj_mbuf_free(&g_mbufs[1], &err); ++ if (g_protos != NULL) free(g_protos); ++ return 1; ++} ++ ++ ++int lj_mem_tracing_stop(lua_State *L) { ++ int i; ++ ++ if (!g_lj_mem_tracing_is_enabled) { ++ if (L && g_error) { ++ lua_pushstring(L, g_error); ++ return 1; ++ } ++ ++ return 0; ++ } ++ ++ /* number of error strings */ ++ int error_strs = 0; ++ ++ g_lj_mem_tracing_is_enabled = 0; ++ _disable_hook_for_call_return(L); ++ ++ if (g_error && L) { ++ lua_pushstring(L, g_error); ++ error_strs++; ++ } ++ ++ for (i = 0; i < sizeof(g_mbufs) / sizeof(g_mbufs[0]); i++) { ++ if (lj_mbuf_is_initialized(&g_mbufs[i]) && lj_mbuf_free(&g_mbufs[i], &g_error)) { ++ if (L) { ++ /* `\n` is for separating multiple error messages */ ++ lua_pushfstring(L, "\nfailed to free mbuf %d: %s", i, g_error); ++ } ++ ++ error_strs++; ++ } ++ } ++ ++ if (error_strs >= 1 && L) { ++ /* concat all error messages into multi-line string */ ++ lua_concat(L, error_strs); ++ } ++ ++ lj_buf_free(G(g_L), &g_sbuf); ++ ++ lua_close(g_L); ++ g_L = NULL; ++ ++ free(g_filename); ++ g_filename = NULL; ++ ++ free(g_protos); ++ g_protos = NULL; ++ ++ g_lj_mem_tracing_G = NULL; ++ g_timeout_at = 0; ++ g_max_stack_depth = 0; ++ g_block_size = 0; ++ g_skip_probes = 0; ++ ++ return g_error ? 1 : 0; ++} ++ ++ ++int lj_mem_tracing_status(lua_State *L) { ++ lua_createtable(L, 0, 6); ++ ++ lua_pushstring(L, "enabled"); ++ lua_pushboolean(L, g_lj_mem_tracing_is_enabled); ++ lua_rawset(L, -3); ++ ++ if (g_lj_mem_tracing_is_enabled) { ++ lj_assertL(g_filename != NULL, "filename is NULL"); ++ lj_assertL(g_block_size > 0, "block_size is not positive"); ++ lj_assertL(g_max_stack_depth > 0, "max_stack_depth is not positive"); ++ lj_assertL(g_timeout_at > 0, "timeout_at is not positive"); ++ ++ lua_pushstring(L, "filename"); ++ lua_pushfstring(L, "%s-.bin", g_filename); ++ lua_rawset(L, -3); ++ ++ lua_pushstring(L, "block_size"); ++ lua_pushinteger(L, g_block_size); ++ lua_rawset(L, -3); ++ ++ lua_pushstring(L, "max_stack_depth"); ++ lua_pushinteger(L, g_max_stack_depth); ++ lua_rawset(L, -3); ++ ++ lua_pushstring(L, "timeout_at"); ++ lua_pushinteger(L, g_timeout_at); ++ lua_rawset(L, -3); ++ } ++ ++ if (g_error) { ++ lua_pushstring(L, "error"); ++ lua_pushstring(L, g_error); ++ lua_rawset(L, -3); ++ } ++ ++ return 1; ++} ++ ++ ++int lj_mem_tracing_is_enabled_(lua_State* L) { ++ if (LJ_UNLIKELY(g_lj_mem_tracing_G != G(L))) return 0; ++ return g_lj_mem_tracing_is_enabled; ++} ++ ++ ++void lj_mem_tracing_probe_(global_State* g, void* op, void* np, size_t osize, size_t nsize) { ++ PROBE_GUARD_MEMORY_OPS(); ++ ++ if (osize == 0 && nsize > 0) { ++ lj_assertX(op == NULL && np != NULL, ++ "expected alloc a new pointer: op=%p, np=%p, osize=%zu, nsize=%zu", op, np, osize, nsize); ++ _write_alloc_event(np, nsize); ++ ++ } else if (osize > 0 && nsize > 0) { ++ lj_assertX(op != NULL && np != NULL, ++ "expected realloc a pointer: op=%p, np=%p, osize=%zu, nsize=%zu", op, np, osize, nsize); ++ _write_realloc_event(op, np, osize, nsize); ++ ++ } else if (osize > 0 && nsize == 0) { ++ lj_assertX(op != NULL && np == NULL, ++ "expected free a pointer: op=%p, np=%p, osize=%zu, nsize=%zu", op, np, osize, nsize); ++ _write_free_event(op, osize); ++ ++ } else if (osize == 0 && nsize == 0) { ++ /* free a NULL pointer is safe for `lj_mem_free()` and `lua_Alloc` */ ++ lj_assertX(op == NULL && np == NULL, ++ "expected free a NULL pointer: op=%p, np=%p, osize=%zu, nsize=%zu", op, np, osize, nsize); ++ ++ } else { ++ lj_assertX(0, "invalid memory operation: osize=%zu, nsize=%zu", osize, nsize); ++ } ++} ++ ++ ++void lj_mem_tracing_ir_probe_(global_State* g, size_t osize, size_t nsize) { ++ PROBE_GUARD_MEMORY_OPS(); ++ _write_ir_event((int64_t)nsize - (int64_t)osize); ++} ++ ++ ++void lj_mem_tracing_jit_trace_probe_(global_State* g, size_t osize, size_t nsize) { ++ PROBE_GUARD_MEMORY_OPS(); ++ _write_jit_trace_event((int64_t)nsize - (int64_t)osize); ++} ++ ++ ++void lj_mem_tracing_snapshot_probe_(global_State* g, size_t osize, size_t nsize) { ++ PROBE_GUARD_MEMORY_OPS(); ++ _write_snapshot_event((int64_t)nsize - (int64_t)osize); ++} ++ ++ ++void lj_mem_tracing_mcode_probe_(global_State* g, size_t osize, size_t nsize) { ++ PROBE_GUARD_MEMORY_OPS(); ++ _write_mcode_event((int64_t)nsize - (int64_t)osize); ++} ++ ++ ++void lj_mem_tracing_skip_nprobes_(global_State* g, size_t cnt) { ++ PROBE_GUARD_NON_MEMORY_OPS(); ++ g_skip_probes += cnt; ++} ++ ++ ++void lj_mem_tracing_on_call_return_(lua_State *L) { ++ PROBE_GUARD_NON_MEMORY_OPS(); ++ ++ /* Basically a copy of `lj_debug_frame()` in `src/lj_debug.c` */ ++ ++ /* ++ ** Although it is basically a copy, ++ ** I have made a few changes for performance. ++ */ ++ ++ cTValue *frame, *bot = tvref(L->stack)+LJ_FR2; ++ int idx = 0, level = 0; ++ ++ for (frame = L->base - 1; frame > bot && idx < g_max_stack_depth; ) { ++ if (frame_gc(frame) == obj2gco(L)) level++; ++ ++ if (level-- == 0) { ++ GCfunc* fn = frame_func(frame); ++ ++ if (isluafunc(fn)) { ++ GCproto* pt = funcproto(fn); ++ g_protos[idx++] = pt; ++ ++ } else { ++ g_protos[idx++] = NULL; ++ } ++ ++ level = 0; ++ } ++ ++ if (frame_islua(frame)) { ++ frame = frame_prevl(frame); ++ ++ } else { ++ if (frame_isvarg(frame)) level++; ++ frame = frame_prevd(frame); ++ } ++ } ++ ++ /* The end of the copy of `lj_debug_frame()` in `src/lj_debug.c` */ ++ ++ memset(g_protos + idx, 0, sizeof(GCproto*) * (g_max_stack_depth - idx)); ++ _write_trace_event(); ++} ++ ++ ++void lj_mem_tracing_append_proto_(global_State* g, GCproto* pt) { ++ PROBE_GUARD_NON_MEMORY_OPS(); ++ _append_proto(pt); ++} ++ ++#endif /* LUAJIT_DISABLE_KPROF */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.h b/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.h +new file mode 100644 +index 00000000..2604f6f5 +--- /dev/null ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_mem_tracing.h +@@ -0,0 +1,88 @@ ++#ifndef _LJ_MEM_TRACING_H ++#define _LJ_MEM_TRACING_H ++ ++#ifndef LUAJIT_DISABLE_KPROF ++ ++#include "lj_def.h" ++#include "lj_obj.h" ++ ++/* ++** 0: disabled, 1: enabled. ++** The reason I put this variable here is that I want to make it ++** accessible to the external C code, ++** which means that we can check if memory tracing is enabled ++** in the external C code wihout calling any uninlined function. ++** This is important because we want to make the overhead of ++** memory tracing as small as possible when it is disabled. ++** Avoiding calling is friendly to the CPU pipeline. ++*/ ++extern int g_lj_mem_tracing_is_enabled; ++extern global_State* g_lj_mem_tracing_G; ++ ++int lj_mem_tracing_start(lua_State *L, ++ const char* filename, ++ size_t block_size, ++ uint32_t max_stack_depth, ++ unsigned int timeout); ++LJ_FUNC int lj_mem_tracing_stop(lua_State *L); ++LJ_FUNC int lj_mem_tracing_status(lua_State *L); ++LJ_FUNC void lj_mem_tracing_probe_(global_State* g, void* op, void* np, size_t osize, size_t nsize); ++LJ_FUNC void lj_mem_tracing_ir_probe_(global_State* g, size_t osize, size_t nsize); ++LJ_FUNC void lj_mem_tracing_jit_trace_probe_(global_State* g, size_t osize, size_t nsize); ++LJ_FUNC void lj_mem_tracing_snapshot_probe_(global_State* g, size_t osize, size_t nsize); ++LJ_FUNC void lj_mem_tracing_mcode_probe_(global_State* g, size_t osize, size_t nsize); ++LJ_FUNC void lj_mem_tracing_skip_nprobes_(global_State* g, size_t cnt); ++LJ_FUNC void lj_mem_tracing_on_call_return_(lua_State* L); ++LJ_FUNC void lj_mem_tracing_append_proto_(global_State* g, GCproto* pt); ++ ++#define lj_mem_tracing_is_enabled(g) (g_lj_mem_tracing_is_enabled != 0 && g_lj_mem_tracing_G == (g)) ++ ++#define lj_mem_tracing_probe(g, op, np, osz, nsz) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_probe_(g, op, np, osz, nsz) : 0) ++ ++#define lj_mem_tracing_ir_probe(g, osz, nsz) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_ir_probe_(g, osz, nsz) : 0) ++ ++#define lj_mem_tracing_jit_trace_probe(g, osz, nsz) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_jit_trace_probe_(g, osz, nsz) : 0) ++ ++#define lj_mem_tracing_snapshot_probe(g, osz, nsz) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_snapshot_probe_(g, osz, nsz) : 0) ++ ++#define lj_mem_tracing_mcode_probe(g, osz, nsz) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_mcode_probe_(g, osz, nsz) : 0) ++ ++#define lj_mem_tracing_skip_nprobes(g, cnt) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_skip_nprobes_(g, cnt) : 0) ++ ++#define lj_mem_tracing_on_call_return(L) \ ++ (lj_mem_tracing_is_enabled(G(L)) ? lj_mem_tracing_on_call_return_(L) : 0) ++ ++#define lj_mem_tracing_append_proto(g, pt) \ ++ (lj_mem_tracing_is_enabled(g) ? lj_mem_tracing_append_proto_(g, pt) : 0) ++ ++ ++#else ++ ++/* ++** If memory tracing is disabled, we can define these functions as no-ops. ++** This way, we don't have to add #ifdef/#ifndef to the code that calls them. ++*/ ++ ++#define lj_mem_tracing_probe(g, op, np, osz, nsz) () ++#define lj_mem_tracing_ir_probe(g, osz, nsz) () ++#define lj_mem_tracing_jit_trace_probe(g, osz, nsz) () ++#define lj_mem_tracing_snapshot_probe(g, osz, nsz) () ++#define lj_mem_tracing_mcode_probe(g, osz, nsz) () ++#define lj_mem_tracing_skip_nprobes(g, cnt) () ++ ++/* Always return false as memory tracing is disabled */ ++#define lj_mem_tracing_is_enabled(g) (0) ++ ++#define lj_mem_tracing_on_call_return(L) () ++#define lj_mem_tracing_append_proto(g, pt) () ++ ++#endif /* LUAJIT_DISABLE_KPROF */ ++ ++#endif /* _LJ_MEM_TRACING_H */ ++ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_parse.c b/bundle/LuaJIT-2.1-20210510/src/lj_parse.c +index 27fcf53c..4b11cd0c 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_parse.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_parse.c +@@ -27,6 +27,7 @@ + #include "lj_parse.h" + #include "lj_vm.h" + #include "lj_vmevent.h" ++#include "lj_mem_tracing.h" + + /* -- Parser structures and definitions ----------------------------------- */ + +@@ -1609,6 +1610,9 @@ static GCproto *fs_finish(LexState *ls, BCLine line) + ls->vtop = fs->vbase; /* Reset variable stack. */ + ls->fs = fs->prev; + lj_assertL(ls->fs != NULL || ls->tok == TK_eof, "bad parser state"); ++ ++ lj_mem_tracing_append_proto(G(L), pt); ++ + return pt; + } + +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_profile.c b/bundle/LuaJIT-2.1-20210510/src/lj_profile.c +index 00fdd6da..a79430b0 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_profile.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_profile.c +@@ -60,7 +60,7 @@ typedef struct ProfileState { + luaJIT_profile_callback cb; /* Profiler callback. */ + void *data; /* Profiler callback data. */ + SBuf sb; /* String buffer for stack dumps. */ +- int interval; /* Sample interval in milliseconds. */ ++ int interval; /* Sample interval in microseconds. */ + int samples; /* Number of samples for next callback. */ + int vmstate; /* VM state when profile timer triggered. */ + #if LJ_PROFILE_SIGPROF +@@ -182,8 +182,8 @@ static void profile_timer_start(ProfileState *ps) + int interval = ps->interval; + struct itimerval tm; + struct sigaction sa; +- tm.it_value.tv_sec = tm.it_interval.tv_sec = interval / 1000; +- tm.it_value.tv_usec = tm.it_interval.tv_usec = (interval % 1000) * 1000; ++ tm.it_value.tv_sec = tm.it_interval.tv_sec = interval / (1000 * 1000); ++ tm.it_value.tv_usec = tm.it_interval.tv_usec = interval - (tm.it_value.tv_sec * (1000 * 1000)); + setitimer(ITIMER_PROF, &tm, NULL); + sa.sa_flags = SA_RESTART; + sa.sa_handler = profile_signal; +@@ -209,8 +209,8 @@ static void *profile_thread(ProfileState *ps) + int interval = ps->interval; + #if !LJ_TARGET_PS3 + struct timespec ts; +- ts.tv_sec = interval / 1000; +- ts.tv_nsec = (interval % 1000) * 1000000; ++ ts.tv_sec = interval / (1000 * 1000); ++ ts.tv_nsec = (interval * 1000) - (ts.tv_sec * 1000 * 1000 * 1000); + #endif + while (1) { + #if LJ_TARGET_PS3 +@@ -248,15 +248,15 @@ static DWORD WINAPI profile_thread(void *psx) + ProfileState *ps = (ProfileState *)psx; + int interval = ps->interval; + #if LJ_TARGET_WINDOWS && !LJ_TARGET_UWP +- ps->wmm_tbp(interval); ++ ps->wmm_tbp(interval / 1000); + #endif + while (1) { +- Sleep(interval); ++ Sleep(interval / 1000); + if (ps->abort) break; + profile_trigger(ps); + } + #if LJ_TARGET_WINDOWS && !LJ_TARGET_UWP +- ps->wmm_tep(interval); ++ ps->wmm_tep(interval / 1000); + #endif + return 0; + } +@@ -299,7 +299,7 @@ LUA_API void luaJIT_profile_start(lua_State *L, const char *mode, + luaJIT_profile_callback cb, void *data) + { + ProfileState *ps = &profile_state; +- int interval = LJ_PROFILE_INTERVAL_DEFAULT; ++ int interval = LJ_PROFILE_INTERVAL_DEFAULT * 1000; + while (*mode) { + int m = *mode++; + switch (m) { +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_snap.c b/bundle/LuaJIT-2.1-20210510/src/lj_snap.c +index 21f27e1f..c7fb3654 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_snap.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_snap.c +@@ -25,6 +25,7 @@ + #include "lj_ctype.h" + #include "lj_cdata.h" + #endif ++#include "lj_mem_tracing.h" + + /* Pass IR on to next optimization in chain (FOLD). */ + #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) +@@ -40,7 +41,12 @@ void lj_snap_grow_buf_(jit_State *J, MSize need) + MSize maxsnap = (MSize)J->param[JIT_P_maxsnap]; + if (need > maxsnap) + lj_trace_err(J, LJ_TRERR_SNAPOV); ++ ++ MSize osz = J->sizesnap * sizeof(SnapShot); ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot); ++ lj_mem_tracing_snapshot_probe(J2G(J), osz, J->sizesnap * sizeof(SnapShot)); ++ + J->cur.snap = J->snapbuf; + } + +@@ -51,8 +57,12 @@ void lj_snap_grow_map_(jit_State *J, MSize need) + need = 2*J->sizesnapmap; + else if (need < 64) + need = 64; ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf, + J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry)); ++ lj_mem_tracing_snapshot_probe(J2G(J), J->sizesnapmap*sizeof(SnapEntry), ++ need*sizeof(SnapEntry)); ++ + J->cur.snapmap = J->snapmapbuf; + J->sizesnapmap = need; + } +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_str.c b/bundle/LuaJIT-2.1-20210510/src/lj_str.c +index b314c095..26d4db21 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_str.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_str.c +@@ -13,6 +13,7 @@ + #include "lj_char.h" + #include "lj_prng.h" + ++ + /* -- String helpers ------------------------------------------------------ */ + + /* Ordered compare of strings. Assumes string data is 4-byte aligned. */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_tab.c b/bundle/LuaJIT-2.1-20210510/src/lj_tab.c +index 7dfdd81c..db3cae2b 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_tab.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_tab.c +@@ -16,6 +16,7 @@ + + #include + ++ + /* -- Object hashing ------------------------------------------------------ */ + + /* Hash values are masked with the table hash mask and used as an index. */ +@@ -101,6 +102,7 @@ static LJ_AINLINE void clearapart(GCtab *t) + /* Create a new table. Note: the slots are not initialized (yet). */ + static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) + { ++ void* tmp; + GCtab *t; + /* First try to colocate the array part. */ + if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) { +@@ -137,7 +139,8 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) + if (asize > 0) { + if (asize > LJ_MAX_ASIZE) + lj_err_msg(L, LJ_ERR_TABOV); +- setmref(t->array, lj_mem_newvec(L, asize, TValue)); ++ tmp = lj_mem_newvec(L, asize, TValue); ++ setmref(t->array, tmp); + t->asize = asize; + } + } +@@ -235,14 +238,21 @@ void LJ_FASTCALL lj_tab_clear(GCtab *t) + /* Free a table. */ + void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t) + { +- if (t->hmask > 0) ++ if (t->hmask > 0) { + lj_mem_freevec(g, noderef(t->node), t->hmask+1, Node); +- if (t->asize > 0 && LJ_MAX_COLOSIZE != 0 && t->colo <= 0) ++ } ++ ++ if (t->asize > 0 && LJ_MAX_COLOSIZE != 0 && t->colo <= 0) { + lj_mem_freevec(g, tvref(t->array), t->asize, TValue); +- if (LJ_MAX_COLOSIZE != 0 && t->colo) ++ } ++ ++ if (LJ_MAX_COLOSIZE != 0 && t->colo) { + lj_mem_free(g, t, sizetabcolo((uint32_t)t->colo & 0x7f)); +- else ++ ++ } else { + lj_mem_freet(g, t); ++ } ++ + } + + /* -- Table resizing ------------------------------------------------------ */ +@@ -294,9 +304,11 @@ void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) + if (!tvisnil(&array[i])) + copyTV(L, lj_tab_setinth(L, t, (int32_t)i), &array[i]); + /* Physically shrink only separated arrays. */ +- if (LJ_MAX_COLOSIZE != 0 && t->colo <= 0) ++ if (LJ_MAX_COLOSIZE != 0 && t->colo <= 0) { + setmref(t->array, lj_mem_realloc(L, array, + oldasize*sizeof(TValue), asize*sizeof(TValue))); ++ } ++ + } + if (oldhmask > 0) { /* Reinsert pairs from old hash part. */ + global_State *g; +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_trace.c b/bundle/LuaJIT-2.1-20210510/src/lj_trace.c +index 8085cdf2..f062e9dd 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_trace.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_trace.c +@@ -31,6 +31,7 @@ + #include "lj_vmevent.h" + #include "lj_target.h" + #include "lj_prng.h" ++#include "lj_mem_tracing.h" + + /* -- Error handling ------------------------------------------------------ */ + +@@ -72,7 +73,10 @@ static TraceNo trace_findfree(jit_State *J) + osz = J->sizetrace; + if (osz >= lim) + return 0; /* Too many traces. */ ++ lj_mem_tracing_skip_nprobes(J2G(J), 1); + lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef); ++ lj_mem_tracing_jit_trace_probe(J2G(J), osz * sizeof(GCRef), J->sizetrace * sizeof(GCRef)); ++ + for (; osz < J->sizetrace; osz++) + setgcrefnull(J->trace[osz]); + return J->freetrace; +@@ -127,7 +131,10 @@ GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T) + size_t sz = sztr + szins + + T->nsnap*sizeof(SnapShot) + + T->nsnapmap*sizeof(SnapEntry); ++ lj_mem_tracing_skip_nprobes(G(L), 1); + GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace); ++ lj_mem_tracing_jit_trace_probe(G(L), 0, sz); ++ + char *p = (char *)T2 + sztr; + T2->gct = ~LJ_TTRACE; + T2->marked = 0; +@@ -175,9 +182,12 @@ void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T) + J->freetrace = T->traceno; + setgcrefnull(J->trace[T->traceno]); + } +- lj_mem_free(g, T, +- ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + +- T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry)); ++ ++ size_t sz = ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + ++ T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry); ++ lj_mem_tracing_skip_nprobes(g, 1); ++ lj_mem_free(g, T, sz); ++ lj_mem_tracing_jit_trace_probe(g, sz, 0); + } + + /* Re-enable compiling a prototype by unpatching any modified bytecode. */ +@@ -364,10 +374,16 @@ void lj_trace_freestate(global_State *g) + } + #endif + lj_mcode_free(J); ++ lj_mem_tracing_skip_nprobes(g, 4); + lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry); + lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot); + lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns); + lj_mem_freevec(g, J->trace, J->sizetrace, GCRef); ++ ++ lj_mem_tracing_jit_trace_probe(g, J->sizesnapmap * sizeof(SnapEntry), 0); ++ lj_mem_tracing_jit_trace_probe(g, J->sizesnap * sizeof(SnapShot), 0); ++ lj_mem_tracing_jit_trace_probe(g, (J->irtoplim - J->irbotlim) * sizeof(IRIns), 0); ++ lj_mem_tracing_jit_trace_probe(g, J->sizetrace * sizeof(GCRef), 0); + } + + /* -- Penalties and blacklisting ------------------------------------------ */ +diff --git a/bundle/LuaJIT-2.1-20210510/src/lj_udata.c b/bundle/LuaJIT-2.1-20210510/src/lj_udata.c +index ec3478c8..cc6e4d78 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lj_udata.c ++++ b/bundle/LuaJIT-2.1-20210510/src/lj_udata.c +@@ -10,6 +10,7 @@ + #include "lj_gc.h" + #include "lj_udata.h" + ++ + GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env) + { + GCudata *ud = lj_mem_newt(L, sizeof(GCudata) + sz, GCudata); +diff --git a/bundle/LuaJIT-2.1-20210510/src/ljamalg.c b/bundle/LuaJIT-2.1-20210510/src/ljamalg.c +index 6cba6d70..934fb1c9 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/ljamalg.c ++++ b/bundle/LuaJIT-2.1-20210510/src/ljamalg.c +@@ -85,4 +85,5 @@ + #include "lib_bit.c" + #include "lib_jit.c" + #include "lib_ffi.c" ++#include "lib_kprof.c" + #include "lib_init.c" +diff --git a/bundle/LuaJIT-2.1-20210510/src/lualib.h b/bundle/LuaJIT-2.1-20210510/src/lualib.h +index 9d1aa5e4..26fea59d 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/lualib.h ++++ b/bundle/LuaJIT-2.1-20210510/src/lualib.h +@@ -22,6 +22,9 @@ + #define LUA_JITLIBNAME "jit" + #define LUA_FFILIBNAME "ffi" + #define LUA_THRLIBNAME "thread" ++#ifndef LUAJIT_DISABLE_KPROF ++#define LUA_KPROFNAME "kprof" ++#endif + + LUALIB_API int luaopen_base(lua_State *L); + LUALIB_API int luaopen_math(lua_State *L); +@@ -35,6 +38,10 @@ LUALIB_API int luaopen_bit(lua_State *L); + LUALIB_API int luaopen_jit(lua_State *L); + LUALIB_API int luaopen_ffi(lua_State *L); + ++#ifndef LUAJIT_DISABLE_KPROF ++LUALIB_API int luaopen_kprof(lua_State *L); ++#endif ++ + LUALIB_API void luaL_openlibs(lua_State *L); + + #ifndef lua_assert +-- +2.34.1 + diff --git a/build/openresty/patches/LuaJIT-2.1-20210510_04_pass_cc_env.patch b/build/openresty/patches/LuaJIT-2.1-20210510_04_pass_cc_env.patch new file mode 100644 index 000000000000..afe165ab78a1 --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20210510_04_pass_cc_env.patch @@ -0,0 +1,40 @@ +diff --git a/bundle/LuaJIT-2.1-20210510/src/Makefile b/bundle/LuaJIT-2.1-20210510/src/Makefile +index 47a21c9..c60b94e 100644 +--- a/bundle/LuaJIT-2.1-20210510/src/Makefile ++++ b/bundle/LuaJIT-2.1-20210510/src/Makefile +@@ -27,7 +27,8 @@ NODOTABIVER= 51 + DEFAULT_CC = gcc + # + # LuaJIT builds as a native 32 or 64 bit binary by default. +-CC= $(DEFAULT_CC) ++CC?= $(DEFAULT_CC) ++AR?= ar + # + # Use this if you want to force a 32 bit build on a 64 bit multilib OS. + #CC= $(DEFAULT_CC) -m32 +@@ -211,7 +212,7 @@ TARGET_CC= $(STATIC_CC) + TARGET_STCC= $(STATIC_CC) + TARGET_DYNCC= $(DYNAMIC_CC) + TARGET_LD= $(CROSS)$(CC) +-TARGET_AR= $(CROSS)ar rcus 2>/dev/null ++TARGET_AR= $(CROSS)$(AR) rcus 2>/dev/null + TARGET_STRIP= $(CROSS)strip + + TARGET_LIBPATH= $(or $(PREFIX),/usr/local)/$(or $(MULTILIB),lib) +@@ -291,11 +292,11 @@ TARGET_XCFLAGS+= $(CCOPT_$(TARGET_LJARCH)) + TARGET_ARCH+= $(patsubst %,-DLUAJIT_TARGET=LUAJIT_ARCH_%,$(TARGET_LJARCH)) + + ifneq (,$(PREFIX)) +-ifneq (/usr/local,$(PREFIX)) +- TARGET_XCFLAGS+= -DLUA_ROOT=\"$(PREFIX)\" +- ifneq (/usr,$(PREFIX)) +- TARGET_DYNXLDOPTS= -Wl,-rpath,$(TARGET_LIBPATH) +- endif ++ifneq (/usr/local,$(LUA_ROOT)) ++ TARGET_XCFLAGS+= -DLUA_ROOT=\"$(LUA_ROOT)\" ++endif ++ifneq (/usr,$(PREFIX)) ++ TARGET_DYNXLDOPTS= -Wl,-rpath,$(TARGET_LIBPATH) + endif + endif + ifneq (,$(MULTILIB)) \ No newline at end of file diff --git a/build/openresty/patches/LuaJIT-2.1-20220411_01_patch_macro_luajit_version.patch b/build/openresty/patches/LuaJIT-2.1-20220411_01_patch_macro_luajit_version.patch new file mode 100644 index 000000000000..fbb6be67bb63 --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20220411_01_patch_macro_luajit_version.patch @@ -0,0 +1,27 @@ +From f53c8fa441f4233b9a3f19fcd870207fe8795456 Mon Sep 17 00:00:00 2001 +From: Qi +Date: Wed, 25 May 2022 18:35:08 +0800 +Subject: [PATCH] Patch macro `LUAJIT_VERSION` + +--- + src/luajit.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/bundle/LuaJIT-2.1-20220411/src/luajit.h b/bundle/LuaJIT-2.1-20220411/src/luajit.h +index a4d33001..e35f4e7e 100644 +--- a/bundle/LuaJIT-2.1-20220411/src/luajit.h ++++ b/bundle/LuaJIT-2.1-20220411/src/luajit.h +@@ -32,7 +32,9 @@ + + #define OPENRESTY_LUAJIT + ++#ifndef LUAJIT_VERSION + #define LUAJIT_VERSION "LuaJIT 2.1.0-beta3" ++#endif + #define LUAJIT_VERSION_NUM 20100 /* Version 2.1.0 = 02.01.00. */ + #define LUAJIT_VERSION_SYM luaJIT_version_2_1_0_beta3 + #define LUAJIT_COPYRIGHT "Copyright (C) 2005-2022 Mike Pall" +-- +2.34.1 + + diff --git a/build/openresty/patches/LuaJIT-2.1-20220411_02.patch b/build/openresty/patches/LuaJIT-2.1-20220411_02.patch new file mode 100644 index 000000000000..971ab37cd5b9 --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20220411_02.patch @@ -0,0 +1,24 @@ +From dad04f1754723e76ba9dcf9f401f3134a0cd3972 Mon Sep 17 00:00:00 2001 +From: Mike Pall +Date: Wed, 14 Sep 2022 12:26:53 +0200 +Subject: [PATCH] Fix trace join to BC_JLOOP originating from BC_ITERN. + +Reported by OpenResty Inc. +--- + src/lj_record.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/bundle/LuaJIT-2.1-20220411/src/lj_record.c b/bundle/LuaJIT-2.1-20220411/src/lj_record.c +index 5d02d24a1..bfd412365 100644 +--- a/bundle/LuaJIT-2.1-20220411/src/lj_record.c ++++ b/bundle/LuaJIT-2.1-20220411/src/lj_record.c +@@ -2566,7 +2566,8 @@ void lj_record_ins(jit_State *J) + break; + case BC_JLOOP: + rec_loop_jit(J, rc, rec_loop(J, ra, +- !bc_isret(bc_op(traceref(J, rc)->startins)))); ++ !bc_isret(bc_op(traceref(J, rc)->startins)) && ++ bc_op(traceref(J, rc)->startins) != BC_ITERN)); + break; + + case BC_IFORL: \ No newline at end of file diff --git a/build/openresty/patches/LuaJIT-2.1-20220411_03_pass_cc_env.patch b/build/openresty/patches/LuaJIT-2.1-20220411_03_pass_cc_env.patch new file mode 100644 index 000000000000..7d07a0b4fae5 --- /dev/null +++ b/build/openresty/patches/LuaJIT-2.1-20220411_03_pass_cc_env.patch @@ -0,0 +1,30 @@ +diff --git a/bundle/LuaJIT-2.1-20220411/src/Makefile b/bundle/LuaJIT-2.1-20220411/src/Makefile +index 68a9a7c..8d2de33 100644 +--- a/bundle/LuaJIT-2.1-20220411/src/Makefile ++++ b/bundle/LuaJIT-2.1-20220411/src/Makefile +@@ -27,7 +27,7 @@ NODOTABIVER= 51 + DEFAULT_CC = gcc + # + # LuaJIT builds as a native 32 or 64 bit binary by default. +-CC= $(DEFAULT_CC) ++CC?= $(DEFAULT_CC) + # + # Use this if you want to force a 32 bit build on a 64 bit multilib OS. + #CC= $(DEFAULT_CC) -m32 +@@ -291,11 +291,11 @@ TARGET_XCFLAGS+= $(CCOPT_$(TARGET_LJARCH)) + TARGET_ARCH+= $(patsubst %,-DLUAJIT_TARGET=LUAJIT_ARCH_%,$(TARGET_LJARCH)) + + ifneq (,$(PREFIX)) +-ifneq (/usr/local,$(PREFIX)) +- TARGET_XCFLAGS+= -DLUA_ROOT=\"$(PREFIX)\" +- ifneq (/usr,$(PREFIX)) +- TARGET_DYNXLDOPTS= -Wl,-rpath,$(TARGET_LIBPATH) +- endif ++ifneq (/usr/local,$(LUA_ROOT)) ++ TARGET_XCFLAGS+= -DLUA_ROOT=\"$(LUA_ROOT)\" ++endif ++ifneq (/usr,$(PREFIX)) ++ TARGET_DYNXLDOPTS= -Wl,-rpath,$(TARGET_LIBPATH) + endif + endif + ifneq (,$(MULTILIB)) \ No newline at end of file diff --git a/build/openresty/patches/lua-cjson-2.1.0.10_01-empty_array.patch b/build/openresty/patches/lua-cjson-2.1.0.10_01-empty_array.patch new file mode 100644 index 000000000000..f0542d6624bc --- /dev/null +++ b/build/openresty/patches/lua-cjson-2.1.0.10_01-empty_array.patch @@ -0,0 +1,12 @@ +diff -ruN a/bundle/lua-cjson-2.1.0.8/lua_cjson.c b/bundle/lua-cjson-2.1.0.8/lua_cjson.c +--- a/bundle/lua-cjson-2.1.0.10/lua_cjson.c 2022-01-11 15:11:17.495464192 +0800 ++++ b/bundle/lua-cjson-2.1.0.10/lua_cjson.c 2022-01-11 14:58:55.150669748 +0800 +@@ -800,7 +800,7 @@ + case LUA_TLIGHTUSERDATA: + if (lua_touserdata(l, -1) == NULL) { + strbuf_append_mem(json, "null", 4); +- } else if (lua_touserdata(l, -1) == &json_array) { ++ } else if (lua_touserdata(l, -1) == json_lightudata_mask(&json_array)) { + json_append_array(l, cfg, current_depth, json, 0); + } + break; diff --git a/build/openresty/patches/lua-cjson-2.1.0.8_01-empty_array.patch b/build/openresty/patches/lua-cjson-2.1.0.8_01-empty_array.patch new file mode 100644 index 000000000000..01e413909d1b --- /dev/null +++ b/build/openresty/patches/lua-cjson-2.1.0.8_01-empty_array.patch @@ -0,0 +1,12 @@ +diff -ruN a/lua-cjson-2.1.0.8/lua_cjson.c b/lua-cjson-2.1.0.8/lua_cjson.c +--- a/bundle/lua-cjson-2.1.0.8/lua_cjson.c 2022-01-11 15:11:17.495464192 +0800 ++++ b/bundle/lua-cjson-2.1.0.8/lua_cjson.c 2022-01-11 14:58:55.150669748 +0800 +@@ -796,7 +796,7 @@ + case LUA_TLIGHTUSERDATA: + if (lua_touserdata(l, -1) == NULL) { + strbuf_append_mem(json, "null", 4); +- } else if (lua_touserdata(l, -1) == &json_array) { ++ } else if (lua_touserdata(l, -1) == json_lightudata_mask(&json_array)) { + json_append_array(l, cfg, current_depth, json, 0); + } + break; diff --git a/build/openresty/patches/lua-cjson-2.1.0.8_02-handle-large-string-correctly.patch b/build/openresty/patches/lua-cjson-2.1.0.8_02-handle-large-string-correctly.patch new file mode 100644 index 000000000000..12a8fd806a41 --- /dev/null +++ b/build/openresty/patches/lua-cjson-2.1.0.8_02-handle-large-string-correctly.patch @@ -0,0 +1,387 @@ +diff --git a/bundle/lua-cjson-2.1.0.8/lua_cjson.c b/bundle/lua-cjson-2.1.0.8/lua_cjson.c +index 875bdaf..4fd2c93 100644 +--- a/bundle/lua-cjson-2.1.0.8/lua_cjson.c ++++ b/bundle/lua-cjson-2.1.0.8/lua_cjson.c +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -173,13 +174,13 @@ typedef struct { + + typedef struct { + json_token_type_t type; +- int index; ++ size_t index; + union { + const char *string; + double number; + int boolean; + } value; +- int string_len; ++ size_t string_len; + } json_token_t; + + static const char *char2escape[256] = { +@@ -540,6 +541,8 @@ static void json_append_string(lua_State *l, strbuf_t *json, int lindex) + * This buffer is reused constantly for small strings + * If there are any excess pages, they won't be hit anyway. + * This gains ~5% speedup. */ ++ if (len > SIZE_MAX / 6 - 3) ++ abort(); /* Overflow check */ + strbuf_ensure_empty_length(json, len * 6 + 2); + + strbuf_append_char_unsafe(json, '\"'); +@@ -814,7 +817,7 @@ static int json_encode(lua_State *l) + strbuf_t local_encode_buf; + strbuf_t *encode_buf; + char *json; +- int len; ++ size_t len; + + luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); + +diff --git a/bundle/lua-cjson-2.1.0.8/strbuf.c b/bundle/lua-cjson-2.1.0.8/strbuf.c +index f0f7f4b..2dc30be 100644 +--- a/bundle/lua-cjson-2.1.0.8/strbuf.c ++++ b/bundle/lua-cjson-2.1.0.8/strbuf.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + + #include "strbuf.h" + +@@ -38,22 +39,22 @@ static void die(const char *fmt, ...) + va_end(arg); + fprintf(stderr, "\n"); + +- exit(-1); ++ abort(); + } + +-void strbuf_init(strbuf_t *s, int len) ++void strbuf_init(strbuf_t *s, size_t len) + { +- int size; ++ size_t size; + +- if (len <= 0) ++ if (!len) + size = STRBUF_DEFAULT_SIZE; + else +- size = len + 1; /* \0 terminator */ +- ++ size = len + 1; ++ if (size < len) ++ die("Overflow, len: %zu", len); + s->buf = NULL; + s->size = size; + s->length = 0; +- s->increment = STRBUF_DEFAULT_INCREMENT; + s->dynamic = 0; + s->reallocs = 0; + s->debug = 0; +@@ -65,7 +66,7 @@ void strbuf_init(strbuf_t *s, int len) + strbuf_ensure_null(s); + } + +-strbuf_t *strbuf_new(int len) ++strbuf_t *strbuf_new(size_t len) + { + strbuf_t *s; + +@@ -81,20 +82,10 @@ strbuf_t *strbuf_new(int len) + return s; + } + +-void strbuf_set_increment(strbuf_t *s, int increment) +-{ +- /* Increment > 0: Linear buffer growth rate +- * Increment < -1: Exponential buffer growth rate */ +- if (increment == 0 || increment == -1) +- die("BUG: Invalid string increment"); +- +- s->increment = increment; +-} +- + static inline void debug_stats(strbuf_t *s) + { + if (s->debug) { +- fprintf(stderr, "strbuf(%lx) reallocs: %d, length: %d, size: %d\n", ++ fprintf(stderr, "strbuf(%lx) reallocs: %d, length: %zd, size: %zd\n", + (long)s, s->reallocs, s->length, s->size); + } + } +@@ -113,7 +104,7 @@ void strbuf_free(strbuf_t *s) + free(s); + } + +-char *strbuf_free_to_string(strbuf_t *s, int *len) ++char *strbuf_free_to_string(strbuf_t *s, size_t *len) + { + char *buf; + +@@ -131,57 +122,63 @@ char *strbuf_free_to_string(strbuf_t *s, int *len) + return buf; + } + +-static int calculate_new_size(strbuf_t *s, int len) ++static size_t calculate_new_size(strbuf_t *s, size_t len) + { +- int reqsize, newsize; ++ size_t reqsize, newsize; + + if (len <= 0) + die("BUG: Invalid strbuf length requested"); + + /* Ensure there is room for optional NULL termination */ + reqsize = len + 1; ++ if (reqsize < len) ++ die("Overflow, len: %zu", len); + + /* If the user has requested to shrink the buffer, do it exactly */ + if (s->size > reqsize) + return reqsize; + + newsize = s->size; +- if (s->increment < 0) { ++ if (reqsize >= SIZE_MAX / 2) { ++ newsize = reqsize; ++ } else { + /* Exponential sizing */ + while (newsize < reqsize) +- newsize *= -s->increment; +- } else { +- /* Linear sizing */ +- newsize = ((newsize + s->increment - 1) / s->increment) * s->increment; ++ newsize *= 2; + } + ++ if (newsize < reqsize) ++ die("BUG: strbuf length would overflow, len: %zu", len); ++ ++ + return newsize; + } + + + /* Ensure strbuf can handle a string length bytes long (ignoring NULL + * optional termination). */ +-void strbuf_resize(strbuf_t *s, int len) ++void strbuf_resize(strbuf_t *s, size_t len) + { +- int newsize; ++ size_t newsize; + + newsize = calculate_new_size(s, len); + + if (s->debug > 1) { +- fprintf(stderr, "strbuf(%lx) resize: %d => %d\n", ++ fprintf(stderr, "strbuf(%lx) resize: %zd => %zd\n", + (long)s, s->size, newsize); + } + + s->size = newsize; + s->buf = realloc(s->buf, s->size); + if (!s->buf) +- die("Out of memory"); ++ die("Out of memory, len: %zu", len); + s->reallocs++; + } + + void strbuf_append_string(strbuf_t *s, const char *str) + { +- int space, i; ++ int i; ++ size_t space; + + space = strbuf_empty_length(s); + +@@ -197,55 +194,6 @@ void strbuf_append_string(strbuf_t *s, const char *str) + } + } + +-/* strbuf_append_fmt() should only be used when an upper bound +- * is known for the output string. */ +-void strbuf_append_fmt(strbuf_t *s, int len, const char *fmt, ...) +-{ +- va_list arg; +- int fmt_len; +- +- strbuf_ensure_empty_length(s, len); +- +- va_start(arg, fmt); +- fmt_len = vsnprintf(s->buf + s->length, len, fmt, arg); +- va_end(arg); +- +- if (fmt_len < 0) +- die("BUG: Unable to convert number"); /* This should never happen.. */ +- +- s->length += fmt_len; +-} +- +-/* strbuf_append_fmt_retry() can be used when the there is no known +- * upper bound for the output string. */ +-void strbuf_append_fmt_retry(strbuf_t *s, const char *fmt, ...) +-{ +- va_list arg; +- int fmt_len, try; +- int empty_len; +- +- /* If the first attempt to append fails, resize the buffer appropriately +- * and try again */ +- for (try = 0; ; try++) { +- va_start(arg, fmt); +- /* Append the new formatted string */ +- /* fmt_len is the length of the string required, excluding the +- * trailing NULL */ +- empty_len = strbuf_empty_length(s); +- /* Add 1 since there is also space to store the terminating NULL. */ +- fmt_len = vsnprintf(s->buf + s->length, empty_len + 1, fmt, arg); +- va_end(arg); +- +- if (fmt_len <= empty_len) +- break; /* SUCCESS */ +- if (try > 0) +- die("BUG: length of formatted string changed"); +- +- strbuf_resize(s, s->length + fmt_len); +- } +- +- s->length += fmt_len; +-} + + /* vi:ai et sw=4 ts=4: + */ +diff --git a/bundle/lua-cjson-2.1.0.8/strbuf.h b/bundle/lua-cjson-2.1.0.8/strbuf.h +index 5df0b7b..d77e0f4 100644 +--- a/bundle/lua-cjson-2.1.0.8/strbuf.h ++++ b/bundle/lua-cjson-2.1.0.8/strbuf.h +@@ -32,15 +32,13 @@ + + /* Size: Total bytes allocated to *buf + * Length: String length, excluding optional NULL terminator. +- * Increment: Allocation increments when resizing the string buffer. + * Dynamic: True if created via strbuf_new() + */ + + typedef struct { + char *buf; +- int size; +- int length; +- int increment; ++ size_t size; ++ size_t length; + int dynamic; + int reallocs; + int debug; +@@ -49,32 +47,27 @@ typedef struct { + #ifndef STRBUF_DEFAULT_SIZE + #define STRBUF_DEFAULT_SIZE 1023 + #endif +-#ifndef STRBUF_DEFAULT_INCREMENT +-#define STRBUF_DEFAULT_INCREMENT -2 +-#endif + + /* Initialise */ +-extern strbuf_t *strbuf_new(int len); +-extern void strbuf_init(strbuf_t *s, int len); +-extern void strbuf_set_increment(strbuf_t *s, int increment); ++extern strbuf_t *strbuf_new(size_t len); ++extern void strbuf_init(strbuf_t *s, size_t len); + + /* Release */ + extern void strbuf_free(strbuf_t *s); +-extern char *strbuf_free_to_string(strbuf_t *s, int *len); ++extern char *strbuf_free_to_string(strbuf_t *s, size_t *len); + + /* Management */ +-extern void strbuf_resize(strbuf_t *s, int len); +-static int strbuf_empty_length(strbuf_t *s); +-static int strbuf_length(strbuf_t *s); +-static char *strbuf_string(strbuf_t *s, int *len); +-static void strbuf_ensure_empty_length(strbuf_t *s, int len); ++extern void strbuf_resize(strbuf_t *s, size_t len); ++static size_t strbuf_empty_length(strbuf_t *s); ++static size_t strbuf_length(strbuf_t *s); ++static char *strbuf_string(strbuf_t *s, size_t *len); ++static void strbuf_ensure_empty_length(strbuf_t *s, size_t len); + static char *strbuf_empty_ptr(strbuf_t *s); +-static void strbuf_extend_length(strbuf_t *s, int len); ++static void strbuf_extend_length(strbuf_t *s, size_t len); ++static void strbuf_set_length(strbuf_t *s, int len); + + /* Update */ +-extern void strbuf_append_fmt(strbuf_t *s, int len, const char *fmt, ...); +-extern void strbuf_append_fmt_retry(strbuf_t *s, const char *format, ...); +-static void strbuf_append_mem(strbuf_t *s, const char *c, int len); ++static void strbuf_append_mem(strbuf_t *s, const char *c, size_t len); + extern void strbuf_append_string(strbuf_t *s, const char *str); + static void strbuf_append_char(strbuf_t *s, const char c); + static void strbuf_ensure_null(strbuf_t *s); +@@ -92,12 +85,12 @@ static inline int strbuf_allocated(strbuf_t *s) + + /* Return bytes remaining in the string buffer + * Ensure there is space for a NULL terminator. */ +-static inline int strbuf_empty_length(strbuf_t *s) ++static inline size_t strbuf_empty_length(strbuf_t *s) + { + return s->size - s->length - 1; + } + +-static inline void strbuf_ensure_empty_length(strbuf_t *s, int len) ++static inline void strbuf_ensure_empty_length(strbuf_t *s, size_t len) + { + if (len > strbuf_empty_length(s)) + strbuf_resize(s, s->length + len); +@@ -108,12 +101,17 @@ static inline char *strbuf_empty_ptr(strbuf_t *s) + return s->buf + s->length; + } + +-static inline void strbuf_extend_length(strbuf_t *s, int len) ++static inline void strbuf_set_length(strbuf_t *s, int len) ++{ ++ s->length = len; ++} ++ ++static inline void strbuf_extend_length(strbuf_t *s, size_t len) + { + s->length += len; + } + +-static inline int strbuf_length(strbuf_t *s) ++static inline size_t strbuf_length(strbuf_t *s) + { + return s->length; + } +@@ -129,14 +127,14 @@ static inline void strbuf_append_char_unsafe(strbuf_t *s, const char c) + s->buf[s->length++] = c; + } + +-static inline void strbuf_append_mem(strbuf_t *s, const char *c, int len) ++static inline void strbuf_append_mem(strbuf_t *s, const char *c, size_t len) + { + strbuf_ensure_empty_length(s, len); + memcpy(s->buf + s->length, c, len); + s->length += len; + } + +-static inline void strbuf_append_mem_unsafe(strbuf_t *s, const char *c, int len) ++static inline void strbuf_append_mem_unsafe(strbuf_t *s, const char *c, size_t len) + { + memcpy(s->buf + s->length, c, len); + s->length += len; +@@ -147,7 +145,7 @@ static inline void strbuf_ensure_null(strbuf_t *s) + s->buf[s->length] = 0; + } + +-static inline char *strbuf_string(strbuf_t *s, int *len) ++static inline char *strbuf_string(strbuf_t *s, size_t *len) + { + if (len) + *len = s->length; diff --git a/build/openresty/patches/lua-resty-core-0.1.22_01-cosocket-mtls.patch b/build/openresty/patches/lua-resty-core-0.1.22_01-cosocket-mtls.patch new file mode 100644 index 000000000000..20470b087f53 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.22_01-cosocket-mtls.patch @@ -0,0 +1,566 @@ +From 4f0f4bf63d23a952179aaf810c10dfffc19ee835 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 20:54:30 +0800 +Subject: [PATCH 1/9] move tcp.lua into socket.lua + +--- + lib/resty/core/socket.lua | 136 +++++++++++++++++++++++++++++++++++++- + 1 file changed, 133 insertions(+), 3 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 1a504ec..cc0081e 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -6,13 +6,21 @@ local ffi = require 'ffi' + + local error = error + local tonumber = tonumber ++local tostring = tostring ++local type = type ++local select = select + local registry = debug.getregistry() ++ ++local C = ffi.C + local ffi_new = ffi.new + local ffi_string = ffi.string +-local C = ffi.C ++local ffi_gc = ffi.gc ++ + local get_string_buf = base.get_string_buf + local get_size_ptr = base.get_size_ptr +-local tostring = tostring ++local get_request = base.get_request ++ ++local co_yield = coroutine._yield + + + local option_index = { +@@ -35,15 +43,29 @@ ngx_http_lua_ffi_socket_tcp_getoption(ngx_http_lua_socket_tcp_upstream_t *u, + int + ngx_http_lua_ffi_socket_tcp_setoption(ngx_http_lua_socket_tcp_upstream_t *u, + int opt, int val, unsigned char *err, size_t *errlen); ++ ++int ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, void *sess, ++ int enable_session_reuse, ngx_str_t *server_name, int verify, ++ int ocsp_status_req, void *chain, void *pkey, char **errmsg); ++ ++int ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, void **sess, char **errmsg, ++ int *openssl_error_code); ++ ++void ngx_http_lua_ffi_ssl_free_session(void *sess); + ]] + + + local output_value_buf = ffi_new("int[1]") + local FFI_OK = base.FFI_OK ++local FFI_ERROR = base.FFI_ERROR ++local FFI_DONE = base.FFI_DONE ++local FFI_AGAIN = base.FFI_AGAIN ++local FFI_NO_REQ_CTX = base.FFI_NO_REQ_CTX + local SOCKET_CTX_INDEX = 1 + local ERR_BUF_SIZE = 4096 + +- + local function get_tcp_socket(cosocket) + local tcp_socket = cosocket[SOCKET_CTX_INDEX] + if not tcp_socket then +@@ -114,10 +136,118 @@ local function setoption(cosocket, option, value) + end + + ++local errmsg = base.get_errmsg_ptr() ++local session_ptr = ffi_new("void *[1]") ++local server_name_str = ffi_new("ngx_str_t[1]") ++local openssl_error_code = ffi_new("int[1]") ++ ++ ++local function setclientcert(self, cert, pkey) ++ if not cert and not pkey then ++ self.client_cert = nil ++ self.client_pkey = nil ++ return ++ end ++ ++ if not cert or not pkey then ++ error("client certificate must be supplied with corresponding " .. ++ "private key", 2) ++ end ++ ++ if type(cert) ~= "cdata" then ++ error("bad client cert type", 2) ++ end ++ ++ if type(pkey) ~= "cdata" then ++ error("bad client pkey type", 2) ++ end ++ ++ self.client_cert = cert ++ self.client_pkey = pkey ++end ++ ++ ++local function sslhandshake(self, reused_session, server_name, ssl_verify, ++ send_status_req, ...) ++ ++ local n = select("#", ...) ++ if not self or n > 1 then ++ error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. ++ "(including the object), but seen " .. (5 + n)) ++ end ++ ++ local r = get_request() ++ if not r then ++ error("no request found", 2) ++ end ++ ++ session_ptr[0] = type(reused_session) == "cdata" and reused_session or nil ++ ++ if server_name then ++ server_name_str[0].data = server_name ++ server_name_str[0].len = #server_name ++ ++ else ++ server_name_str[0].data = nil ++ server_name_str[0].len = 0 ++ end ++ ++ local u = self[SOCKET_CTX_INDEX] ++ ++ local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, ++ session_ptr[0], ++ reused_session ~= false, ++ server_name_str, ++ ssl_verify and 1 or 0, ++ send_status_req and 1 or 0, ++ self.client_cert, self.client_pkey, errmsg) ++ ++ if rc == FFI_NO_REQ_CTX then ++ error("no request ctx found", 2) ++ end ++ ++ while true do ++ if rc == FFI_ERROR then ++ if openssl_error_code[0] ~= 0 then ++ return nil, openssl_error_code[0] .. ": " .. ffi_string(errmsg[0]) ++ end ++ ++ return nil, ffi_string(errmsg[0]) ++ end ++ ++ if rc == FFI_DONE then ++ return reused_session ++ end ++ ++ if rc == FFI_OK then ++ if reused_session == false then ++ return true ++ end ++ ++ rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, ++ session_ptr, errmsg, openssl_error_code) ++ ++ if session_ptr[0] == nil then ++ return nil ++ end ++ ++ return ffi_gc(session_ptr[0], C.ngx_http_lua_ffi_ssl_free_session) ++ end ++ ++ co_yield() ++ ++ rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, ++ session_ptr, errmsg, openssl_error_code) ++ end ++end ++ ++ + do + local method_table = registry.__tcp_cosocket_mt + method_table.getoption = getoption + method_table.setoption = setoption ++ method_table.setclientcert = setclientcert ++ method_table.sslhandshake = sslhandshake + end + + +-- +2.32.0 (Apple Git-132) + + +From 4eab5793d741c739d9c5cfe14e0671c1d70fd6e5 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 21:37:45 +0800 +Subject: [PATCH 2/9] revert assert in sslhandshake + +--- + lib/resty/core/socket.lua | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index cc0081e..7c61d06 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -5,6 +5,7 @@ local ffi = require 'ffi' + + + local error = error ++local assert = assert + local tonumber = tonumber + local tostring = tostring + local type = type +@@ -227,6 +228,8 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, + session_ptr, errmsg, openssl_error_code) + ++ assert(rc == FFI_OK) ++ + if session_ptr[0] == nil then + return nil + end +@@ -234,6 +237,8 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + return ffi_gc(session_ptr[0], C.ngx_http_lua_ffi_ssl_free_session) + end + ++ assert(rc == FFI_AGAIN) ++ + co_yield() + + rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, +-- +2.32.0 (Apple Git-132) + + +From 58de9a44c89f07eda98bb7fd978a9e04a244d2f2 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 21:45:42 +0800 +Subject: [PATCH 3/9] rename ffi_string to ffi_str + +--- + lib/resty/core/socket.lua | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 7c61d06..14457da 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -14,7 +14,7 @@ local registry = debug.getregistry() + + local C = ffi.C + local ffi_new = ffi.new +-local ffi_string = ffi.string ++local ffi_str = ffi.string + local ffi_gc = ffi.gc + + local get_string_buf = base.get_string_buf +@@ -98,7 +98,7 @@ local function getoption(cosocket, option) + err, + errlen) + if rc ~= FFI_OK then +- return nil, ffi_string(err, errlen[0]) ++ return nil, ffi_str(err, errlen[0]) + end + + return tonumber(output_value_buf[0]) +@@ -130,7 +130,7 @@ local function setoption(cosocket, option, value) + err, + errlen) + if rc ~= FFI_OK then +- return nil, ffi_string(err, errlen[0]) ++ return nil, ffi_str(err, errlen[0]) + end + + return true +@@ -210,10 +210,10 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + while true do + if rc == FFI_ERROR then + if openssl_error_code[0] ~= 0 then +- return nil, openssl_error_code[0] .. ": " .. ffi_string(errmsg[0]) ++ return nil, openssl_error_code[0] .. ": " .. ffi_str(errmsg[0]) + end + +- return nil, ffi_string(errmsg[0]) ++ return nil, ffi_str(errmsg[0]) + end + + if rc == FFI_DONE then +-- +2.32.0 (Apple Git-132) + + +From ff138619432bda6b9bd4f37403c12600a4739e47 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:23:16 +0800 +Subject: [PATCH 4/9] minor style fix + +--- + lib/resty/core/socket.lua | 15 +++++++++------ + 1 file changed, 9 insertions(+), 6 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 14457da..3c882af 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -1,7 +1,7 @@ + local base = require "resty.core.base" +-base.allows_subsystem('http') +-local debug = require 'debug' +-local ffi = require 'ffi' ++base.allows_subsystem("http") ++local debug = require "debug" ++local ffi = require "ffi" + + + local error = error +@@ -45,16 +45,19 @@ int + ngx_http_lua_ffi_socket_tcp_setoption(ngx_http_lua_socket_tcp_upstream_t *u, + int opt, int val, unsigned char *err, size_t *errlen); + +-int ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, ++int ++ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, void *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, + int ocsp_status_req, void *chain, void *pkey, char **errmsg); + +-int ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, ++int ++ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, void **sess, char **errmsg, + int *openssl_error_code); + +-void ngx_http_lua_ffi_ssl_free_session(void *sess); ++void ++ngx_http_lua_ffi_ssl_free_session(void *sess); + ]] + + +-- +2.32.0 (Apple Git-132) + + +From a843a258987efba49f0b6979389f75ee32c2150c Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:28:41 +0800 +Subject: [PATCH 5/9] rename self to cosocket + +--- + lib/resty/core/socket.lua | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 3c882af..374d583 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -146,10 +146,10 @@ local server_name_str = ffi_new("ngx_str_t[1]") + local openssl_error_code = ffi_new("int[1]") + + +-local function setclientcert(self, cert, pkey) ++local function setclientcert(cosocket, cert, pkey) + if not cert and not pkey then +- self.client_cert = nil +- self.client_pkey = nil ++ cosocket.client_cert = nil ++ cosocket.client_pkey = nil + return + end + +@@ -166,16 +166,16 @@ local function setclientcert(self, cert, pkey) + error("bad client pkey type", 2) + end + +- self.client_cert = cert +- self.client_pkey = pkey ++ cosocket.client_cert = cert ++ cosocket.client_pkey = pkey + end + + +-local function sslhandshake(self, reused_session, server_name, ssl_verify, ++local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + send_status_req, ...) + + local n = select("#", ...) +- if not self or n > 1 then ++ if not cosocket or n > 1 then + error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. + "(including the object), but seen " .. (5 + n)) + end +@@ -196,7 +196,7 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + server_name_str[0].len = 0 + end + +- local u = self[SOCKET_CTX_INDEX] ++ local u = cosocket[SOCKET_CTX_INDEX] + + local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, + session_ptr[0], +@@ -204,7 +204,7 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + server_name_str, + ssl_verify and 1 or 0, + send_status_req and 1 or 0, +- self.client_cert, self.client_pkey, errmsg) ++ cosocket.client_cert, cosocket.client_pkey, errmsg) + + if rc == FFI_NO_REQ_CTX then + error("no request ctx found", 2) +-- +2.32.0 (Apple Git-132) + + +From db95a049a019ff6f0d3b4e550412e40c25dda41f Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:35:04 +0800 +Subject: [PATCH 6/9] use get_tcp_socket() in sslhandshake + +--- + lib/resty/core/socket.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 374d583..ecff453 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -196,7 +196,7 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + server_name_str[0].len = 0 + end + +- local u = cosocket[SOCKET_CTX_INDEX] ++ local u = get_tcp_socket(cosocket) + + local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, + session_ptr[0], +-- +2.32.0 (Apple Git-132) + + +From 6767f0c2e8a73fd1a09d727431bed457c5cac4c0 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 08:58:52 +0800 +Subject: [PATCH 7/9] fix arguments check in sslhandshake + +--- + lib/resty/core/socket.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index ecff453..15e3065 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -177,7 +177,7 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + local n = select("#", ...) + if not cosocket or n > 1 then + error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. +- "(including the object), but seen " .. (5 + n)) ++ "(including the object), but seen " .. (cosocket and 5 + n or 0)) + end + + local r = get_request() +-- +2.32.0 (Apple Git-132) + + +From 4eeddcd2114d0097e4b9cb11f2f93d30c70d573e Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Mon, 7 Feb 2022 10:59:35 +0800 +Subject: [PATCH 8/9] setclientcert return err + +--- + lib/resty/core/socket.lua | 13 ++++++++----- + 1 file changed, 8 insertions(+), 5 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 15e3065..879d678 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -150,24 +150,27 @@ local function setclientcert(cosocket, cert, pkey) + if not cert and not pkey then + cosocket.client_cert = nil + cosocket.client_pkey = nil +- return ++ return true + end + + if not cert or not pkey then +- error("client certificate must be supplied with corresponding " .. +- "private key", 2) ++ return nil, ++ "client certificate must be supplied with corresponding " .. ++ "private key" + end + + if type(cert) ~= "cdata" then +- error("bad client cert type", 2) ++ return nil, "bad client cert type" + end + + if type(pkey) ~= "cdata" then +- error("bad client pkey type", 2) ++ return nil, "bad client pkey type" + end + + cosocket.client_cert = cert + cosocket.client_pkey = pkey ++ ++ return true + end + + +-- +2.32.0 (Apple Git-132) + + +From fead2a28f409117ad1b6c98d02edb6a38a64fde0 Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Wed, 9 Feb 2022 16:05:11 +0000 +Subject: [PATCH 9/9] fix(socket) add temporary backwards compatability for + tlshandshake + +--- + lib/resty/core/socket.lua | 22 ++++++++++++++++++++++ + 1 file changed, 22 insertions(+) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/socket.lua b/lua-resty-core-0.1.22/lib/resty/core/socket.lua +index 879d678..448bf36 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/socket.lua +@@ -253,12 +253,34 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + end + + ++-- Temporary patch for backwards compatablity with existing Kong tech debt ++local function tlshandshake(cosocket, options) ++ local options = options or {} ++ ++ if options.client_cert then ++ local ok, err = cosocket:setclientcert(options.client_cert, options.client_priv_key) ++ if not ok then ++ return nil, err ++ end ++ end ++ ++ return sslhandshake( ++ cosocket, ++ options.reused_session, ++ options.server_name, ++ options.ssl_verify, ++ options.ocsp_status_req ++ ) ++end ++ ++ + do + local method_table = registry.__tcp_cosocket_mt + method_table.getoption = getoption + method_table.setoption = setoption + method_table.setclientcert = setclientcert + method_table.sslhandshake = sslhandshake ++ method_table.tlshandshake = tlshandshake + end + + +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/lua-resty-core-0.1.22_02-dyn_upstream_keepalive.patch b/build/openresty/patches/lua-resty-core-0.1.22_02-dyn_upstream_keepalive.patch new file mode 100644 index 000000000000..f1663e178fc2 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.22_02-dyn_upstream_keepalive.patch @@ -0,0 +1,230 @@ +From 37feb95041f183ae4fbafeebc47dc104995e6f27 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:44:33 -0700 +Subject: [PATCH] feature: implemented the 'balancer.enable_keepalive()' API. + +--- + lua-resty-core-0.1.22/lib/ngx/balancer.lua | 165 +++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 151 insertions(+), 14 deletions(-) + +diff --git a/lua-resty-core-0.1.22/lib/ngx/balancer.lua b/lua-resty-core-0.1.22/lib/ngx/balancer.lua +index d584639..614312f 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/ngx/balancer.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/ngx/balancer.lua +@@ -3,6 +3,7 @@ + + local base = require "resty.core.base" + base.allows_subsystem('http', 'stream') ++require "resty.core.hash" + + + local ffi = require "ffi" +@@ -17,8 +18,10 @@ local error = error + local type = type + local tonumber = tonumber + local max = math.max ++local ngx_crc32_long = ngx.crc32_long + local subsystem = ngx.config.subsystem + local ngx_lua_ffi_balancer_set_current_peer ++local ngx_lua_ffi_balancer_enable_keepalive + local ngx_lua_ffi_balancer_set_more_tries + local ngx_lua_ffi_balancer_get_last_failure + local ngx_lua_ffi_balancer_set_timeouts -- used by both stream and http +@@ -27,7 +30,11 @@ local ngx_lua_ffi_balancer_set_timeouts -- used by both stream and http + if subsystem == 'http' then + ffi.cdef[[ + int ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, +- const unsigned char *addr, size_t addr_len, int port, char **err); ++ const unsigned char *addr, size_t addr_len, int port, ++ unsigned int cpool_crc32, unsigned int cpool_size, char **err); ++ ++ int ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, ++ unsigned long timeout, unsigned int max_requests, char **err); + + int ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + int count, char **err); +@@ -46,6 +53,9 @@ if subsystem == 'http' then + ngx_lua_ffi_balancer_set_current_peer = + C.ngx_http_lua_ffi_balancer_set_current_peer + ++ ngx_lua_ffi_balancer_enable_keepalive = ++ C.ngx_http_lua_ffi_balancer_enable_keepalive ++ + ngx_lua_ffi_balancer_set_more_tries = + C.ngx_http_lua_ffi_balancer_set_more_tries + +@@ -96,6 +106,11 @@ else + end + + ++local DEFAULT_KEEPALIVE_POOL_SIZE = 30 ++local DEFAULT_KEEPALIVE_IDLE_TIMEOUT = 60000 ++local DEFAULT_KEEPALIVE_MAX_REQUESTS = 100 ++ ++ + local peer_state_names = { + [1] = "keepalive", + [2] = "next", +@@ -106,25 +121,147 @@ local peer_state_names = { + local _M = { version = base.version } + + +-function _M.set_current_peer(addr, port) +- local r = get_request() +- if not r then +- error("no request found") ++if subsystem == "http" then ++ function _M.set_current_peer(addr, port, opts) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ local pool_crc32 ++ local pool_size ++ ++ if opts then ++ if type(opts) ~= "table" then ++ error("bad argument #3 to 'set_current_peer' " .. ++ "(table expected, got " .. type(opts) .. ")", 2) ++ end ++ ++ local pool = opts.pool ++ pool_size = opts.pool_size ++ ++ if pool then ++ if type(pool) ~= "string" then ++ error("bad option 'pool' to 'set_current_peer' " .. ++ "(string expected, got " .. type(pool) .. ")", 2) ++ end ++ ++ pool_crc32 = ngx_crc32_long(pool) ++ end ++ ++ if pool_size then ++ if type(pool_size) ~= "number" then ++ error("bad option 'pool_size' to 'set_current_peer' " .. ++ "(number expected, got " .. type(pool_size) .. ")", 2) ++ ++ elseif pool_size < 1 then ++ error("bad option 'pool_size' to 'set_current_peer' " .. ++ "(expected > 0)", 2) ++ end ++ end ++ end ++ ++ if not port then ++ port = 0 ++ ++ elseif type(port) ~= "number" then ++ port = tonumber(port) ++ end ++ ++ if not pool_crc32 then ++ pool_crc32 = 0 ++ end ++ ++ if not pool_size then ++ pool_size = DEFAULT_KEEPALIVE_POOL_SIZE ++ end ++ ++ local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, port, ++ pool_crc32, pool_size, ++ errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end + +- if not port then +- port = 0 +- elseif type(port) ~= "number" then +- port = tonumber(port) ++else ++ function _M.set_current_peer(addr, port, opts) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ if opts then ++ error("bad argument #3 to 'set_current_peer' ('opts' not yet " .. ++ "implemented in " .. subsystem .. " subsystem)", 2) ++ end ++ ++ if not port then ++ port = 0 ++ ++ elseif type(port) ~= "number" then ++ port = tonumber(port) ++ end ++ ++ local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, ++ port, errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end ++end + +- local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, +- port, errmsg) +- if rc == FFI_OK then +- return true ++ ++if subsystem == "http" then ++ function _M.enable_keepalive(idle_timeout, max_requests) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ if not idle_timeout then ++ idle_timeout = DEFAULT_KEEPALIVE_IDLE_TIMEOUT ++ ++ elseif type(idle_timeout) ~= "number" then ++ error("bad argument #1 to 'enable_keepalive' " .. ++ "(number expected, got " .. type(idle_timeout) .. ")", 2) ++ ++ elseif idle_timeout < 0 then ++ error("bad argument #1 to 'enable_keepalive' (expected >= 0)", 2) ++ ++ else ++ idle_timeout = idle_timeout * 1000 ++ end ++ ++ if not max_requests then ++ max_requests = DEFAULT_KEEPALIVE_MAX_REQUESTS ++ ++ elseif type(max_requests) ~= "number" then ++ error("bad argument #2 to 'enable_keepalive' " .. ++ "(number expected, got " .. type(max_requests) .. ")", 2) ++ ++ elseif max_requests < 0 then ++ error("bad argument #2 to 'enable_keepalive' (expected >= 0)", 2) ++ end ++ ++ local rc = ngx_lua_ffi_balancer_enable_keepalive(r, idle_timeout, ++ max_requests, errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end + +- return nil, ffi_str(errmsg[0]) ++else ++ function _M.enable_keepalive() ++ error("'enable_keepalive' not yet implemented in " .. subsystem .. ++ " subsystem", 2) ++ end + end + + +-- +2.25.2 diff --git a/build/openresty/patches/lua-resty-core-0.1.22_03-make-resty.core.shdict-compatible-with-m1.patch b/build/openresty/patches/lua-resty-core-0.1.22_03-make-resty.core.shdict-compatible-with-m1.patch new file mode 100644 index 000000000000..d394d5b651ee --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.22_03-make-resty.core.shdict-compatible-with-m1.patch @@ -0,0 +1,270 @@ +From 85202b4306db143de55926564bf6ce981f3631b4 Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Thu, 16 Dec 2021 19:28:43 +0200 +Subject: [PATCH] fix(shdict) make resty.core.shdict compatible with m1 (using + wrappers) + +--- + lua-resty-core-0.1.22/lib/resty/core/shdict.lua | 174 ++++++++++++++++++++++++++++++++++++++ + 1 file changed, 174 insertions(+) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/shdict.lua b/lua-resty-core-0.1.22/lib/resty/core/shdict.lua +index dedf12c..e501a38 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/shdict.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/shdict.lua +@@ -32,8 +32,11 @@ local subsystem = ngx.config.subsystem + + + local ngx_lua_ffi_shdict_get ++local ngx_lua_ffi_shdict_get_m1 + local ngx_lua_ffi_shdict_incr ++local ngx_lua_ffi_shdict_incr_m1 + local ngx_lua_ffi_shdict_store ++local ngx_lua_ffi_shdict_store_m1 + local ngx_lua_ffi_shdict_flush_all + local ngx_lua_ffi_shdict_get_ttl + local ngx_lua_ffi_shdict_set_expire +@@ -42,6 +45,53 @@ local ngx_lua_ffi_shdict_free_space + local ngx_lua_ffi_shdict_udata_to_zone + + ++local M1 = jit and jit.os == "OSX" and jit.arch == "arm64" ++if M1 then ++ ffi.cdef[[ ++typedef struct { ++ void *zone; ++ const unsigned char *key; ++ size_t key_len; ++ int *value_type; ++ unsigned char **str_value_buf; ++ size_t *str_value_len; ++ double *num_value; ++ int *user_flags; ++ int get_stale; ++ int *is_stale; ++ char **errmsg; ++} ngx_shdict_get_t; ++ ++typedef struct { ++ void *zone; ++ int op; ++ const unsigned char *key; ++ size_t key_len; ++ int value_type; ++ const unsigned char *str_value_buf; ++ size_t str_value_len; ++ double num_value; ++ long exptime; ++ int user_flags; ++ char **errmsg; ++ int *forcible; ++} ngx_shdict_store_t; ++ ++typedef struct { ++ void *zone; ++ const unsigned char *key; ++ size_t key_len; ++ double *num_value; ++ char **errmsg; ++ int has_init; ++ double init; ++ long init_ttl; ++ int *forcible; ++} ngx_shdict_incr_t; ++]] ++end ++ ++ + if subsystem == 'http' then + ffi.cdef[[ + int ngx_http_lua_ffi_shdict_get(void *zone, const unsigned char *key, +@@ -72,6 +122,18 @@ size_t ngx_http_lua_ffi_shdict_capacity(void *zone); + void *ngx_http_lua_ffi_shdict_udata_to_zone(void *zone_udata); + ]] + ++ if M1 then ++ ffi.cdef [[ ++int ngx_http_lua_ffi_shdict_get_m1(ngx_shdict_get_t *s); ++int ngx_http_lua_ffi_shdict_store_m1(ngx_shdict_store_t *s); ++int ngx_http_lua_ffi_shdict_incr_m1(ngx_shdict_incr_t *s); ++ ]] ++ ++ ngx_lua_ffi_shdict_get_m1 = C.ngx_http_lua_ffi_shdict_get_m1 ++ ngx_lua_ffi_shdict_store_m1 = C.ngx_http_lua_ffi_shdict_store_m1 ++ ngx_lua_ffi_shdict_incr_m1 = C.ngx_http_lua_ffi_shdict_incr_m1 ++ end ++ + ngx_lua_ffi_shdict_get = C.ngx_http_lua_ffi_shdict_get + ngx_lua_ffi_shdict_incr = C.ngx_http_lua_ffi_shdict_incr + ngx_lua_ffi_shdict_store = C.ngx_http_lua_ffi_shdict_store +@@ -126,6 +188,17 @@ size_t ngx_stream_lua_ffi_shdict_capacity(void *zone); + void *ngx_stream_lua_ffi_shdict_udata_to_zone(void *zone_udata); + ]] + ++ if M1 then ++ ffi.cdef [[ ++int ngx_stream_lua_ffi_shdict_get_m1(ngx_shdict_get_t *s); ++int ngx_stream_lua_ffi_shdict_store_m1(ngx_shdict_store_t *s); ++int ngx_stream_lua_ffi_shdict_incr_m1(ngx_shdict_incr_t *s); ++ ]] ++ ngx_lua_ffi_shdict_get_m1 = C.ngx_stream_lua_ffi_shdict_get_m1 ++ ngx_lua_ffi_shdict_store_m1 = C.ngx_stream_lua_ffi_shdict_store_m1 ++ ngx_lua_ffi_shdict_incr_m1 = C.ngx_stream_lua_ffi_shdict_incr_m1 ++ end ++ + ngx_lua_ffi_shdict_get = C.ngx_stream_lua_ffi_shdict_get + ngx_lua_ffi_shdict_incr = C.ngx_stream_lua_ffi_shdict_incr + ngx_lua_ffi_shdict_store = C.ngx_stream_lua_ffi_shdict_store +@@ -245,6 +318,31 @@ local function shdict_store(zone, op, key, value, exptime, flags) + return nil, "bad value type" + end + ++ local rc ++ if M1 then ++ local q = ffi_new("ngx_shdict_store_t") ++ q.zone = zone ++ q.op = op ++ q.key = key ++ q.key_len = key_len ++ q.value_type = valtyp ++ q.str_value_buf = str_val_buf ++ q.str_value_len = str_val_len ++ q.num_value = num_val ++ q.exptime = exptime * 1000 ++ q.user_flags = flags ++ q.errmsg = errmsg ++ q.forcible = forcible ++ ++ local rc = ngx_lua_ffi_shdict_store_m1(q) ++ if rc == 0 then -- NGX_OK ++ return true, nil, forcible[0] == 1 ++ end ++ ++ -- NGX_DECLINED or NGX_ERROR ++ return false, ffi_str(errmsg[0]), forcible[0] == 1 ++ end ++ + local rc = ngx_lua_ffi_shdict_store(zone, op, key, key_len, + valtyp, str_val_buf, + str_val_len, num_val, +@@ -317,6 +415,30 @@ local function shdict_get(zone, key) + local value_len = get_size_ptr() + value_len[0] = size + ++ if M1 then ++ local q = ffi_new("ngx_shdict_get_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.value_type = value_type ++ q.str_value_buf = str_value_buf ++ q.str_value_len = value_len ++ q.num_value = num_value ++ q.user_flags = user_flags ++ q.get_stale = 0 ++ q.is_stale = is_stale ++ q.errmsg = errmsg ++ ++ local rc = ngx_lua_ffi_shdict_get_m1(q) ++ if rc ~= 0 then ++ if errmsg[0] ~= nil then ++ return nil, ffi_str(errmsg[0]) ++ end ++ ++ error("failed to get the key") ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_get(zone, key, key_len, value_type, + str_value_buf, value_len, + num_value, user_flags, 0, +@@ -329,6 +451,8 @@ local function shdict_get(zone, key) + error("failed to get the key") + end + ++ end ++ + local typ = value_type[0] + + if typ == 0 then -- LUA_TNIL +@@ -392,6 +516,30 @@ local function shdict_get_stale(zone, key) + local value_len = get_size_ptr() + value_len[0] = size + ++ if M1 then ++ local q = ffi_new("ngx_shdict_get_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.value_type = value_type ++ q.str_value_buf = str_value_buf ++ q.str_value_len = value_len ++ q.num_value = num_value ++ q.user_flags = user_flags ++ q.get_stale = 1 ++ q.is_stale = is_stale ++ q.errmsg = errmsg ++ ++ local rc = ngx_lua_ffi_shdict_get_m1(q) ++ if rc ~= 0 then ++ if errmsg[0] ~= nil then ++ return nil, ffi_str(errmsg[0]) ++ end ++ ++ error("failed to get the key") ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_get(zone, key, key_len, value_type, + str_value_buf, value_len, + num_value, user_flags, 1, +@@ -404,6 +552,8 @@ local function shdict_get_stale(zone, key) + error("failed to get the key") + end + ++ end ++ + local typ = value_type[0] + + if typ == 0 then -- LUA_TNIL +@@ -498,6 +648,28 @@ local function shdict_incr(zone, key, value, init, init_ttl) + init_ttl = 0 + end + ++ if M1 then ++ local q = ffi_new("ngx_shdict_incr_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.num_value = num_value ++ q.errmsg = errmsg ++ if init then ++ q.has_init = 1 ++ q.init = init ++ else ++ q.has_init = 0 ++ end ++ q.init_ttl = init_ttl * 1000 ++ q.forcible = forcible ++ ++ local rc = ngx_lua_ffi_shdict_incr_m1(q) ++ if rc ~= 0 then -- ~= NGX_OK ++ return nil, ffi_str(errmsg[0]) ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_incr(zone, key, key_len, num_value, + errmsg, init and 1 or 0, + init or 0, init_ttl * 1000, +@@ -506,6 +678,8 @@ local function shdict_incr(zone, key, value, init, init_ttl) + return nil, ffi_str(errmsg[0]) + end + ++ end ++ + if not init then + return tonumber(num_value[0]) + end +-- +2.34.1 + diff --git a/build/openresty/patches/lua-resty-core-0.1.22_04-make-resty.core.response-compatible-with-m1.patch b/build/openresty/patches/lua-resty-core-0.1.22_04-make-resty.core.response-compatible-with-m1.patch new file mode 100644 index 000000000000..810eb285c0b2 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.22_04-make-resty.core.response-compatible-with-m1.patch @@ -0,0 +1,101 @@ +From 94efefb9aaede738ec9e29e639cf5e934e9a1d5a Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Thu, 16 Dec 2021 19:28:13 +0200 +Subject: [PATCH] fix(response) make resty.core.response compatible with m1 + (using kong wrappers) + +--- + lua-resty-core-0.1.22/lib/resty/core/response.lua | 58 +++++++++++++++++++++++++++++++++++++ + 1 file changed, 58 insertions(+) + +diff --git a/lua-resty-core-0.1.22/lib/resty/core/response.lua b/lua-resty-core-0.1.22/lib/resty/core/response.lua +index 891a07e..1efdf56 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/response.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/response.lua +@@ -45,6 +45,27 @@ ffi.cdef[[ + ]] + + ++local M1 = jit and jit.os == "OSX" and jit.arch == "arm64" ++if M1 then ++ffi.cdef[[ ++ typedef struct { ++ ngx_http_request_t *r; ++ const char *key_data; ++ size_t key_len; ++ int is_nil; ++ const char *sval; ++ size_t sval_len; ++ void *mvals; ++ size_t mvals_len; ++ int override; ++ char **errmsg; ++ } ngx_set_resp_header_t; ++ ++ int ngx_http_lua_ffi_set_resp_header_m1(ngx_set_resp_header_t *s); ++]] ++end ++ ++ + local function set_resp_header(tb, key, value, no_override) + local r = get_request() + if not r then +@@ -61,6 +82,22 @@ local function set_resp_header(tb, key, value, no_override) + error("invalid header value", 3) + end + ++ if M1 then ++ local q = ffi.new("ngx_set_resp_header_t") ++ q.r = r ++ q.key_data = key ++ q.key_len = #key ++ q.is_nil = true ++ q.sval_len = 0 ++ q.mvals_len = 0 ++ q.override = 1 ++ q.errmsg = errmsg ++ ++ rc = C.ngx_http_lua_ffi_set_resp_header_m1(q) ++ ++ goto results ++ end ++ + rc = C.ngx_http_lua_ffi_set_resp_header(r, key, #key, true, nil, 0, nil, + 0, 1, errmsg) + else +@@ -99,11 +136,32 @@ local function set_resp_header(tb, key, value, no_override) + end + + local override_int = no_override and 0 or 1 ++ ++ if M1 then ++ local s = ffi.new("ngx_set_resp_header_t") ++ s.r = r ++ s.key_data = key ++ s.key_len = #key ++ s.is_nil = false ++ s.sval = sval ++ s.sval_len = sval_len ++ s.mvals = mvals ++ s.mvals_len = mvals_len ++ s.override = override_int ++ s.errmsg = errmsg ++ ++ rc = C.ngx_http_lua_ffi_set_resp_header_m1(s) ++ ++ goto results ++ end ++ + rc = C.ngx_http_lua_ffi_set_resp_header(r, key, #key, false, sval, + sval_len, mvals, mvals_len, + override_int, errmsg) + end + ++ ::results:: ++ + if rc == 0 or rc == FFI_DECLINED then + return + end +-- +2.34.1 + diff --git a/build/openresty/patches/lua-resty-core-0.1.22_05-ngx-worker-pids.patch b/build/openresty/patches/lua-resty-core-0.1.22_05-ngx-worker-pids.patch new file mode 100644 index 000000000000..96ef6e9a26b4 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.22_05-ngx-worker-pids.patch @@ -0,0 +1,81 @@ +From 79f520183bb5b1a278d8a8be3f53659737232253 Mon Sep 17 00:00:00 2001 +From: attenuation +Date: Sun, 21 Aug 2022 22:17:30 +0800 +Subject: [PATCH] feat: add ngx.worker.pids to get all workers pid map + +--- + +diff --git a/bundle/lua-resty-core-0.1.22/lib/resty/core/worker.lua b/bundle/lua-resty-core-0.1.22/lib/resty/core/worker.lua +index c336debdb..187289786 100644 +--- a/bundle/lua-resty-core-0.1.22/lib/resty/core/worker.lua ++++ b/bundle/lua-resty-core-0.1.22/lib/resty/core/worker.lua +@@ -6,12 +6,14 @@ local base = require "resty.core.base" + + + local C = ffi.C ++local ffi_new = ffi.new + local new_tab = base.new_tab + local subsystem = ngx.config.subsystem + + + local ngx_lua_ffi_worker_id + local ngx_lua_ffi_worker_pid ++local ngx_lua_ffi_worker_pids + local ngx_lua_ffi_worker_count + local ngx_lua_ffi_worker_exiting + +@@ -23,12 +25,14 @@ if subsystem == "http" then + ffi.cdef[[ + int ngx_http_lua_ffi_worker_id(void); + int ngx_http_lua_ffi_worker_pid(void); ++ int ngx_http_lua_ffi_worker_pids(int *pids, size_t *pids_len); + int ngx_http_lua_ffi_worker_count(void); + int ngx_http_lua_ffi_worker_exiting(void); + ]] + + ngx_lua_ffi_worker_id = C.ngx_http_lua_ffi_worker_id + ngx_lua_ffi_worker_pid = C.ngx_http_lua_ffi_worker_pid ++ ngx_lua_ffi_worker_pids = C.ngx_http_lua_ffi_worker_pids + ngx_lua_ffi_worker_count = C.ngx_http_lua_ffi_worker_count + ngx_lua_ffi_worker_exiting = C.ngx_http_lua_ffi_worker_exiting + +@@ -36,12 +40,14 @@ elseif subsystem == "stream" then + ffi.cdef[[ + int ngx_stream_lua_ffi_worker_id(void); + int ngx_stream_lua_ffi_worker_pid(void); ++ int ngx_stream_lua_ffi_worker_pids(int *pids, size_t *pids_len); + int ngx_stream_lua_ffi_worker_count(void); + int ngx_stream_lua_ffi_worker_exiting(void); + ]] + + ngx_lua_ffi_worker_id = C.ngx_stream_lua_ffi_worker_id + ngx_lua_ffi_worker_pid = C.ngx_stream_lua_ffi_worker_pid ++ ngx_lua_ffi_worker_pids = C.ngx_stream_lua_ffi_worker_pids + ngx_lua_ffi_worker_count = C.ngx_stream_lua_ffi_worker_count + ngx_lua_ffi_worker_exiting = C.ngx_stream_lua_ffi_worker_exiting + end +@@ -56,6 +62,24 @@ function ngx.worker.pid() + return ngx_lua_ffi_worker_pid() + end + ++local size_ptr = ffi_new("size_t[1]") ++local pids_ptr = ffi_new("int[1024]") -- using NGX_MAX_PROCESSES ++ ++function ngx.worker.pids() ++ if ngx.get_phase() == "init" or ngx.get_phase() == "init_worker" then ++ return nil, "API disabled in the current context" ++ end ++ ++ local res = ngx_lua_ffi_worker_pids(pids_ptr, size_ptr) ++ ++ local pids = {} ++ if res == 0 then ++ for i = 1, tonumber(size_ptr[0]) do ++ pids[i] = pids_ptr[i-1] ++ end ++ end ++ return pids ++end + + function ngx.worker.id() + local id = ngx_lua_ffi_worker_id() \ No newline at end of file diff --git a/build/openresty/patches/lua-resty-core-0.1.23_01-cosocket-mtls.patch b/build/openresty/patches/lua-resty-core-0.1.23_01-cosocket-mtls.patch new file mode 100644 index 000000000000..9240a38568d3 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.23_01-cosocket-mtls.patch @@ -0,0 +1,566 @@ +From 4f0f4bf63d23a952179aaf810c10dfffc19ee835 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 20:54:30 +0800 +Subject: [PATCH 1/9] move tcp.lua into socket.lua + +--- + lib/resty/core/socket.lua | 136 +++++++++++++++++++++++++++++++++++++- + 1 file changed, 133 insertions(+), 3 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 1a504ec..cc0081e 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -6,13 +6,21 @@ local ffi = require 'ffi' + + local error = error + local tonumber = tonumber ++local tostring = tostring ++local type = type ++local select = select + local registry = debug.getregistry() ++ ++local C = ffi.C + local ffi_new = ffi.new + local ffi_string = ffi.string +-local C = ffi.C ++local ffi_gc = ffi.gc ++ + local get_string_buf = base.get_string_buf + local get_size_ptr = base.get_size_ptr +-local tostring = tostring ++local get_request = base.get_request ++ ++local co_yield = coroutine._yield + + + local option_index = { +@@ -35,15 +43,29 @@ ngx_http_lua_ffi_socket_tcp_getoption(ngx_http_lua_socket_tcp_upstream_t *u, + int + ngx_http_lua_ffi_socket_tcp_setoption(ngx_http_lua_socket_tcp_upstream_t *u, + int opt, int val, unsigned char *err, size_t *errlen); ++ ++int ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, void *sess, ++ int enable_session_reuse, ngx_str_t *server_name, int verify, ++ int ocsp_status_req, void *chain, void *pkey, char **errmsg); ++ ++int ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, void **sess, char **errmsg, ++ int *openssl_error_code); ++ ++void ngx_http_lua_ffi_ssl_free_session(void *sess); + ]] + + + local output_value_buf = ffi_new("int[1]") + local FFI_OK = base.FFI_OK ++local FFI_ERROR = base.FFI_ERROR ++local FFI_DONE = base.FFI_DONE ++local FFI_AGAIN = base.FFI_AGAIN ++local FFI_NO_REQ_CTX = base.FFI_NO_REQ_CTX + local SOCKET_CTX_INDEX = 1 + local ERR_BUF_SIZE = 4096 + +- + local function get_tcp_socket(cosocket) + local tcp_socket = cosocket[SOCKET_CTX_INDEX] + if not tcp_socket then +@@ -114,10 +136,118 @@ local function setoption(cosocket, option, value) + end + + ++local errmsg = base.get_errmsg_ptr() ++local session_ptr = ffi_new("void *[1]") ++local server_name_str = ffi_new("ngx_str_t[1]") ++local openssl_error_code = ffi_new("int[1]") ++ ++ ++local function setclientcert(self, cert, pkey) ++ if not cert and not pkey then ++ self.client_cert = nil ++ self.client_pkey = nil ++ return ++ end ++ ++ if not cert or not pkey then ++ error("client certificate must be supplied with corresponding " .. ++ "private key", 2) ++ end ++ ++ if type(cert) ~= "cdata" then ++ error("bad client cert type", 2) ++ end ++ ++ if type(pkey) ~= "cdata" then ++ error("bad client pkey type", 2) ++ end ++ ++ self.client_cert = cert ++ self.client_pkey = pkey ++end ++ ++ ++local function sslhandshake(self, reused_session, server_name, ssl_verify, ++ send_status_req, ...) ++ ++ local n = select("#", ...) ++ if not self or n > 1 then ++ error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. ++ "(including the object), but seen " .. (5 + n)) ++ end ++ ++ local r = get_request() ++ if not r then ++ error("no request found", 2) ++ end ++ ++ session_ptr[0] = type(reused_session) == "cdata" and reused_session or nil ++ ++ if server_name then ++ server_name_str[0].data = server_name ++ server_name_str[0].len = #server_name ++ ++ else ++ server_name_str[0].data = nil ++ server_name_str[0].len = 0 ++ end ++ ++ local u = self[SOCKET_CTX_INDEX] ++ ++ local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, ++ session_ptr[0], ++ reused_session ~= false, ++ server_name_str, ++ ssl_verify and 1 or 0, ++ send_status_req and 1 or 0, ++ self.client_cert, self.client_pkey, errmsg) ++ ++ if rc == FFI_NO_REQ_CTX then ++ error("no request ctx found", 2) ++ end ++ ++ while true do ++ if rc == FFI_ERROR then ++ if openssl_error_code[0] ~= 0 then ++ return nil, openssl_error_code[0] .. ": " .. ffi_string(errmsg[0]) ++ end ++ ++ return nil, ffi_string(errmsg[0]) ++ end ++ ++ if rc == FFI_DONE then ++ return reused_session ++ end ++ ++ if rc == FFI_OK then ++ if reused_session == false then ++ return true ++ end ++ ++ rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, ++ session_ptr, errmsg, openssl_error_code) ++ ++ if session_ptr[0] == nil then ++ return nil ++ end ++ ++ return ffi_gc(session_ptr[0], C.ngx_http_lua_ffi_ssl_free_session) ++ end ++ ++ co_yield() ++ ++ rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, ++ session_ptr, errmsg, openssl_error_code) ++ end ++end ++ ++ + do + local method_table = registry.__tcp_cosocket_mt + method_table.getoption = getoption + method_table.setoption = setoption ++ method_table.setclientcert = setclientcert ++ method_table.sslhandshake = sslhandshake + end + + +-- +2.32.0 (Apple Git-132) + + +From 4eab5793d741c739d9c5cfe14e0671c1d70fd6e5 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 21:37:45 +0800 +Subject: [PATCH 2/9] revert assert in sslhandshake + +--- + lib/resty/core/socket.lua | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index cc0081e..7c61d06 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -5,6 +5,7 @@ local ffi = require 'ffi' + + + local error = error ++local assert = assert + local tonumber = tonumber + local tostring = tostring + local type = type +@@ -227,6 +228,8 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, + session_ptr, errmsg, openssl_error_code) + ++ assert(rc == FFI_OK) ++ + if session_ptr[0] == nil then + return nil + end +@@ -234,6 +237,8 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + return ffi_gc(session_ptr[0], C.ngx_http_lua_ffi_ssl_free_session) + end + ++ assert(rc == FFI_AGAIN) ++ + co_yield() + + rc = C.ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(r, u, +-- +2.32.0 (Apple Git-132) + + +From 58de9a44c89f07eda98bb7fd978a9e04a244d2f2 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 28 Jan 2022 21:45:42 +0800 +Subject: [PATCH 3/9] rename ffi_string to ffi_str + +--- + lib/resty/core/socket.lua | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 7c61d06..14457da 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -14,7 +14,7 @@ local registry = debug.getregistry() + + local C = ffi.C + local ffi_new = ffi.new +-local ffi_string = ffi.string ++local ffi_str = ffi.string + local ffi_gc = ffi.gc + + local get_string_buf = base.get_string_buf +@@ -98,7 +98,7 @@ local function getoption(cosocket, option) + err, + errlen) + if rc ~= FFI_OK then +- return nil, ffi_string(err, errlen[0]) ++ return nil, ffi_str(err, errlen[0]) + end + + return tonumber(output_value_buf[0]) +@@ -130,7 +130,7 @@ local function setoption(cosocket, option, value) + err, + errlen) + if rc ~= FFI_OK then +- return nil, ffi_string(err, errlen[0]) ++ return nil, ffi_str(err, errlen[0]) + end + + return true +@@ -210,10 +210,10 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + while true do + if rc == FFI_ERROR then + if openssl_error_code[0] ~= 0 then +- return nil, openssl_error_code[0] .. ": " .. ffi_string(errmsg[0]) ++ return nil, openssl_error_code[0] .. ": " .. ffi_str(errmsg[0]) + end + +- return nil, ffi_string(errmsg[0]) ++ return nil, ffi_str(errmsg[0]) + end + + if rc == FFI_DONE then +-- +2.32.0 (Apple Git-132) + + +From ff138619432bda6b9bd4f37403c12600a4739e47 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:23:16 +0800 +Subject: [PATCH 4/9] minor style fix + +--- + lib/resty/core/socket.lua | 15 +++++++++------ + 1 file changed, 9 insertions(+), 6 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 14457da..3c882af 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -1,7 +1,7 @@ + local base = require "resty.core.base" +-base.allows_subsystem('http') +-local debug = require 'debug' +-local ffi = require 'ffi' ++base.allows_subsystem("http") ++local debug = require "debug" ++local ffi = require "ffi" + + + local error = error +@@ -45,16 +45,19 @@ int + ngx_http_lua_ffi_socket_tcp_setoption(ngx_http_lua_socket_tcp_upstream_t *u, + int opt, int val, unsigned char *err, size_t *errlen); + +-int ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, ++int ++ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, void *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, + int ocsp_status_req, void *chain, void *pkey, char **errmsg); + +-int ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, ++int ++ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, void **sess, char **errmsg, + int *openssl_error_code); + +-void ngx_http_lua_ffi_ssl_free_session(void *sess); ++void ++ngx_http_lua_ffi_ssl_free_session(void *sess); + ]] + + +-- +2.32.0 (Apple Git-132) + + +From a843a258987efba49f0b6979389f75ee32c2150c Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:28:41 +0800 +Subject: [PATCH 5/9] rename self to cosocket + +--- + lib/resty/core/socket.lua | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 3c882af..374d583 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -146,10 +146,10 @@ local server_name_str = ffi_new("ngx_str_t[1]") + local openssl_error_code = ffi_new("int[1]") + + +-local function setclientcert(self, cert, pkey) ++local function setclientcert(cosocket, cert, pkey) + if not cert and not pkey then +- self.client_cert = nil +- self.client_pkey = nil ++ cosocket.client_cert = nil ++ cosocket.client_pkey = nil + return + end + +@@ -166,16 +166,16 @@ local function setclientcert(self, cert, pkey) + error("bad client pkey type", 2) + end + +- self.client_cert = cert +- self.client_pkey = pkey ++ cosocket.client_cert = cert ++ cosocket.client_pkey = pkey + end + + +-local function sslhandshake(self, reused_session, server_name, ssl_verify, ++local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + send_status_req, ...) + + local n = select("#", ...) +- if not self or n > 1 then ++ if not cosocket or n > 1 then + error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. + "(including the object), but seen " .. (5 + n)) + end +@@ -196,7 +196,7 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + server_name_str[0].len = 0 + end + +- local u = self[SOCKET_CTX_INDEX] ++ local u = cosocket[SOCKET_CTX_INDEX] + + local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, + session_ptr[0], +@@ -204,7 +204,7 @@ local function sslhandshake(self, reused_session, server_name, ssl_verify, + server_name_str, + ssl_verify and 1 or 0, + send_status_req and 1 or 0, +- self.client_cert, self.client_pkey, errmsg) ++ cosocket.client_cert, cosocket.client_pkey, errmsg) + + if rc == FFI_NO_REQ_CTX then + error("no request ctx found", 2) +-- +2.32.0 (Apple Git-132) + + +From db95a049a019ff6f0d3b4e550412e40c25dda41f Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 07:35:04 +0800 +Subject: [PATCH 6/9] use get_tcp_socket() in sslhandshake + +--- + lib/resty/core/socket.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 374d583..ecff453 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -196,7 +196,7 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + server_name_str[0].len = 0 + end + +- local u = cosocket[SOCKET_CTX_INDEX] ++ local u = get_tcp_socket(cosocket) + + local rc = C.ngx_http_lua_ffi_socket_tcp_sslhandshake(r, u, + session_ptr[0], +-- +2.32.0 (Apple Git-132) + + +From 6767f0c2e8a73fd1a09d727431bed457c5cac4c0 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 08:58:52 +0800 +Subject: [PATCH 7/9] fix arguments check in sslhandshake + +--- + lib/resty/core/socket.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index ecff453..15e3065 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -177,7 +177,7 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + local n = select("#", ...) + if not cosocket or n > 1 then + error("ngx.socket sslhandshake: expecting 1 ~ 5 arguments " .. +- "(including the object), but seen " .. (5 + n)) ++ "(including the object), but seen " .. (cosocket and 5 + n or 0)) + end + + local r = get_request() +-- +2.32.0 (Apple Git-132) + + +From 4eeddcd2114d0097e4b9cb11f2f93d30c70d573e Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Mon, 7 Feb 2022 10:59:35 +0800 +Subject: [PATCH 8/9] setclientcert return err + +--- + lib/resty/core/socket.lua | 13 ++++++++----- + 1 file changed, 8 insertions(+), 5 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 15e3065..879d678 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -150,24 +150,27 @@ local function setclientcert(cosocket, cert, pkey) + if not cert and not pkey then + cosocket.client_cert = nil + cosocket.client_pkey = nil +- return ++ return true + end + + if not cert or not pkey then +- error("client certificate must be supplied with corresponding " .. +- "private key", 2) ++ return nil, ++ "client certificate must be supplied with corresponding " .. ++ "private key" + end + + if type(cert) ~= "cdata" then +- error("bad client cert type", 2) ++ return nil, "bad client cert type" + end + + if type(pkey) ~= "cdata" then +- error("bad client pkey type", 2) ++ return nil, "bad client pkey type" + end + + cosocket.client_cert = cert + cosocket.client_pkey = pkey ++ ++ return true + end + + +-- +2.32.0 (Apple Git-132) + + +From fead2a28f409117ad1b6c98d02edb6a38a64fde0 Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Wed, 9 Feb 2022 16:05:11 +0000 +Subject: [PATCH 9/9] fix(socket) add temporary backwards compatability for + tlshandshake + +--- + lib/resty/core/socket.lua | 22 ++++++++++++++++++++++ + 1 file changed, 22 insertions(+) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +index 879d678..448bf36 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/socket.lua +@@ -253,12 +253,34 @@ local function sslhandshake(cosocket, reused_session, server_name, ssl_verify, + end + + ++-- Temporary patch for backwards compatablity with existing Kong tech debt ++local function tlshandshake(cosocket, options) ++ local options = options or {} ++ ++ if options.client_cert then ++ local ok, err = cosocket:setclientcert(options.client_cert, options.client_priv_key) ++ if not ok then ++ return nil, err ++ end ++ end ++ ++ return sslhandshake( ++ cosocket, ++ options.reused_session, ++ options.server_name, ++ options.ssl_verify, ++ options.ocsp_status_req ++ ) ++end ++ ++ + do + local method_table = registry.__tcp_cosocket_mt + method_table.getoption = getoption + method_table.setoption = setoption + method_table.setclientcert = setclientcert + method_table.sslhandshake = sslhandshake ++ method_table.tlshandshake = tlshandshake + end + + +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/lua-resty-core-0.1.23_02-dyn_upstream_keepalive.patch b/build/openresty/patches/lua-resty-core-0.1.23_02-dyn_upstream_keepalive.patch new file mode 100644 index 000000000000..b3b5e3f66f57 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.23_02-dyn_upstream_keepalive.patch @@ -0,0 +1,230 @@ +From 37feb95041f183ae4fbafeebc47dc104995e6f27 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:44:33 -0700 +Subject: [PATCH] feature: implemented the 'balancer.enable_keepalive()' API. + +--- + lua-resty-core-0.1.23/lib/ngx/balancer.lua | 165 +++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 151 insertions(+), 14 deletions(-) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/ngx/balancer.lua b/bundle/lua-resty-core-0.1.23/lib/ngx/balancer.lua +index d584639..614312f 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/ngx/balancer.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/ngx/balancer.lua +@@ -3,6 +3,7 @@ + + local base = require "resty.core.base" + base.allows_subsystem('http', 'stream') ++require "resty.core.hash" + + + local ffi = require "ffi" +@@ -17,8 +18,10 @@ local error = error + local type = type + local tonumber = tonumber + local max = math.max ++local ngx_crc32_long = ngx.crc32_long + local subsystem = ngx.config.subsystem + local ngx_lua_ffi_balancer_set_current_peer ++local ngx_lua_ffi_balancer_enable_keepalive + local ngx_lua_ffi_balancer_set_more_tries + local ngx_lua_ffi_balancer_get_last_failure + local ngx_lua_ffi_balancer_set_timeouts -- used by both stream and http +@@ -27,7 +30,11 @@ local ngx_lua_ffi_balancer_set_timeouts -- used by both stream and http + if subsystem == 'http' then + ffi.cdef[[ + int ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, +- const unsigned char *addr, size_t addr_len, int port, char **err); ++ const unsigned char *addr, size_t addr_len, int port, ++ unsigned int cpool_crc32, unsigned int cpool_size, char **err); ++ ++ int ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, ++ unsigned long timeout, unsigned int max_requests, char **err); + + int ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + int count, char **err); +@@ -46,6 +53,9 @@ if subsystem == 'http' then + ngx_lua_ffi_balancer_set_current_peer = + C.ngx_http_lua_ffi_balancer_set_current_peer + ++ ngx_lua_ffi_balancer_enable_keepalive = ++ C.ngx_http_lua_ffi_balancer_enable_keepalive ++ + ngx_lua_ffi_balancer_set_more_tries = + C.ngx_http_lua_ffi_balancer_set_more_tries + +@@ -96,6 +106,11 @@ else + end + + ++local DEFAULT_KEEPALIVE_POOL_SIZE = 30 ++local DEFAULT_KEEPALIVE_IDLE_TIMEOUT = 60000 ++local DEFAULT_KEEPALIVE_MAX_REQUESTS = 100 ++ ++ + local peer_state_names = { + [1] = "keepalive", + [2] = "next", +@@ -106,25 +121,147 @@ local peer_state_names = { + local _M = { version = base.version } + + +-function _M.set_current_peer(addr, port) +- local r = get_request() +- if not r then +- error("no request found") ++if subsystem == "http" then ++ function _M.set_current_peer(addr, port, opts) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ local pool_crc32 ++ local pool_size ++ ++ if opts then ++ if type(opts) ~= "table" then ++ error("bad argument #3 to 'set_current_peer' " .. ++ "(table expected, got " .. type(opts) .. ")", 2) ++ end ++ ++ local pool = opts.pool ++ pool_size = opts.pool_size ++ ++ if pool then ++ if type(pool) ~= "string" then ++ error("bad option 'pool' to 'set_current_peer' " .. ++ "(string expected, got " .. type(pool) .. ")", 2) ++ end ++ ++ pool_crc32 = ngx_crc32_long(pool) ++ end ++ ++ if pool_size then ++ if type(pool_size) ~= "number" then ++ error("bad option 'pool_size' to 'set_current_peer' " .. ++ "(number expected, got " .. type(pool_size) .. ")", 2) ++ ++ elseif pool_size < 1 then ++ error("bad option 'pool_size' to 'set_current_peer' " .. ++ "(expected > 0)", 2) ++ end ++ end ++ end ++ ++ if not port then ++ port = 0 ++ ++ elseif type(port) ~= "number" then ++ port = tonumber(port) ++ end ++ ++ if not pool_crc32 then ++ pool_crc32 = 0 ++ end ++ ++ if not pool_size then ++ pool_size = DEFAULT_KEEPALIVE_POOL_SIZE ++ end ++ ++ local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, port, ++ pool_crc32, pool_size, ++ errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end + +- if not port then +- port = 0 +- elseif type(port) ~= "number" then +- port = tonumber(port) ++else ++ function _M.set_current_peer(addr, port, opts) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ if opts then ++ error("bad argument #3 to 'set_current_peer' ('opts' not yet " .. ++ "implemented in " .. subsystem .. " subsystem)", 2) ++ end ++ ++ if not port then ++ port = 0 ++ ++ elseif type(port) ~= "number" then ++ port = tonumber(port) ++ end ++ ++ local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, ++ port, errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end ++end + +- local rc = ngx_lua_ffi_balancer_set_current_peer(r, addr, #addr, +- port, errmsg) +- if rc == FFI_OK then +- return true ++ ++if subsystem == "http" then ++ function _M.enable_keepalive(idle_timeout, max_requests) ++ local r = get_request() ++ if not r then ++ error("no request found") ++ end ++ ++ if not idle_timeout then ++ idle_timeout = DEFAULT_KEEPALIVE_IDLE_TIMEOUT ++ ++ elseif type(idle_timeout) ~= "number" then ++ error("bad argument #1 to 'enable_keepalive' " .. ++ "(number expected, got " .. type(idle_timeout) .. ")", 2) ++ ++ elseif idle_timeout < 0 then ++ error("bad argument #1 to 'enable_keepalive' (expected >= 0)", 2) ++ ++ else ++ idle_timeout = idle_timeout * 1000 ++ end ++ ++ if not max_requests then ++ max_requests = DEFAULT_KEEPALIVE_MAX_REQUESTS ++ ++ elseif type(max_requests) ~= "number" then ++ error("bad argument #2 to 'enable_keepalive' " .. ++ "(number expected, got " .. type(max_requests) .. ")", 2) ++ ++ elseif max_requests < 0 then ++ error("bad argument #2 to 'enable_keepalive' (expected >= 0)", 2) ++ end ++ ++ local rc = ngx_lua_ffi_balancer_enable_keepalive(r, idle_timeout, ++ max_requests, errmsg) ++ if rc == FFI_OK then ++ return true ++ end ++ ++ return nil, ffi_str(errmsg[0]) + end + +- return nil, ffi_str(errmsg[0]) ++else ++ function _M.enable_keepalive() ++ error("'enable_keepalive' not yet implemented in " .. subsystem .. ++ " subsystem", 2) ++ end + end + + +-- +2.25.2 diff --git a/build/openresty/patches/lua-resty-core-0.1.23_03-make-resty.core.shdict-compatible-with-m1.patch b/build/openresty/patches/lua-resty-core-0.1.23_03-make-resty.core.shdict-compatible-with-m1.patch new file mode 100644 index 000000000000..e9dd92810562 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.23_03-make-resty.core.shdict-compatible-with-m1.patch @@ -0,0 +1,270 @@ +From 85202b4306db143de55926564bf6ce981f3631b4 Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Thu, 16 Dec 2021 19:28:43 +0200 +Subject: [PATCH] fix(shdict) make resty.core.shdict compatible with m1 (using + wrappers) + +--- + lua-resty-core-0.1.23/lib/resty/core/shdict.lua | 174 ++++++++++++++++++++++++++++++++++++++ + 1 file changed, 174 insertions(+) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/shdict.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/shdict.lua +index dedf12c..e501a38 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/shdict.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/shdict.lua +@@ -32,8 +32,11 @@ local subsystem = ngx.config.subsystem + + + local ngx_lua_ffi_shdict_get ++local ngx_lua_ffi_shdict_get_m1 + local ngx_lua_ffi_shdict_incr ++local ngx_lua_ffi_shdict_incr_m1 + local ngx_lua_ffi_shdict_store ++local ngx_lua_ffi_shdict_store_m1 + local ngx_lua_ffi_shdict_flush_all + local ngx_lua_ffi_shdict_get_ttl + local ngx_lua_ffi_shdict_set_expire +@@ -42,6 +45,53 @@ local ngx_lua_ffi_shdict_free_space + local ngx_lua_ffi_shdict_udata_to_zone + + ++local M1 = jit and jit.os == "OSX" and jit.arch == "arm64" ++if M1 then ++ ffi.cdef[[ ++typedef struct { ++ void *zone; ++ const unsigned char *key; ++ size_t key_len; ++ int *value_type; ++ unsigned char **str_value_buf; ++ size_t *str_value_len; ++ double *num_value; ++ int *user_flags; ++ int get_stale; ++ int *is_stale; ++ char **errmsg; ++} ngx_shdict_get_t; ++ ++typedef struct { ++ void *zone; ++ int op; ++ const unsigned char *key; ++ size_t key_len; ++ int value_type; ++ const unsigned char *str_value_buf; ++ size_t str_value_len; ++ double num_value; ++ long exptime; ++ int user_flags; ++ char **errmsg; ++ int *forcible; ++} ngx_shdict_store_t; ++ ++typedef struct { ++ void *zone; ++ const unsigned char *key; ++ size_t key_len; ++ double *num_value; ++ char **errmsg; ++ int has_init; ++ double init; ++ long init_ttl; ++ int *forcible; ++} ngx_shdict_incr_t; ++]] ++end ++ ++ + if subsystem == 'http' then + ffi.cdef[[ + int ngx_http_lua_ffi_shdict_get(void *zone, const unsigned char *key, +@@ -72,6 +122,18 @@ size_t ngx_http_lua_ffi_shdict_capacity(void *zone); + void *ngx_http_lua_ffi_shdict_udata_to_zone(void *zone_udata); + ]] + ++ if M1 then ++ ffi.cdef [[ ++int ngx_http_lua_ffi_shdict_get_m1(ngx_shdict_get_t *s); ++int ngx_http_lua_ffi_shdict_store_m1(ngx_shdict_store_t *s); ++int ngx_http_lua_ffi_shdict_incr_m1(ngx_shdict_incr_t *s); ++ ]] ++ ++ ngx_lua_ffi_shdict_get_m1 = C.ngx_http_lua_ffi_shdict_get_m1 ++ ngx_lua_ffi_shdict_store_m1 = C.ngx_http_lua_ffi_shdict_store_m1 ++ ngx_lua_ffi_shdict_incr_m1 = C.ngx_http_lua_ffi_shdict_incr_m1 ++ end ++ + ngx_lua_ffi_shdict_get = C.ngx_http_lua_ffi_shdict_get + ngx_lua_ffi_shdict_incr = C.ngx_http_lua_ffi_shdict_incr + ngx_lua_ffi_shdict_store = C.ngx_http_lua_ffi_shdict_store +@@ -126,6 +188,17 @@ size_t ngx_stream_lua_ffi_shdict_capacity(void *zone); + void *ngx_stream_lua_ffi_shdict_udata_to_zone(void *zone_udata); + ]] + ++ if M1 then ++ ffi.cdef [[ ++int ngx_stream_lua_ffi_shdict_get_m1(ngx_shdict_get_t *s); ++int ngx_stream_lua_ffi_shdict_store_m1(ngx_shdict_store_t *s); ++int ngx_stream_lua_ffi_shdict_incr_m1(ngx_shdict_incr_t *s); ++ ]] ++ ngx_lua_ffi_shdict_get_m1 = C.ngx_stream_lua_ffi_shdict_get_m1 ++ ngx_lua_ffi_shdict_store_m1 = C.ngx_stream_lua_ffi_shdict_store_m1 ++ ngx_lua_ffi_shdict_incr_m1 = C.ngx_stream_lua_ffi_shdict_incr_m1 ++ end ++ + ngx_lua_ffi_shdict_get = C.ngx_stream_lua_ffi_shdict_get + ngx_lua_ffi_shdict_incr = C.ngx_stream_lua_ffi_shdict_incr + ngx_lua_ffi_shdict_store = C.ngx_stream_lua_ffi_shdict_store +@@ -245,6 +318,31 @@ local function shdict_store(zone, op, key, value, exptime, flags) + return nil, "bad value type" + end + ++ local rc ++ if M1 then ++ local q = ffi_new("ngx_shdict_store_t") ++ q.zone = zone ++ q.op = op ++ q.key = key ++ q.key_len = key_len ++ q.value_type = valtyp ++ q.str_value_buf = str_val_buf ++ q.str_value_len = str_val_len ++ q.num_value = num_val ++ q.exptime = exptime * 1000 ++ q.user_flags = flags ++ q.errmsg = errmsg ++ q.forcible = forcible ++ ++ local rc = ngx_lua_ffi_shdict_store_m1(q) ++ if rc == 0 then -- NGX_OK ++ return true, nil, forcible[0] == 1 ++ end ++ ++ -- NGX_DECLINED or NGX_ERROR ++ return false, ffi_str(errmsg[0]), forcible[0] == 1 ++ end ++ + local rc = ngx_lua_ffi_shdict_store(zone, op, key, key_len, + valtyp, str_val_buf, + str_val_len, num_val, +@@ -317,6 +415,30 @@ local function shdict_get(zone, key) + local value_len = get_size_ptr() + value_len[0] = size + ++ if M1 then ++ local q = ffi_new("ngx_shdict_get_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.value_type = value_type ++ q.str_value_buf = str_value_buf ++ q.str_value_len = value_len ++ q.num_value = num_value ++ q.user_flags = user_flags ++ q.get_stale = 0 ++ q.is_stale = is_stale ++ q.errmsg = errmsg ++ ++ local rc = ngx_lua_ffi_shdict_get_m1(q) ++ if rc ~= 0 then ++ if errmsg[0] ~= nil then ++ return nil, ffi_str(errmsg[0]) ++ end ++ ++ error("failed to get the key") ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_get(zone, key, key_len, value_type, + str_value_buf, value_len, + num_value, user_flags, 0, +@@ -329,6 +451,8 @@ local function shdict_get(zone, key) + error("failed to get the key") + end + ++ end ++ + local typ = value_type[0] + + if typ == 0 then -- LUA_TNIL +@@ -392,6 +516,30 @@ local function shdict_get_stale(zone, key) + local value_len = get_size_ptr() + value_len[0] = size + ++ if M1 then ++ local q = ffi_new("ngx_shdict_get_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.value_type = value_type ++ q.str_value_buf = str_value_buf ++ q.str_value_len = value_len ++ q.num_value = num_value ++ q.user_flags = user_flags ++ q.get_stale = 1 ++ q.is_stale = is_stale ++ q.errmsg = errmsg ++ ++ local rc = ngx_lua_ffi_shdict_get_m1(q) ++ if rc ~= 0 then ++ if errmsg[0] ~= nil then ++ return nil, ffi_str(errmsg[0]) ++ end ++ ++ error("failed to get the key") ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_get(zone, key, key_len, value_type, + str_value_buf, value_len, + num_value, user_flags, 1, +@@ -404,6 +552,8 @@ local function shdict_get_stale(zone, key) + error("failed to get the key") + end + ++ end ++ + local typ = value_type[0] + + if typ == 0 then -- LUA_TNIL +@@ -498,6 +648,28 @@ local function shdict_incr(zone, key, value, init, init_ttl) + init_ttl = 0 + end + ++ if M1 then ++ local q = ffi_new("ngx_shdict_incr_t") ++ q.zone = zone ++ q.key = key ++ q.key_len = key_len ++ q.num_value = num_value ++ q.errmsg = errmsg ++ if init then ++ q.has_init = 1 ++ q.init = init ++ else ++ q.has_init = 0 ++ end ++ q.init_ttl = init_ttl * 1000 ++ q.forcible = forcible ++ ++ local rc = ngx_lua_ffi_shdict_incr_m1(q) ++ if rc ~= 0 then -- ~= NGX_OK ++ return nil, ffi_str(errmsg[0]) ++ end ++ else ++ + local rc = ngx_lua_ffi_shdict_incr(zone, key, key_len, num_value, + errmsg, init and 1 or 0, + init or 0, init_ttl * 1000, +@@ -506,6 +678,8 @@ local function shdict_incr(zone, key, value, init, init_ttl) + return nil, ffi_str(errmsg[0]) + end + ++ end ++ + if not init then + return tonumber(num_value[0]) + end +-- +2.34.1 + diff --git a/build/openresty/patches/lua-resty-core-0.1.23_04-make-resty.core.response-compatible-with-m1.patch b/build/openresty/patches/lua-resty-core-0.1.23_04-make-resty.core.response-compatible-with-m1.patch new file mode 100644 index 000000000000..db06d9272066 --- /dev/null +++ b/build/openresty/patches/lua-resty-core-0.1.23_04-make-resty.core.response-compatible-with-m1.patch @@ -0,0 +1,101 @@ +From 94efefb9aaede738ec9e29e639cf5e934e9a1d5a Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Thu, 16 Dec 2021 19:28:13 +0200 +Subject: [PATCH] fix(response) make resty.core.response compatible with m1 + (using kong wrappers) + +--- + lua-resty-core-0.1.23/lib/resty/core/response.lua | 58 +++++++++++++++++++++++++++++++++++++ + 1 file changed, 58 insertions(+) + +diff --git a/bundle/lua-resty-core-0.1.23/lib/resty/core/response.lua b/bundle/lua-resty-core-0.1.23/lib/resty/core/response.lua +index 891a07e..1efdf56 100644 +--- a/bundle/lua-resty-core-0.1.23/lib/resty/core/response.lua ++++ b/bundle/lua-resty-core-0.1.23/lib/resty/core/response.lua +@@ -45,6 +45,27 @@ ffi.cdef[[ + ]] + + ++local M1 = jit and jit.os == "OSX" and jit.arch == "arm64" ++if M1 then ++ffi.cdef[[ ++ typedef struct { ++ ngx_http_request_t *r; ++ const char *key_data; ++ size_t key_len; ++ int is_nil; ++ const char *sval; ++ size_t sval_len; ++ void *mvals; ++ size_t mvals_len; ++ int override; ++ char **errmsg; ++ } ngx_set_resp_header_t; ++ ++ int ngx_http_lua_ffi_set_resp_header_m1(ngx_set_resp_header_t *s); ++]] ++end ++ ++ + local function set_resp_header(tb, key, value, no_override) + local r = get_request() + if not r then +@@ -61,6 +82,22 @@ local function set_resp_header(tb, key, value, no_override) + error("invalid header value", 3) + end + ++ if M1 then ++ local q = ffi.new("ngx_set_resp_header_t") ++ q.r = r ++ q.key_data = key ++ q.key_len = #key ++ q.is_nil = true ++ q.sval_len = 0 ++ q.mvals_len = 0 ++ q.override = 1 ++ q.errmsg = errmsg ++ ++ rc = C.ngx_http_lua_ffi_set_resp_header_m1(q) ++ ++ goto results ++ end ++ + rc = C.ngx_http_lua_ffi_set_resp_header(r, key, #key, true, nil, 0, nil, + 0, 1, errmsg) + else +@@ -99,11 +136,32 @@ local function set_resp_header(tb, key, value, no_override) + end + + local override_int = no_override and 0 or 1 ++ ++ if M1 then ++ local s = ffi.new("ngx_set_resp_header_t") ++ s.r = r ++ s.key_data = key ++ s.key_len = #key ++ s.is_nil = false ++ s.sval = sval ++ s.sval_len = sval_len ++ s.mvals = mvals ++ s.mvals_len = mvals_len ++ s.override = override_int ++ s.errmsg = errmsg ++ ++ rc = C.ngx_http_lua_ffi_set_resp_header_m1(s) ++ ++ goto results ++ end ++ + rc = C.ngx_http_lua_ffi_set_resp_header(r, key, #key, false, sval, + sval_len, mvals, mvals_len, + override_int, errmsg) + end + ++ ::results:: ++ + if rc == 0 or rc == FFI_DECLINED then + return + end +-- +2.34.1 + diff --git a/build/openresty/patches/lua-resty-websocket-0.08_01-client-mtls.patch b/build/openresty/patches/lua-resty-websocket-0.08_01-client-mtls.patch new file mode 100644 index 000000000000..da796efb358c --- /dev/null +++ b/build/openresty/patches/lua-resty-websocket-0.08_01-client-mtls.patch @@ -0,0 +1,92 @@ +From 05d0832cf96c216297810cb495706c50309b8c5a Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Mon, 7 Feb 2022 11:36:25 +0000 +Subject: [PATCH 1/2] feat: add mtls client cert support + +--- + lib/resty/websocket/client.lua | 26 ++++++++++++++++++++++---- + 1 file changed, 22 insertions(+), 4 deletions(-) + +diff --git a/lua-resty-websocket-0.08/lib/resty/websocket/client.lua b/lua-resty-websocket-0.08/lib/resty/websocket/client.lua +index 067b2a5..2ec96dd 100644 +--- a/bundle/lua-resty-websocket-0.08/lib/resty/websocket/client.lua ++++ b/bundle/lua-resty-websocket-0.08/lib/resty/websocket/client.lua +@@ -98,7 +98,8 @@ function _M.connect(self, uri, opts) + path = "/" + end + +- local ssl_verify, headers, proto_header, origin_header, sock_opts = false ++ local ssl_verify, server_name, headers, proto_header, origin_header, sock_opts = false ++ local client_cert, client_priv_key + + if opts then + local protos = opts.protocols +@@ -122,11 +123,20 @@ function _M.connect(self, uri, opts) + sock_opts = { pool = pool } + end + +- if opts.ssl_verify then ++ client_cert = opts.client_cert ++ client_priv_key = opts.client_priv_key ++ ++ if client_cert then ++ assert(client_priv_key, ++ "client_priv_key must be provided with client_cert") ++ end ++ ++ if opts.ssl_verify or opts.server_name then + if not ssl_support then + return nil, "ngx_lua 0.9.11+ required for SSL sockets" + end +- ssl_verify = true ++ ssl_verify = opts.ssl_verify ++ server_name = opts.server_name or host + end + + if opts.headers then +@@ -151,7 +161,15 @@ function _M.connect(self, uri, opts) + if not ssl_support then + return nil, "ngx_lua 0.9.11+ required for SSL sockets" + end +- ok, err = sock:sslhandshake(false, host, ssl_verify) ++ ++ if client_cert then ++ ok, err = sock:setclientcert(client_cert, client_priv_key) ++ if not ok then ++ return nil, "ssl client cert failued: " .. err ++ end ++ end ++ ++ ok, err = sock:sslhandshake(false, server_name, ssl_verify) + if not ok then + return nil, "ssl handshake failed: " .. err + end +-- +2.32.0 (Apple Git-132) + + +From fcf3370eef554cd4e1791ac92c43b420d25d66a1 Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Mon, 7 Feb 2022 15:20:48 +0000 +Subject: [PATCH 2/2] fix(client) fix typo in error message + +--- + lib/resty/websocket/client.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lua-resty-websocket-0.08/lib/resty/websocket/client.lua b/lua-resty-websocket-0.08/lib/resty/websocket/client.lua +index 2ec96dd..598543f 100644 +--- a/bundle/lua-resty-websocket-0.08/lib/resty/websocket/client.lua ++++ b/bundle/lua-resty-websocket-0.08/lib/resty/websocket/client.lua +@@ -165,7 +165,7 @@ function _M.connect(self, uri, opts) + if client_cert then + ok, err = sock:setclientcert(client_cert, client_priv_key) + if not ok then +- return nil, "ssl client cert failued: " .. err ++ return nil, "ssl client cert failed: " .. err + end + end + +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/lua-resty-websocket-0.09_01-client-mtls.patch b/build/openresty/patches/lua-resty-websocket-0.09_01-client-mtls.patch new file mode 100644 index 000000000000..0b705896fb5c --- /dev/null +++ b/build/openresty/patches/lua-resty-websocket-0.09_01-client-mtls.patch @@ -0,0 +1,92 @@ +From 05d0832cf96c216297810cb495706c50309b8c5a Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Mon, 7 Feb 2022 11:36:25 +0000 +Subject: [PATCH 1/2] feat: add mtls client cert support + +--- + lib/resty/websocket/client.lua | 26 ++++++++++++++++++++++---- + 1 file changed, 22 insertions(+), 4 deletions(-) + +diff --git a/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua b/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua +index 067b2a5..2ec96dd 100644 +--- a/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua ++++ b/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua +@@ -98,7 +98,8 @@ function _M.connect(self, uri, opts) + path = "/" + end + +- local ssl_verify, headers, proto_header, origin_header, sock_opts = false ++ local ssl_verify, server_name, headers, proto_header, origin_header, sock_opts = false ++ local client_cert, client_priv_key + + if opts then + local protos = opts.protocols +@@ -122,11 +123,20 @@ function _M.connect(self, uri, opts) + sock_opts = { pool = pool } + end + +- if opts.ssl_verify then ++ client_cert = opts.client_cert ++ client_priv_key = opts.client_priv_key ++ ++ if client_cert then ++ assert(client_priv_key, ++ "client_priv_key must be provided with client_cert") ++ end ++ ++ if opts.ssl_verify or opts.server_name then + if not ssl_support then + return nil, "ngx_lua 0.9.11+ required for SSL sockets" + end +- ssl_verify = true ++ ssl_verify = opts.ssl_verify ++ server_name = opts.server_name or host + end + + if opts.headers then +@@ -151,7 +161,15 @@ function _M.connect(self, uri, opts) + if not ssl_support then + return nil, "ngx_lua 0.9.11+ required for SSL sockets" + end +- ok, err = sock:sslhandshake(false, host, ssl_verify) ++ ++ if client_cert then ++ ok, err = sock:setclientcert(client_cert, client_priv_key) ++ if not ok then ++ return nil, "ssl client cert failued: " .. err ++ end ++ end ++ ++ ok, err = sock:sslhandshake(false, server_name, ssl_verify) + if not ok then + return nil, "ssl handshake failed: " .. err + end +-- +2.32.0 (Apple Git-132) + + +From fcf3370eef554cd4e1791ac92c43b420d25d66a1 Mon Sep 17 00:00:00 2001 +From: James Hurst +Date: Mon, 7 Feb 2022 15:20:48 +0000 +Subject: [PATCH 2/2] fix(client) fix typo in error message + +--- + lib/resty/websocket/client.lua | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua b/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua +index 2ec96dd..598543f 100644 +--- a/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua ++++ b/bundle/lua-resty-websocket-0.09/lib/resty/websocket/client.lua +@@ -165,7 +165,7 @@ function _M.connect(self, uri, opts) + if client_cert then + ok, err = sock:setclientcert(client_cert, client_priv_key) + if not ok then +- return nil, "ssl client cert failued: " .. err ++ return nil, "ssl client cert failed: " .. err + end + end + +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/nginx-1.19.9_01-upstream_client_certificate_and_ssl_verify.patch b/build/openresty/patches/nginx-1.19.9_01-upstream_client_certificate_and_ssl_verify.patch new file mode 100644 index 000000000000..480092978b30 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_01-upstream_client_certificate_and_ssl_verify.patch @@ -0,0 +1,52 @@ +diff --git a/nginx-1.19.9/src/http/ngx_http_upstream.c b/nginx-1.19.9/src/http/ngx_http_upstream.c +index 90710557..539a4db9 100644 +--- a/bundle/nginx-1.19.9/src/http/ngx_http_upstream.c ++++ b/bundle/nginx-1.19.9/src/http/ngx_http_upstream.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_HTTP_LUA_KONG) ++#include ++#endif + + + #if (NGX_HTTP_CACHE) +@@ -1696,7 +1696,14 @@ + c->sendfile = 0; + u->output.sendfile = 0; + ++ ++#if (NGX_HTTP_LUA_KONG) ++ if (u->conf->ssl_server_name ++ || ngx_http_lua_kong_get_upstream_ssl_verify(r, u->conf->ssl_verify)) ++ { ++#else + if (u->conf->ssl_server_name || u->conf->ssl_verify) { ++#endif + if (ngx_http_upstream_ssl_name(r, u, c) != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); +@@ -1724,6 +1727,10 @@ ngx_http_upstream_ssl_init_connection(ngx_http_request_t *r, + } + } + ++#if (NGX_HTTP_LUA_KONG) ++ ngx_http_lua_kong_set_upstream_ssl(r, c); ++#endif ++ + r->connection->log->action = "SSL handshaking to upstream"; + + rc = ngx_ssl_handshake(c); +@@ -1773,7 +1773,11 @@ + + if (c->ssl->handshaked) { + ++#if (NGX_HTTP_LUA_KONG) ++ if (ngx_http_lua_kong_get_upstream_ssl_verify(r, u->conf->ssl_verify)) { ++#else + if (u->conf->ssl_verify) { ++#endif + rc = SSL_get_verify_result(c->ssl->connection); + + if (rc != X509_V_OK) { diff --git a/build/openresty/patches/nginx-1.19.9_02-remove-server-tokens-from-special-responses-output.patch b/build/openresty/patches/nginx-1.19.9_02-remove-server-tokens-from-special-responses-output.patch new file mode 100644 index 000000000000..e8f9e07323c8 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_02-remove-server-tokens-from-special-responses-output.patch @@ -0,0 +1,37 @@ +From 66f96c49ec4a222c4061e18aa8c3f8655b52327d Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Fri, 16 Aug 2019 13:41:49 +0300 +Subject: [PATCH] remove server tokens from special responses output + +--- + nginx-1.19.9/src/http/ngx_http_special_response.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/nginx-1.19.9/src/http/ngx_http_special_response.c b/nginx-1.19.9/src/http/ngx_http_special_response.c +index 4b8bbf5..524cc7b 100644 +--- a/bundle/nginx-1.19.9/src/http/ngx_http_special_response.c ++++ b/bundle/nginx-1.19.9/src/http/ngx_http_special_response.c +@@ -19,21 +19,18 @@ static ngx_int_t ngx_http_send_refresh(ngx_http_request_t *r); + + + static u_char ngx_http_error_full_tail[] = +-"
" NGINX_VER "
" CRLF + "" CRLF + "" CRLF + ; + + + static u_char ngx_http_error_build_tail[] = +-"
" NGINX_VER_BUILD "
" CRLF + "" CRLF + "" CRLF + ; + + + static u_char ngx_http_error_tail[] = +-"
openresty
" CRLF + "" CRLF + "" CRLF + ; +-- +2.22.0 diff --git a/build/openresty/patches/nginx-1.19.9_03-stream_proxy_ssl_disable.patch b/build/openresty/patches/nginx-1.19.9_03-stream_proxy_ssl_disable.patch new file mode 100644 index 000000000000..9053745cb894 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_03-stream_proxy_ssl_disable.patch @@ -0,0 +1,33 @@ +diff --git a/nginx-1.19.9/src/stream/ngx_stream_proxy_module.c b/nginx-1.19.9/src/stream/ngx_stream_proxy_module.c +index 09493135..fc8f8479 100644 +--- a/bundle/nginx-1.19.9/src/stream/ngx_stream_proxy_module.c ++++ b/bundle/nginx-1.19.9/src/stream/ngx_stream_proxy_module.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_STREAM_LUA_KONG) ++#include ++#endif + + + typedef struct { +@@ -812,8 +815,18 @@ ngx_stream_proxy_init_upstream(ngx_stream_session_t *s) + + #if (NGX_STREAM_SSL) + ++#if (NGX_STREAM_LUA_KONG) ++ ++ if (pc->type == SOCK_STREAM && pscf->ssl ++ && !ngx_stream_lua_kong_get_proxy_ssl_disable(s)) ++ { ++ ++#else ++ + if (pc->type == SOCK_STREAM && pscf->ssl) { + ++#endif ++ + if (u->proxy_protocol) { + if (ngx_stream_proxy_send_proxy_protocol(s) != NGX_OK) { + return; diff --git a/build/openresty/patches/nginx-1.19.9_04-grpc_authority_override.patch b/build/openresty/patches/nginx-1.19.9_04-grpc_authority_override.patch new file mode 100644 index 000000000000..6822292e60ab --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_04-grpc_authority_override.patch @@ -0,0 +1,25 @@ +diff --git a/nginx-1.19.3/src/http/modules/ngx_http_grpc_module.c b/nginx-1.19.3/src/http/modules/ngx_http_grpc_module.c +index d4af66db..10d3aaed 100644 +--- a/bundle/nginx-1.19.9/src/http/modules/ngx_http_grpc_module.c ++++ b/bundle/nginx-1.19.9/src/http/modules/ngx_http_grpc_module.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_HTTP_LUA_KONG) ++#include ++#endif + + + typedef struct { +@@ -733,6 +736,10 @@ ngx_http_grpc_create_request(ngx_http_request_t *r) + len = sizeof(ngx_http_grpc_connection_start) - 1 + + sizeof(ngx_http_grpc_frame_t); /* headers frame */ + ++#if (NGX_HTTP_LUA_KONG) ++ ngx_http_lua_kong_set_grpc_authority(r, &ctx->host); ++#endif ++ + /* :method header */ + + if (r->method == NGX_HTTP_GET || r->method == NGX_HTTP_POST) { diff --git a/build/openresty/patches/nginx-1.19.9_05-remove-server-headers-from-ngx-header-filter-module.patch b/build/openresty/patches/nginx-1.19.9_05-remove-server-headers-from-ngx-header-filter-module.patch new file mode 100644 index 000000000000..a12c3192c254 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_05-remove-server-headers-from-ngx-header-filter-module.patch @@ -0,0 +1,70 @@ +From 42a44843445e9db12a8fc5eaf1f3e10b22a0065b Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Tue, 15 Jun 2021 16:04:06 +0300 +Subject: [PATCH] remove server headers from nginx header filter module + +--- + nginx-1.19.9/src/http/ngx_http_header_filter_module.c | 34 ------------------- + 1 file changed, 34 deletions(-) + +diff --git a/nginx-1.19.9/src/http/ngx_http_header_filter_module.c b/nginx-1.19.9/src/http/ngx_http_header_filter_module.c +index ca13f2a..1a07dac 100644 +--- a/bundle/nginx-1.19.9/src/http/ngx_http_header_filter_module.c ++++ b/bundle/nginx-1.19.9/src/http/ngx_http_header_filter_module.c +@@ -46,11 +46,6 @@ ngx_module_t ngx_http_header_filter_module = { + }; + + +-static u_char ngx_http_server_string[] = "Server: openresty" CRLF; +-static u_char ngx_http_server_full_string[] = "Server: " NGINX_VER CRLF; +-static u_char ngx_http_server_build_string[] = "Server: " NGINX_VER_BUILD CRLF; +- +- + static ngx_str_t ngx_http_status_lines[] = { + + ngx_string("200 OK"), +@@ -279,18 +274,6 @@ ngx_http_header_filter(ngx_http_request_t *r) + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + +- if (r->headers_out.server == NULL) { +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- len += sizeof(ngx_http_server_full_string) - 1; +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- len += sizeof(ngx_http_server_build_string) - 1; +- +- } else { +- len += sizeof(ngx_http_server_string) - 1; +- } +- } +- + if (r->headers_out.date == NULL) { + len += sizeof("Date: Mon, 28 Sep 1970 06:00:00 GMT" CRLF) - 1; + } +@@ -448,23 +431,6 @@ ngx_http_header_filter(ngx_http_request_t *r) + } + *b->last++ = CR; *b->last++ = LF; + +- if (r->headers_out.server == NULL) { +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- p = ngx_http_server_full_string; +- len = sizeof(ngx_http_server_full_string) - 1; +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- p = ngx_http_server_build_string; +- len = sizeof(ngx_http_server_build_string) - 1; +- +- } else { +- p = ngx_http_server_string; +- len = sizeof(ngx_http_server_string) - 1; +- } +- +- b->last = ngx_cpymem(b->last, p, len); +- } +- + if (r->headers_out.date == NULL) { + b->last = ngx_cpymem(b->last, "Date: ", sizeof("Date: ") - 1); + b->last = ngx_cpymem(b->last, ngx_cached_http_time.data, +-- +2.31.1 diff --git a/build/openresty/patches/nginx-1.19.9_06-set-ssl-option-ignore-unexpected-eof.patch b/build/openresty/patches/nginx-1.19.9_06-set-ssl-option-ignore-unexpected-eof.patch new file mode 100644 index 000000000000..d42f8902a4a3 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_06-set-ssl-option-ignore-unexpected-eof.patch @@ -0,0 +1,15 @@ +diff --git a/bundle/nginx-1.19.9/src/event/ngx_event_openssl.c b/bundle/nginx-1.19.9/src/event/ngx_event_openssl.c +index 6361810..54ff114 100644 +--- a/bundle/nginx-1.19.9/src/event/ngx_event_openssl.c ++++ b/bundle/nginx-1.19.9/src/event/ngx_event_openssl.c +@@ -378,6 +378,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data) + SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_CLIENT_RENEGOTIATION); + #endif + ++#ifdef SSL_OP_IGNORE_UNEXPECTED_EOF ++ SSL_CTX_set_options(ssl->ctx, SSL_OP_IGNORE_UNEXPECTED_EOF); ++#endif ++ + #ifdef SSL_MODE_RELEASE_BUFFERS + SSL_CTX_set_mode(ssl->ctx, SSL_MODE_RELEASE_BUFFERS); + #endif diff --git a/build/openresty/patches/nginx-1.19.9_09-http2-rapid-reset-ddos-attack-cve-2023-44487.patch b/build/openresty/patches/nginx-1.19.9_09-http2-rapid-reset-ddos-attack-cve-2023-44487.patch new file mode 100644 index 000000000000..4ff6a377ac57 --- /dev/null +++ b/build/openresty/patches/nginx-1.19.9_09-http2-rapid-reset-ddos-attack-cve-2023-44487.patch @@ -0,0 +1,53 @@ +diff --git a/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.c b/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.c +index 3afa8b6..228b060 100644 +--- a/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.c ++++ b/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.c +@@ -361,6 +361,7 @@ ngx_http_v2_read_handler(ngx_event_t *rev) + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http2 read handler"); + + h2c->blocked = 1; ++ h2c->new_streams = 0; + + if (c->close) { + c->close = 0; +@@ -1321,6 +1322,14 @@ ngx_http_v2_state_headers(ngx_http_v2_connection_t *h2c, u_char *pos, + goto rst_stream; + } + ++ if (h2c->new_streams++ >= 2 * h2scf->concurrent_streams) { ++ ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0, ++ "client sent too many streams at once"); ++ ++ status = NGX_HTTP_V2_REFUSED_STREAM; ++ goto rst_stream; ++ } ++ + if (!h2c->settings_ack + && !(h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG) + && h2scf->preread_size < NGX_HTTP_V2_DEFAULT_WINDOW) +@@ -1386,6 +1395,12 @@ ngx_http_v2_state_headers(ngx_http_v2_connection_t *h2c, u_char *pos, + + rst_stream: + ++ if (h2c->refused_streams++ > ngx_max(h2scf->concurrent_streams, 100)) { ++ ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0, ++ "client sent too many refused streams"); ++ return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_NO_ERROR); ++ } ++ + if (ngx_http_v2_send_rst_stream(h2c, h2c->state.sid, status) != NGX_OK) { + return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_INTERNAL_ERROR); + } +diff --git a/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.h b/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.h +index 0eceae3..aef40bb 100644 +--- a/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.h ++++ b/bundle/nginx-1.19.9/src/http/v2/ngx_http_v2.h +@@ -124,6 +124,8 @@ struct ngx_http_v2_connection_s { + ngx_uint_t processing; + ngx_uint_t frames; + ngx_uint_t idle; ++ ngx_uint_t new_streams; ++ ngx_uint_t refused_streams; + ngx_uint_t priority_limit; + + ngx_uint_t pushing; diff --git a/build/openresty/patches/nginx-1.21.4_01-upstream_client_certificate_and_ssl_verify.patch b/build/openresty/patches/nginx-1.21.4_01-upstream_client_certificate_and_ssl_verify.patch new file mode 100644 index 000000000000..ddb98273fee4 --- /dev/null +++ b/build/openresty/patches/nginx-1.21.4_01-upstream_client_certificate_and_ssl_verify.patch @@ -0,0 +1,52 @@ +diff --git a/bundle/nginx-1.21.4/src/http/ngx_http_upstream.c b/bundle/nginx-1.21.4/src/http/ngx_http_upstream.c +index 90710557..539a4db9 100644 +--- a/bundle/nginx-1.21.4/src/http/ngx_http_upstream.c ++++ b/bundle/nginx-1.21.4/src/http/ngx_http_upstream.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_HTTP_LUA_KONG) ++#include ++#endif + + + #if (NGX_HTTP_CACHE) +@@ -1698,7 +1698,14 @@ + return; + } + ++ ++#if (NGX_HTTP_LUA_KONG) ++ if (u->conf->ssl_server_name ++ || ngx_http_lua_kong_get_upstream_ssl_verify(r, u->conf->ssl_verify)) ++ { ++#else + if (u->conf->ssl_server_name || u->conf->ssl_verify) { ++#endif + if (ngx_http_upstream_ssl_name(r, u, c) != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); +@@ -1736,6 +1739,10 @@ ngx_http_upstream_ssl_init_connection(ngx_http_request_t *r, + } + } + ++#if (NGX_HTTP_LUA_KONG) ++ ngx_http_lua_kong_set_upstream_ssl(r, c); ++#endif ++ + r->connection->log->action = "SSL handshaking to upstream"; + + rc = ngx_ssl_handshake(c); +@@ -1785,7 +1785,11 @@ + + if (c->ssl->handshaked) { + ++#if (NGX_HTTP_LUA_KONG) ++ if (ngx_http_lua_kong_get_upstream_ssl_verify(r, u->conf->ssl_verify)) { ++#else + if (u->conf->ssl_verify) { ++#endif + rc = SSL_get_verify_result(c->ssl->connection); + + if (rc != X509_V_OK) { diff --git a/build/openresty/patches/nginx-1.21.4_02-remove-server-tokens-from-special-responses-output.patch b/build/openresty/patches/nginx-1.21.4_02-remove-server-tokens-from-special-responses-output.patch new file mode 100644 index 000000000000..51143949e432 --- /dev/null +++ b/build/openresty/patches/nginx-1.21.4_02-remove-server-tokens-from-special-responses-output.patch @@ -0,0 +1,37 @@ +From 66f96c49ec4a222c4061e18aa8c3f8655b52327d Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Fri, 16 Aug 2019 13:41:49 +0300 +Subject: [PATCH] remove server tokens from special responses output + +--- + nginx-1.21.4/src/http/ngx_http_special_response.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/bundle/nginx-1.21.4/src/http/ngx_http_special_response.c b/bundle/nginx-1.21.4/src/http/ngx_http_special_response.c +index 4b8bbf5..524cc7b 100644 +--- a/bundle/nginx-1.21.4/src/http/ngx_http_special_response.c ++++ b/bundle/nginx-1.21.4/src/http/ngx_http_special_response.c +@@ -19,21 +19,18 @@ static ngx_int_t ngx_http_send_refresh(ngx_http_request_t *r); + + + static u_char ngx_http_error_full_tail[] = +-"
" NGINX_VER "
" CRLF + "" CRLF + "" CRLF + ; + + + static u_char ngx_http_error_build_tail[] = +-"
" NGINX_VER_BUILD "
" CRLF + "" CRLF + "" CRLF + ; + + + static u_char ngx_http_error_tail[] = +-"
openresty
" CRLF + "" CRLF + "" CRLF + ; +-- +2.22.0 diff --git a/build/openresty/patches/nginx-1.21.4_03-stream_upstream_client_certificate_and_ssl_verify.patch b/build/openresty/patches/nginx-1.21.4_03-stream_upstream_client_certificate_and_ssl_verify.patch new file mode 100644 index 000000000000..bc9ea8732ecc --- /dev/null +++ b/build/openresty/patches/nginx-1.21.4_03-stream_upstream_client_certificate_and_ssl_verify.patch @@ -0,0 +1,78 @@ +diff --git a/bundle/nginx-1.21.4/src/stream/ngx_stream_proxy_module.c b/bundle/nginx-1.21.4/src/stream/ngx_stream_proxy_module.c +index b11c288..4ae9e7b 100644 +--- a/bundle/nginx-1.21.4/src/stream/ngx_stream_proxy_module.c ++++ b/bundle/nginx-1.21.4/src/stream/ngx_stream_proxy_module.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_STREAM_LUA_KONG) ++#include ++#endif + + + typedef struct { +@@ -821,8 +824,18 @@ ngx_stream_proxy_init_upstream(ngx_stream_session_t *s) + + #if (NGX_STREAM_SSL) + ++#if (NGX_STREAM_LUA_KONG) ++ ++ if (pc->type == SOCK_STREAM && pscf->ssl ++ && !ngx_stream_lua_kong_get_proxy_ssl_disable(s)) ++ { ++ ++#else ++ + if (pc->type == SOCK_STREAM && pscf->ssl) { + ++#endif ++ + if (u->proxy_protocol) { + if (ngx_stream_proxy_send_proxy_protocol(s) != NGX_OK) { + return; +@@ -1085,7 +1098,16 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s) + return; + } + +- if (pscf->ssl_server_name || pscf->ssl_verify) { ++#if (NGX_STREAM_LUA_KONG) ++ ++ if (pscf->ssl_server_name || ngx_stream_lua_kong_get_upstream_ssl_verify(s, pscf->ssl_verify)) { ++ ++#else ++ ++ if (pscf->ssl_server_name || pscf->ssl_verify) { ++ ++#endif ++ + if (ngx_stream_proxy_ssl_name(s) != NGX_OK) { + ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); + return; +@@ -1110,6 +1132,10 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s) + } + } + ++#if (NGX_STREAM_LUA_KONG) ++ ngx_stream_lua_kong_set_upstream_ssl(s, pc); ++#endif ++ + s->connection->log->action = "SSL handshaking to upstream"; + + rc = ngx_ssl_handshake(pc); +@@ -1142,7 +1168,15 @@ ngx_stream_proxy_ssl_handshake(ngx_connection_t *pc) + + if (pc->ssl->handshaked) { + ++#if (NGX_STREAM_LUA_KONG) ++ ++ if (ngx_stream_lua_kong_get_upstream_ssl_verify(s, pscf->ssl_verify)) { ++ ++#else ++ + if (pscf->ssl_verify) { ++ ++#endif + rc = SSL_get_verify_result(pc->ssl->connection); + + if (rc != X509_V_OK) { diff --git a/build/openresty/patches/nginx-1.21.4_04-grpc_authority_override.patch b/build/openresty/patches/nginx-1.21.4_04-grpc_authority_override.patch new file mode 100644 index 000000000000..2f7cded85208 --- /dev/null +++ b/build/openresty/patches/nginx-1.21.4_04-grpc_authority_override.patch @@ -0,0 +1,25 @@ +diff --git a/bundle/nginx-1.19.3/src/http/modules/ngx_http_grpc_module.c b/bundle/nginx-1.19.3/src/http/modules/ngx_http_grpc_module.c +index d4af66db..10d3aaed 100644 +--- a/bundle/nginx-1.21.4/src/http/modules/ngx_http_grpc_module.c ++++ b/bundle/nginx-1.21.4/src/http/modules/ngx_http_grpc_module.c +@@ -8,6 +8,9 @@ + #include + #include + #include ++#if (NGX_HTTP_LUA_KONG) ++#include ++#endif + + + typedef struct { +@@ -731,6 +734,10 @@ ngx_http_grpc_create_request(ngx_http_request_t *r) + len = sizeof(ngx_http_grpc_connection_start) - 1 + + sizeof(ngx_http_grpc_frame_t); /* headers frame */ + ++#if (NGX_HTTP_LUA_KONG) ++ ngx_http_lua_kong_set_grpc_authority(r, &ctx->host); ++#endif ++ + /* :method header */ + + if (r->method == NGX_HTTP_GET || r->method == NGX_HTTP_POST) { diff --git a/build/openresty/patches/nginx-1.21.4_05-remove-server-headers-from-ngx-header-filter-module.patch b/build/openresty/patches/nginx-1.21.4_05-remove-server-headers-from-ngx-header-filter-module.patch new file mode 100644 index 000000000000..e76dcb87fbdb --- /dev/null +++ b/build/openresty/patches/nginx-1.21.4_05-remove-server-headers-from-ngx-header-filter-module.patch @@ -0,0 +1,70 @@ +From 42a44843445e9db12a8fc5eaf1f3e10b22a0065b Mon Sep 17 00:00:00 2001 +From: Aapo Talvensaari +Date: Tue, 15 Jun 2021 16:04:06 +0300 +Subject: [PATCH] remove server headers from nginx header filter module + +--- + nginx-1.21.4/src/http/ngx_http_header_filter_module.c | 34 ------------------- + 1 file changed, 34 deletions(-) + +diff --git a/bundle/nginx-1.21.4/src/http/ngx_http_header_filter_module.c b/bundle/nginx-1.21.4/src/http/ngx_http_header_filter_module.c +index ca13f2a..1a07dac 100644 +--- a/bundle/nginx-1.21.4/src/http/ngx_http_header_filter_module.c ++++ b/bundle/nginx-1.21.4/src/http/ngx_http_header_filter_module.c +@@ -46,11 +46,6 @@ ngx_module_t ngx_http_header_filter_module = { + }; + + +-static u_char ngx_http_server_string[] = "Server: openresty" CRLF; +-static u_char ngx_http_server_full_string[] = "Server: " NGINX_VER CRLF; +-static u_char ngx_http_server_build_string[] = "Server: " NGINX_VER_BUILD CRLF; +- +- + static ngx_str_t ngx_http_status_lines[] = { + + ngx_string("200 OK"), +@@ -279,18 +274,6 @@ ngx_http_header_filter(ngx_http_request_t *r) + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + +- if (r->headers_out.server == NULL) { +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- len += sizeof(ngx_http_server_full_string) - 1; +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- len += sizeof(ngx_http_server_build_string) - 1; +- +- } else { +- len += sizeof(ngx_http_server_string) - 1; +- } +- } +- + if (r->headers_out.date == NULL) { + len += sizeof("Date: Mon, 28 Sep 1970 06:00:00 GMT" CRLF) - 1; + } +@@ -448,23 +431,6 @@ ngx_http_header_filter(ngx_http_request_t *r) + } + *b->last++ = CR; *b->last++ = LF; + +- if (r->headers_out.server == NULL) { +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- p = ngx_http_server_full_string; +- len = sizeof(ngx_http_server_full_string) - 1; +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- p = ngx_http_server_build_string; +- len = sizeof(ngx_http_server_build_string) - 1; +- +- } else { +- p = ngx_http_server_string; +- len = sizeof(ngx_http_server_string) - 1; +- } +- +- b->last = ngx_cpymem(b->last, p, len); +- } +- + if (r->headers_out.date == NULL) { + b->last = ngx_cpymem(b->last, "Date: ", sizeof("Date: ") - 1); + b->last = ngx_cpymem(b->last, ngx_cached_http_time.data, +-- +2.31.1 diff --git a/build/openresty/patches/nginx-cross-endianness-fix.patch b/build/openresty/patches/nginx-cross-endianness-fix.patch new file mode 100644 index 000000000000..da3d67457055 --- /dev/null +++ b/build/openresty/patches/nginx-cross-endianness-fix.patch @@ -0,0 +1,79 @@ +# http://cgit.openembedded.org/meta-openembedded/tree/meta-webserver/recipes-httpd/nginx/files/0001-Allow-the-overriding-of-the-endianness-via-the-confi.patch +From be9970aa16c5142ef814531d74a07990a8e9eb14 Mon Sep 17 00:00:00 2001 +From: Derek Straka +Date: Fri, 1 Dec 2017 10:32:29 -0500 +Subject: [PATCH] Allow the overriding of the endianness via the configure flag + --with-endian + +The existing configure options contain the --with-endian; however, the command +line flag does not actually function. It does not set the endianness and it +appears to do nothing. + +Upstream-Status: Pending + +Signed-off-by: Derek Straka + +diff --git a/auto/endianness b/auto/endianness +index 1b552b6..be84487 100644 +--- a/bundle/nginx-1.19.9/endianness ++++ b/bundle/nginx-1.19.9/auto/endianness +@@ -13,7 +13,13 @@ checking for system byte ordering + END + + +-cat << END > $NGX_AUTOTEST.c ++if [ ".$NGX_WITH_ENDIAN" = ".little" ]; then ++ echo " little endian" ++ have=NGX_HAVE_LITTLE_ENDIAN . auto/have ++elif [ ".$NGX_WITH_ENDIAN" = ".big" ]; then ++ echo " big endian" ++else ++ cat << END > $NGX_AUTOTEST.c + + int main(void) { + int i = 0x11223344; +@@ -26,25 +32,26 @@ int main(void) { + + END + +-ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS \ +- -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_LD_OPT $ngx_feature_libs" ++ ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS \ ++ -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_LD_OPT $ngx_feature_libs" + +-eval "$ngx_test >> $NGX_AUTOCONF_ERR 2>&1" ++ eval "$ngx_test >> $NGX_AUTOCONF_ERR 2>&1" + +-if [ -x $NGX_AUTOTEST ]; then +- if $NGX_AUTOTEST >/dev/null 2>&1; then +- echo " little endian" +- have=NGX_HAVE_LITTLE_ENDIAN . auto/have +- else +- echo " big endian" +- fi ++ if [ -x $NGX_AUTOTEST ]; then ++ if $NGX_AUTOTEST >/dev/null 2>&1; then ++ echo " little endian" ++ have=NGX_HAVE_LITTLE_ENDIAN . auto/have ++ else ++ echo " big endian" ++ fi + +- rm -rf $NGX_AUTOTEST* ++ rm -rf $NGX_AUTOTEST* + +-else +- rm -rf $NGX_AUTOTEST* ++ else ++ rm -rf $NGX_AUTOTEST* + +- echo +- echo "$0: error: cannot detect system byte ordering" +- exit 1 ++ echo ++ echo "$0: error: cannot detect system byte ordering" ++ exit 1 ++ fi + fi +-- +2.7.4 \ No newline at end of file diff --git a/build/openresty/patches/nginx-cross.patch b/build/openresty/patches/nginx-cross.patch new file mode 100644 index 000000000000..53abdfdb1517 --- /dev/null +++ b/build/openresty/patches/nginx-cross.patch @@ -0,0 +1,214 @@ +Rebased from http://cgit.openembedded.org/meta-openembedded/tree/meta-webserver/recipes-httpd/nginx/files/nginx-cross.patch + + +=================================================================== +diff --git a/bundle/nginx-1.21.4/auto/feature b/bundle/nginx-1.21.4/auto/feature +index 3561f59..d6a2889 100644 +--- a/bundle/nginx-1.21.4/auto/feature ++++ b/bundle/nginx-1.21.4/auto/feature +@@ -49,12 +49,20 @@ eval "/bin/sh -c \"$ngx_test\" >> $NGX_AUTOCONF_ERR 2>&1" + + if [ -x $NGX_AUTOTEST ]; then + ++ if [ ".$NGX_CROSS_COMPILE" = ".yes" ]; then ++ NGX_AUTOTEST_EXEC="true" ++ NGX_FOUND_MSG=" (not tested, cross compiling)" ++ else ++ NGX_AUTOTEST_EXEC="$NGX_AUTOTEST" ++ NGX_FOUND_MSG="" ++ fi ++ + case "$ngx_feature_run" in + + yes) + # /bin/sh is used to intercept "Killed" or "Abort trap" messages +- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then +- echo " found" ++ if /bin/sh -c $NGX_AUTOTEST_EXEC >> $NGX_AUTOCONF_ERR 2>&1; then ++ echo " found$NGX_FOUND_MSG" + ngx_found=yes + + if test -n "$ngx_feature_name"; then +@@ -68,17 +76,27 @@ if [ -x $NGX_AUTOTEST ]; then + + value) + # /bin/sh is used to intercept "Killed" or "Abort trap" messages +- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then +- echo " found" ++ if /bin/sh -c $NGX_AUTOTEST_EXEC >> $NGX_AUTOCONF_ERR 2>&1; then ++ echo " found$NGX_FOUND_MSG" + ngx_found=yes + +- cat << END >> $NGX_AUTO_CONFIG_H ++ if [ ".$NGX_CROSS_COMPILE" = ".yes" ]; then ++ cat << END >> $NGX_AUTO_CONFIG_H + + #ifndef $ngx_feature_name +-#define $ngx_feature_name `$NGX_AUTOTEST` ++#define $ngx_feature_name $(eval "echo \$NGX_WITH_${ngx_feature_name}") + #endif + + END ++ else ++ cat << END >> $NGX_AUTO_CONFIG_H ++ ++#ifndef $ngx_feature_name ++#define $ngx_feature_name `$NGX_AUTOTEST_EXEC` ++#endif ++ ++END ++ fi + else + echo " found but is not working" + fi +@@ -86,7 +104,7 @@ END + + bug) + # /bin/sh is used to intercept "Killed" or "Abort trap" messages +- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then ++ if /bin/sh -c $NGX_AUTOTEST_EXEC >> $NGX_AUTOCONF_ERR 2>&1; then + echo " not found" + + else +diff --git a/bundle/nginx-1.21.4/auto/options b/bundle/nginx-1.21.4/auto/options +index 182c799..e9eb7b8 100644 +--- a/bundle/nginx-1.21.4/auto/options ++++ b/bundle/nginx-1.21.4/auto/options +@@ -400,6 +400,18 @@ $0: warning: the \"--with-sha1-asm\" option is deprecated" + --test-build-epoll) NGX_TEST_BUILD_EPOLL=YES ;; + --test-build-solaris-sendfilev) NGX_TEST_BUILD_SOLARIS_SENDFILEV=YES ;; + ++ # cross compile support ++ --with-int=*) NGX_WITH_INT="$value" ;; ++ --with-long=*) NGX_WITH_LONG="$value" ;; ++ --with-long-long=*) NGX_WITH_LONG_LONG="$value" ;; ++ --with-ptr-size=*) NGX_WITH_PTR_SIZE="$value" ;; ++ --with-sig-atomic-t=*) NGX_WITH_SIG_ATOMIC_T="$value" ;; ++ --with-size-t=*) NGX_WITH_SIZE_T="$value" ;; ++ --with-off-t=*) NGX_WITH_OFF_T="$value" ;; ++ --with-time-t=*) NGX_WITH_TIME_T="$value" ;; ++ --with-sys-nerr=*) NGX_WITH_NGX_SYS_NERR="$value" ;; ++ --with-endian=*) NGX_WITH_ENDIAN="$value" ;; ++ + *) + echo "$0: error: invalid option \"$option\"" + exit 1 +@@ -590,6 +602,17 @@ cat << END + + --with-debug enable debug logging + ++ --with-int=VALUE force int size ++ --with-long=VALUE force long size ++ --with-long-long=VALUE force long long size ++ --with-ptr-size=VALUE force pointer size ++ --with-sig-atomic-t=VALUE force sig_atomic_t size ++ --with-size-t=VALUE force size_t size ++ --with-off-t=VALUE force off_t size ++ --with-time-t=VALUE force time_t size ++ --with-sys-nerr=VALUE force sys_nerr value ++ --with-endian=VALUE force system endianess ++ + END + + exit 1 +@@ -598,6 +621,8 @@ fi + + if [ ".$NGX_PLATFORM" = ".win32" ]; then + NGX_WINE=$WINE ++elif [ ! -z "$NGX_PLATFORM" ]; then ++ NGX_CROSS_COMPILE="yes" + fi + + +diff --git a/bundle/nginx-1.21.4/auto/types/sizeof b/bundle/nginx-1.21.4/auto/types/sizeof +index 480d8cf..23c5171 100644 +--- a/bundle/nginx-1.21.4/auto/types/sizeof ++++ b/bundle/nginx-1.21.4/auto/types/sizeof +@@ -12,9 +12,12 @@ checking for $ngx_type size + + END + +-ngx_size= ++ngx_size=$(eval "echo \$NGX_WITH_${ngx_param}") + +-cat << END > $NGX_AUTOTEST.c ++if [ ".$ngx_size" != "." ]; then ++ echo " $ngx_size bytes" ++else ++ cat << END > $NGX_AUTOTEST.c + + #include + #include +@@ -33,15 +36,16 @@ int main(void) { + END + + +-ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS \ +- -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_LD_OPT $ngx_feature_libs" ++ ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS \ ++ -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_LD_OPT $ngx_feature_libs" + +-eval "$ngx_test >> $NGX_AUTOCONF_ERR 2>&1" ++ eval "$ngx_test >> $NGX_AUTOCONF_ERR 2>&1" + + +-if [ -x $NGX_AUTOTEST ]; then +- ngx_size=`$NGX_AUTOTEST` +- echo " $ngx_size bytes" ++ if [ -x $NGX_AUTOTEST ]; then ++ ngx_size=`$NGX_AUTOTEST` ++ echo " $ngx_size bytes" ++ fi + fi + + +diff --git a/bundle/nginx-1.21.4/auto/unix b/bundle/nginx-1.21.4/auto/unix +index b41c70f..febbf3c 100644 +--- a/bundle/nginx-1.21.4/auto/unix ++++ b/bundle/nginx-1.21.4/auto/unix +@@ -592,13 +592,13 @@ ngx_feature_libs= + + # C types + +-ngx_type="int"; . auto/types/sizeof ++ngx_type="int"; ngx_param="INT"; . auto/types/sizeof + +-ngx_type="long"; . auto/types/sizeof ++ngx_type="long"; ngx_param="LONG"; . auto/types/sizeof + +-ngx_type="long long"; . auto/types/sizeof ++ngx_type="long long"; ngx_param="LONG_LONG"; . auto/types/sizeof + +-ngx_type="void *"; . auto/types/sizeof; ngx_ptr_size=$ngx_size ++ngx_type="void *"; ngx_param="PTR_SIZE"; . auto/types/sizeof; ngx_ptr_size=$ngx_size + ngx_param=NGX_PTR_SIZE; ngx_value=$ngx_size; . auto/types/value + + +@@ -609,7 +609,7 @@ NGX_INCLUDE_AUTO_CONFIG_H="#include \"ngx_auto_config.h\"" + ngx_type="uint32_t"; ngx_types="u_int32_t"; . auto/types/typedef + ngx_type="uint64_t"; ngx_types="u_int64_t"; . auto/types/typedef + +-ngx_type="sig_atomic_t"; ngx_types="int"; . auto/types/typedef ++ngx_type="sig_atomic_t"; ngx_param="SIG_ATOMIC_T"; ngx_types="int"; . auto/types/typedef + . auto/types/sizeof + ngx_param=NGX_SIG_ATOMIC_T_SIZE; ngx_value=$ngx_size; . auto/types/value + +@@ -625,15 +625,15 @@ ngx_type="rlim_t"; ngx_types="int"; . auto/types/typedef + + . auto/endianness + +-ngx_type="size_t"; . auto/types/sizeof ++ngx_type="size_t"; ngx_param="SIZE_T"; . auto/types/sizeof + ngx_param=NGX_MAX_SIZE_T_VALUE; ngx_value=$ngx_max_value; . auto/types/value + ngx_param=NGX_SIZE_T_LEN; ngx_value=$ngx_max_len; . auto/types/value + +-ngx_type="off_t"; . auto/types/sizeof ++ngx_type="off_t"; ngx_param="OFF_T"; . auto/types/sizeof + ngx_param=NGX_MAX_OFF_T_VALUE; ngx_value=$ngx_max_value; . auto/types/value + ngx_param=NGX_OFF_T_LEN; ngx_value=$ngx_max_len; . auto/types/value + +-ngx_type="time_t"; . auto/types/sizeof ++ngx_type="time_t"; ngx_param="TIME_T"; . auto/types/sizeof + ngx_param=NGX_TIME_T_SIZE; ngx_value=$ngx_size; . auto/types/value + ngx_param=NGX_TIME_T_LEN; ngx_value=$ngx_max_len; . auto/types/value + ngx_param=NGX_MAX_TIME_T_VALUE; ngx_value=$ngx_max_value; . auto/types/value diff --git a/build/openresty/patches/ngx_lua-0.10.20_01-cosocket-mtls.patch b/build/openresty/patches/ngx_lua-0.10.20_01-cosocket-mtls.patch new file mode 100644 index 000000000000..f339a76d9b99 --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.20_01-cosocket-mtls.patch @@ -0,0 +1,1554 @@ +From acd53645754ce42b436cecb0d7a10b547d41fef6 Mon Sep 17 00:00:00 2001 +From: lijunlong +Date: Wed, 13 Oct 2021 23:42:17 +0800 +Subject: [PATCH 01/17] bugfix: nginx crash when resolve an not exist domain in + thread create by ngx.thread.spawn.(#1931) + +FIX #1915 +The resolve ctxes were dded to a link list. function ngx_resolver_process_a iterate through the link list when got the DNS reply. +When processing the first resolve ctx, all the three ctxes were freed. So when ngx_resolver_process_a continues to process the second ctx, it will free the ctx again. + +Co-authored-by: doujiang24 +--- + src/ngx_http_lua_socket_tcp.c | 15 ++------------- + 1 file changed, 2 insertions(+), 13 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 55bd203d..b7c3bdd4 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -147,8 +147,6 @@ static void ngx_http_lua_socket_free_pool(ngx_log_t *log, + static int ngx_http_lua_socket_shutdown_pool(lua_State *L); + static void ngx_http_lua_socket_shutdown_pool_helper( + ngx_http_lua_socket_pool_t *spool); +-static void +- ngx_http_lua_socket_empty_resolve_handler(ngx_resolver_ctx_t *ctx); + static int ngx_http_lua_socket_prepare_error_retvals(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L, ngx_uint_t ft_type); + #if (NGX_HTTP_SSL) +@@ -1151,13 +1149,6 @@ ngx_http_lua_socket_tcp_connect(lua_State *L) + } + + +-static void +-ngx_http_lua_socket_empty_resolve_handler(ngx_resolver_ctx_t *ctx) +-{ +- /* do nothing */ +-} +- +- + static void + ngx_http_lua_socket_resolve_handler(ngx_resolver_ctx_t *ctx) + { +@@ -6084,10 +6075,8 @@ ngx_http_lua_tcp_resolve_cleanup(void *data) + return; + } + +- /* just to be safer */ +- rctx->handler = ngx_http_lua_socket_empty_resolve_handler; +- +- ngx_resolve_name_done(rctx); ++ /* postpone free the rctx in the handler */ ++ rctx->handler = ngx_resolve_name_done; + } + + +-- +2.32.0 (Apple Git-132) + + +From 59d39ca2f0963695052c2593f957053f1a1779a2 Mon Sep 17 00:00:00 2001 +From: Josh Soref <2119212+jsoref@users.noreply.github.com> +Date: Mon, 25 Oct 2021 03:07:01 -0400 +Subject: [PATCH 02/17] doc: fixed spelling errors in the docs and code. + (#1947) + +--- + src/ngx_http_lua_socket_tcp.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index b7c3bdd4..ace18a0f 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -3112,7 +3112,7 @@ ngx_http_lua_socket_tcp_settimeout(lua_State *L) + n = lua_gettop(L); + + if (n != 2) { +- return luaL_error(L, "ngx.socket settimout: expecting 2 arguments " ++ return luaL_error(L, "ngx.socket settimeout: expecting 2 arguments " + "(including the object) but seen %d", lua_gettop(L)); + } + +@@ -3159,7 +3159,7 @@ ngx_http_lua_socket_tcp_settimeouts(lua_State *L) + n = lua_gettop(L); + + if (n != 4) { +- return luaL_error(L, "ngx.socket settimout: expecting 4 arguments " ++ return luaL_error(L, "ngx.socket settimeout: expecting 4 arguments " + "(including the object) but seen %d", lua_gettop(L)); + } + +-- +2.32.0 (Apple Git-132) + + +From 617cb5dadc14dddb4796d4fff8821dae325f4229 Mon Sep 17 00:00:00 2001 +From: Sharp Liu +Date: Tue, 26 Oct 2021 17:53:54 +0800 +Subject: [PATCH 03/17] style: removed extra space. (#1952) + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index ace18a0f..26467fdd 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -2825,7 +2825,7 @@ ngx_http_lua_socket_tcp_send(lua_State *L) + + switch (type) { + case LUA_TNUMBER: +- b->last = ngx_http_lua_write_num(L, 2, b->last); ++ b->last = ngx_http_lua_write_num(L, 2, b->last); + break; + + case LUA_TSTRING: +-- +2.32.0 (Apple Git-132) + + +From 287d58810c450f912a8d31a94a1c86ccc039c0e1 Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 16:39:05 -0700 +Subject: [PATCH 04/17] cosocket: add function `tcpsock:tlshandshake`, retired + the Lua C API based `tcpsock:sslhandshake` implementation. + +--- + src/ngx_http_lua_socket_tcp.c | 387 +++++++++++++++------------------- + src/ngx_http_lua_socket_tcp.h | 3 + + 2 files changed, 177 insertions(+), 213 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 26467fdd..4ef22c11 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -23,6 +23,9 @@ static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) + static int ngx_http_lua_socket_tcp_sslhandshake(lua_State *L); ++static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); ++static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); + #endif + static int ngx_http_lua_socket_tcp_receive(lua_State *L); + static int ngx_http_lua_socket_tcp_receiveany(lua_State *L); +@@ -149,12 +152,6 @@ static void ngx_http_lua_socket_shutdown_pool_helper( + ngx_http_lua_socket_pool_t *spool); + static int ngx_http_lua_socket_prepare_error_retvals(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L, ngx_uint_t ft_type); +-#if (NGX_HTTP_SSL) +-static int ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, +- ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); +-static void ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c); +-static int ngx_http_lua_ssl_free_session(lua_State *L); +-#endif + static void ngx_http_lua_socket_tcp_close_connection(ngx_connection_t *c); + + +@@ -324,13 +321,6 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + lua_pushcfunction(L, ngx_http_lua_socket_tcp_connect); + lua_setfield(L, -2, "connect"); + +-#if (NGX_HTTP_SSL) +- +- lua_pushcfunction(L, ngx_http_lua_socket_tcp_sslhandshake); +- lua_setfield(L, -2, "sslhandshake"); +- +-#endif +- + lua_pushcfunction(L, ngx_http_lua_socket_tcp_receive); + lua_setfield(L, -2, "receive"); + +@@ -404,19 +394,6 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + lua_setfield(L, -2, "__gc"); + lua_rawset(L, LUA_REGISTRYINDEX); + /* }}} */ +- +-#if (NGX_HTTP_SSL) +- +- /* {{{ssl session userdata metatable */ +- lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( +- ssl_session_metatable_key)); +- lua_createtable(L, 0 /* narr */, 1 /* nrec */); /* metatable */ +- lua_pushcfunction(L, ngx_http_lua_ssl_free_session); +- lua_setfield(L, -2, "__gc"); +- lua_rawset(L, LUA_REGISTRYINDEX); +- /* }}} */ +- +-#endif + } + + +@@ -1559,64 +1536,69 @@ ngx_http_lua_socket_conn_error_retval_handler(ngx_http_request_t *r, + + #if (NGX_HTTP_SSL) + +-static int +-ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) ++static const char * ++ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, unsigned int ops) + { +- int n, top; +- ngx_int_t rc; +- ngx_str_t name = ngx_null_string; +- ngx_connection_t *c; +- ngx_ssl_session_t **psession; +- ngx_http_request_t *r; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_co_ctx_t *coctx; +- +- ngx_http_lua_socket_tcp_upstream_t *u; +- +- /* Lua function arguments: self [,session] [,host] [,verify] +- [,send_status_req] */ ++ if (ops & SOCKET_OP_CONNECT && u->conn_waiting) { ++ return "socket busy connecting"; ++ } + +- n = lua_gettop(L); +- if (n < 1 || n > 5) { +- return luaL_error(L, "ngx.socket sslhandshake: expecting 1 ~ 5 " +- "arguments (including the object), but seen %d", n); ++ if (ops & SOCKET_OP_READ && u->read_waiting) { ++ return "socket busy reading"; + } + +- r = ngx_http_lua_get_req(L); +- if (r == NULL) { +- return luaL_error(L, "no request found"); ++ if (ops & SOCKET_OP_WRITE ++ && (u->write_waiting ++ || (u->raw_downstream ++ && (r->connection->buffered & NGX_HTTP_LOWLEVEL_BUFFERED)))) ++ { ++ return "socket busy writing"; + } + +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tcp socket ssl handshake"); ++ return NULL; ++} + +- luaL_checktype(L, 1, LUA_TTABLE); ++int ++ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, ++ int enable_session_reuse, ngx_str_t *server_name, int verify, ++ int ocsp_status_req, const char **errmsg) ++{ ++ ngx_int_t rc; ++ ngx_connection_t *c; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_co_ctx_t *coctx; ++ const char *busy_rc; + +- lua_rawgeti(L, 1, SOCKET_CTX_INDEX); +- u = lua_touserdata(L, -1); ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua tcp socket tls handshake"); + + if (u == NULL + || u->peer.connection == NULL + || u->read_closed + || u->write_closed) + { +- lua_pushnil(L); +- lua_pushliteral(L, "closed"); +- return 2; ++ *errmsg = "closed"; ++ return NGX_ERROR; + } + + if (u->request != r) { +- return luaL_error(L, "bad request"); ++ *errmsg = "bad request"; ++ return NGX_ERROR; + } + +- ngx_http_lua_socket_check_busy_connecting(r, u, L); +- ngx_http_lua_socket_check_busy_reading(r, u, L); +- ngx_http_lua_socket_check_busy_writing(r, u, L); ++ busy_rc = ngx_http_lua_socket_tcp_check_busy(r, u, SOCKET_OP_CONNECT ++ | SOCKET_OP_READ ++ | SOCKET_OP_WRITE); ++ if (busy_rc != NULL) { ++ *errmsg = busy_rc; ++ return NGX_ERROR; ++ } + + if (u->raw_downstream || u->body_downstream) { +- lua_pushnil(L); +- lua_pushliteral(L, "not supported for downstream"); +- return 2; ++ *errmsg = "not supported for downstream"; ++ return NGX_ERROR; + } + + c = u->peer.connection; +@@ -1624,122 +1606,96 @@ ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) + u->ssl_session_reuse = 1; + + if (c->ssl && c->ssl->handshaked) { +- switch (lua_type(L, 2)) { +- case LUA_TUSERDATA: +- lua_pushvalue(L, 2); +- break; ++ if (sess != NULL) { ++ return NGX_DONE; ++ } + +- case LUA_TBOOLEAN: +- if (!lua_toboolean(L, 2)) { +- /* avoid generating the ssl session */ +- lua_pushboolean(L, 1); +- break; +- } +- /* fall through */ ++ u->ssl_session_reuse = enable_session_reuse; + +- default: +- ngx_http_lua_ssl_handshake_retval_handler(r, u, L); +- break; +- } ++ (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); + +- return 1; ++ return NGX_OK; + } + + if (ngx_ssl_create_connection(u->conf->ssl, c, + NGX_SSL_BUFFER|NGX_SSL_CLIENT) + != NGX_OK) + { +- lua_pushnil(L); +- lua_pushliteral(L, "failed to create ssl connection"); +- return 2; ++ *errmsg = "failed to create ssl connection"; ++ return NGX_ERROR; + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { +- return luaL_error(L, "no ctx found"); ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "no ngx_lua ctx found while TLS handshaking"); ++ ++ ngx_http_lua_assert(NULL); ++ ++ *errmsg = "no ctx found"; ++ return NGX_ERROR; + } + + coctx = ctx->cur_co_ctx; + + c->sendfile = 0; + +- if (n >= 2) { +- if (lua_type(L, 2) == LUA_TBOOLEAN) { +- u->ssl_session_reuse = lua_toboolean(L, 2); +- +- } else { +- psession = lua_touserdata(L, 2); +- +- if (psession != NULL && *psession != NULL) { +- if (ngx_ssl_set_session(c, *psession) != NGX_OK) { +- lua_pushnil(L); +- lua_pushliteral(L, "lua ssl set session failed"); +- return 2; +- } +- +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua ssl set session: %p", *psession); +- } ++ if (sess != NULL) { ++ if (ngx_ssl_set_session(c, sess) != NGX_OK) { ++ *errmsg = "lua tls set session failed"; ++ return NGX_ERROR; + } + +- if (n >= 3) { +- name.data = (u_char *) lua_tolstring(L, 3, &name.len); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "lua tls set session: %p", sess); + +- if (name.data) { +- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua ssl server name: \"%*s\"", name.len, +- name.data); ++ } else { ++ u->ssl_session_reuse = enable_session_reuse; ++ } + +-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME ++ if (server_name != NULL && server_name->data != NULL) { ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua tls server name: \"%V\"", server_name); + +- if (SSL_set_tlsext_host_name(c->ssl->connection, +- (char *) name.data) +- == 0) +- { +- lua_pushnil(L); +- lua_pushliteral(L, "SSL_set_tlsext_host_name failed"); +- return 2; +- } ++#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME ++ if (SSL_set_tlsext_host_name(c->ssl->connection, ++ (char *) server_name->data) ++ == 0) ++ { ++ *errmsg = "SSL_set_tlsext_host_name failed"; ++ return NGX_ERROR; ++ } + + #else +- +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua socket SNI disabled because the current " +- "version of OpenSSL lacks the support"); +- ++ *errmsg = "OpenSSL has no SNI support"; ++ return NGX_ERROR; + #endif +- } ++ } + +- if (n >= 4) { +- u->ssl_verify = lua_toboolean(L, 4); ++ u->ssl_verify = verify; + +- if (n >= 5) { +- if (lua_toboolean(L, 5)) { ++ if (ocsp_status_req) { + #ifdef NGX_HTTP_LUA_USE_OCSP +- SSL_set_tlsext_status_type(c->ssl->connection, +- TLSEXT_STATUSTYPE_ocsp); ++ SSL_set_tlsext_status_type(c->ssl->connection, ++ TLSEXT_STATUSTYPE_ocsp); ++ + #else +- return luaL_error(L, "no OCSP support"); ++ *errmsg = "no OCSP support"; ++ return NGX_ERROR; + #endif +- } +- } +- } +- } + } + +- dd("found sni name: %.*s %p", (int) name.len, name.data, name.data); +- +- if (name.len == 0) { ++ if (server_name->len == 0) { + u->ssl_name.len = 0; + + } else { + if (u->ssl_name.data) { + /* buffer already allocated */ + +- if (u->ssl_name.len >= name.len) { ++ if (u->ssl_name.len >= server_name->len) { + /* reuse it */ +- ngx_memcpy(u->ssl_name.data, name.data, name.len); +- u->ssl_name.len = name.len; ++ ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ u->ssl_name.len = server_name->len; + + } else { + ngx_free(u->ssl_name.data); +@@ -1750,17 +1706,16 @@ ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) + + new_ssl_name: + +- u->ssl_name.data = ngx_alloc(name.len, ngx_cycle->log); ++ u->ssl_name.data = ngx_alloc(server_name->len, ngx_cycle->log); + if (u->ssl_name.data == NULL) { + u->ssl_name.len = 0; + +- lua_pushnil(L); +- lua_pushliteral(L, "no memory"); +- return 2; ++ *errmsg = "no memory"; ++ return NGX_ERROR; + } + +- ngx_memcpy(u->ssl_name.data, name.data, name.len); +- u->ssl_name.len = name.len; ++ ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ u->ssl_name.len = server_name->len; + } + } + +@@ -1774,7 +1729,8 @@ new_ssl_name: + + rc = ngx_ssl_handshake(c); + +- dd("ngx_ssl_handshake returned %d", (int) rc); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "ngx_ssl_handshake returned %d", rc); + + if (rc == NGX_AGAIN) { + if (c->write->timer_set) { +@@ -1784,13 +1740,13 @@ new_ssl_name: + ngx_add_timer(c->read, u->connect_timeout); + + u->conn_waiting = 1; +- u->write_prepare_retvals = ngx_http_lua_ssl_handshake_retval_handler; ++ u->write_prepare_retvals = ngx_http_lua_tls_handshake_retval_handler; + + ngx_http_lua_cleanup_pending_operation(coctx); + coctx->cleanup = ngx_http_lua_coctx_cleanup; + coctx->data = u; + +- c->ssl->handler = ngx_http_lua_ssl_handshake_handler; ++ c->ssl->handler = ngx_http_lua_tls_handshake_handler; + + if (ctx->entered_content_phase) { + r->write_event_handler = ngx_http_lua_content_wev_handler; +@@ -1799,21 +1755,25 @@ new_ssl_name: + r->write_event_handler = ngx_http_core_run_phases; + } + +- return lua_yield(L, 0); ++ return NGX_AGAIN; ++ } ++ ++ ngx_http_lua_tls_handshake_handler(c); ++ ++ if (rc == NGX_ERROR) { ++ *errmsg = u->error_ret; ++ ++ return NGX_ERROR; + } + +- top = lua_gettop(L); +- ngx_http_lua_ssl_handshake_handler(c); +- return lua_gettop(L) - top; ++ return NGX_OK; + } + + + static void +-ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) ++ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + { +- const char *err; + int waiting; +- lua_State *L; + ngx_int_t rc; + ngx_connection_t *dc; /* downstream connection */ + ngx_http_request_t *r; +@@ -1836,11 +1796,9 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + waiting = u->conn_waiting; + + dc = r->connection; +- L = u->write_co_ctx->co; + + if (c->read->timedout) { +- lua_pushnil(L); +- lua_pushliteral(L, "timeout"); ++ u->error_ret = "timeout"; + goto failed; + } + +@@ -1849,19 +1807,18 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + } + + if (c->ssl->handshaked) { +- + if (u->ssl_verify) { + rc = SSL_get_verify_result(c->ssl->connection); + + if (rc != X509_V_OK) { +- lua_pushnil(L); +- err = lua_pushfstring(L, "%d: %s", (int) rc, +- X509_verify_cert_error_string(rc)); ++ u->error_ret = X509_verify_cert_error_string(rc); ++ u->openssl_error_code_ret = rc; + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " +- "certificate verify error: (%s)", err); ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ "certificate verify error: (%d: %s)", ++ rc, u->error_ret); + } + + goto failed; +@@ -1872,12 +1829,11 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + if (u->ssl_name.len + && ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) + { +- lua_pushnil(L); +- lua_pushliteral(L, "certificate host mismatch"); ++ u->error_ret = "certificate host mismatch"; + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " + "certificate does not match host \"%V\"", + &u->ssl_name); + } +@@ -1892,7 +1848,7 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + ngx_http_lua_socket_handle_conn_success(r, u); + + } else { +- (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, L); ++ (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); + } + + if (waiting) { +@@ -1902,60 +1858,84 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + return; + } + +- lua_pushnil(L); +- lua_pushliteral(L, "handshake failed"); ++ u->error_ret = "handshake failed"; + + failed: + + if (waiting) { + u->write_prepare_retvals = +- ngx_http_lua_socket_conn_error_retval_handler; +- ngx_http_lua_socket_handle_conn_error(r, u, +- NGX_HTTP_LUA_SOCKET_FT_SSL); ++ ngx_http_lua_socket_conn_error_retval_handler; ++ ngx_http_lua_socket_handle_conn_error(r, u, NGX_HTTP_LUA_SOCKET_FT_SSL); + ngx_http_run_posted_requests(dc); + + } else { +- (void) ngx_http_lua_socket_conn_error_retval_handler(r, u, L); ++ u->ft_type |= NGX_HTTP_LUA_SOCKET_FT_SSL; ++ ++ (void) ngx_http_lua_socket_conn_error_retval_handler(r, u, NULL); ++ } ++} ++ ++ ++ ++int ++ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, ++ const char **errmsg, int *openssl_error_code) ++{ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua cosocket get TLS handshake result for upstream: %p", u); ++ ++ if (u->error_ret != NULL) { ++ *errmsg = u->error_ret; ++ *openssl_error_code = u->openssl_error_code_ret; ++ ++ return NGX_ERROR; + } ++ ++ *sess = u->ssl_session_ret; ++ ++ return NGX_OK; + } + + + static int +-ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, ++ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L) + { + ngx_connection_t *c; +- ngx_ssl_session_t *ssl_session, **ud; ++ ngx_ssl_session_t *ssl_session; + + if (!u->ssl_session_reuse) { +- lua_pushboolean(L, 1); +- return 1; ++ return 0; + } + +- ud = lua_newuserdata(L, sizeof(ngx_ssl_session_t *)); +- + c = u->peer.connection; + + ssl_session = ngx_ssl_get_session(c); + if (ssl_session == NULL) { +- *ud = NULL; ++ u->ssl_session_ret = NULL; + + } else { +- *ud = ssl_session; ++ u->ssl_session_ret = ssl_session; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua ssl save session: %p", ssl_session); +- +- /* set up the __gc metamethod */ +- lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( +- ssl_session_metatable_key)); +- lua_rawget(L, LUA_REGISTRYINDEX); +- lua_setmetatable(L, -2); ++ "lua tls save session: %p", ssl_session); + } + +- return 1; ++ return 0; ++} ++ ++ ++void ++ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) ++{ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, ++ "lua tls free session: %p", sess); ++ ++ ngx_ssl_free_session(sess); + } + ++ + #endif /* NGX_HTTP_SSL */ + + +@@ -2008,12 +1988,14 @@ ngx_http_lua_socket_prepare_error_retvals(ngx_http_request_t *r, + u_char errstr[NGX_MAX_ERROR_STR]; + u_char *p; + +- if (ft_type & (NGX_HTTP_LUA_SOCKET_FT_RESOLVER +- | NGX_HTTP_LUA_SOCKET_FT_SSL)) +- { ++ if (ft_type & NGX_HTTP_LUA_SOCKET_FT_RESOLVER) { + return 2; + } + ++ if (ft_type & NGX_HTTP_LUA_SOCKET_FT_SSL) { ++ return 0; ++ } ++ + lua_pushnil(L); + + if (ft_type & NGX_HTTP_LUA_SOCKET_FT_TIMEOUT) { +@@ -6101,27 +6083,6 @@ ngx_http_lua_coctx_cleanup(void *data) + } + + +-#if (NGX_HTTP_SSL) +- +-static int +-ngx_http_lua_ssl_free_session(lua_State *L) +-{ +- ngx_ssl_session_t **psession; +- +- psession = lua_touserdata(L, 1); +- if (psession && *psession != NULL) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, +- "lua ssl free session: %p", *psession); +- +- ngx_ssl_free_session(*psession); +- } +- +- return 0; +-} +- +-#endif /* NGX_HTTP_SSL */ +- +- + void + ngx_http_lua_cleanup_conn_pools(lua_State *L) + { +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.h b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.h +index a0a5a518..ee9411bc 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.h ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.h +@@ -120,6 +120,9 @@ struct ngx_http_lua_socket_tcp_upstream_s { + + #if (NGX_HTTP_SSL) + ngx_str_t ssl_name; ++ ngx_ssl_session_t *ssl_session_ret; ++ const char *error_ret; ++ int openssl_error_code_ret; + #endif + + unsigned ft_type:16; +-- +2.32.0 (Apple Git-132) + + +From f5ba21d6f742e6b169d972a81b6124b27c076016 Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 16:54:32 -0700 +Subject: [PATCH 05/17] change: better error when request context couldn't be + found. + +--- + src/ngx_http_lua_socket_tcp.c | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 4ef22c11..abd487fa 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -1627,13 +1627,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { +- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, +- "no ngx_lua ctx found while TLS handshaking"); +- +- ngx_http_lua_assert(NULL); +- +- *errmsg = "no ctx found"; +- return NGX_ERROR; ++ return NGX_HTTP_LUA_FFI_NO_REQ_CTX; + } + + coctx = ctx->cur_co_ctx; +-- +2.32.0 (Apple Git-132) + + +From 78a450d571febf7ba918ecc13369144925d02bcb Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 17:24:07 -0700 +Subject: [PATCH 06/17] feature: TCP cosocket client certificate support. + closes #534 + +--- + src/ngx_http_lua_socket_tcp.c | 60 +++++++++++++++++++++++++++++++---- + 1 file changed, 54 insertions(+), 6 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index abd487fa..61671b70 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -22,7 +22,6 @@ + static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) +-static int ngx_http_lua_socket_tcp_sslhandshake(lua_State *L); + static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); + static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); +@@ -219,9 +218,6 @@ static char ngx_http_lua_upstream_udata_metatable_key; + static char ngx_http_lua_downstream_udata_metatable_key; + static char ngx_http_lua_pool_udata_metatable_key; + static char ngx_http_lua_pattern_udata_metatable_key; +-#if (NGX_HTTP_SSL) +-static char ngx_http_lua_ssl_session_metatable_key; +-#endif + + + #define ngx_http_lua_tcp_socket_metatable_literal_key "__tcp_cosocket_mt" +@@ -1563,13 +1559,16 @@ int + ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, +- int ocsp_status_req, const char **errmsg) ++ int ocsp_status_req, STACK_OF(X509) *chain, EVP_PKEY *pkey, ++ const char **errmsg) + { +- ngx_int_t rc; ++ ngx_int_t rc, i; + ngx_connection_t *c; + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_co_ctx_t *coctx; + const char *busy_rc; ++ ngx_ssl_conn_t *ssl_conn; ++ X509 *x509; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "lua tcp socket tls handshake"); +@@ -1625,6 +1624,8 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ ssl_conn = c->ssl->connection; ++ + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return NGX_HTTP_LUA_FFI_NO_REQ_CTX; +@@ -1647,6 +1648,53 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + u->ssl_session_reuse = enable_session_reuse; + } + ++ if (chain != NULL) { ++ ngx_http_lua_assert(pkey != NULL); /* ensured by resty.core */ ++ ++ if (sk_X509_num(chain) < 1) { ++ ERR_clear_error(); ++ *errmsg = "invalid client certificate chain"; ++ return NGX_ERROR; ++ } ++ ++ x509 = sk_X509_value(chain, 0); ++ if (x509 == NULL) { ++ ERR_clear_error(); ++ *errmsg = "lua tls fetch client certificate from chain failed"; ++ return NGX_ERROR; ++ } ++ ++ if (SSL_use_certificate(ssl_conn, x509) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua tls set client certificate failed"; ++ return NGX_ERROR; ++ } ++ ++ /* read rest of the chain */ ++ ++ for (i = 1; i < sk_X509_num(chain); i++) { ++ x509 = sk_X509_value(chain, i); ++ if (x509 == NULL) { ++ ERR_clear_error(); ++ *errmsg = "lua tls fetch client intermediate certificate " ++ "from chain failed"; ++ return NGX_ERROR; ++ } ++ ++ if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua tls set client intermediate certificate failed"; ++ return NGX_ERROR; ++ } ++ } ++ ++ if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua ssl set client private key failed"; ++ return NGX_ERROR; ++ } ++ } ++ + if (server_name != NULL && server_name->data != NULL) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "lua tls server name: \"%V\"", server_name); +-- +2.32.0 (Apple Git-132) + + +From 6cc0c89e946ef42adfbc55e8a461ccc2f367254a Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 17:25:20 -0700 +Subject: [PATCH 07/17] style: style fixes. + +--- + src/ngx_http_lua_socket_tcp.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 61671b70..a7d410c9 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -1736,7 +1736,8 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (u->ssl_name.len >= server_name->len) { + /* reuse it */ +- ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ ngx_memcpy(u->ssl_name.data, server_name->data, ++ server_name->len); + u->ssl_name.len = server_name->len; + + } else { +-- +2.32.0 (Apple Git-132) + + +From 21cd7779252732a02fa0e596b66a1d4663d2fd64 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Mon, 6 Jan 2020 17:56:10 -0800 +Subject: [PATCH 08/17] cleanup + +--- + src/ngx_http_lua_socket_tcp.c | 24 +++++++++++------------- + 1 file changed, 11 insertions(+), 13 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index a7d410c9..bd7cc7ca 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -1555,6 +1555,7 @@ ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + return NULL; + } + ++ + int + ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, +@@ -1596,7 +1597,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + } + + if (u->raw_downstream || u->body_downstream) { +- *errmsg = "not supported for downstream"; ++ *errmsg = "not supported for downstream sockets"; + return NGX_ERROR; + } + +@@ -1637,7 +1638,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (sess != NULL) { + if (ngx_ssl_set_session(c, sess) != NGX_OK) { +- *errmsg = "lua tls set session failed"; ++ *errmsg = "tls set session failed"; + return NGX_ERROR; + } + +@@ -1660,13 +1661,13 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, 0); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "lua tls fetch client certificate from chain failed"; ++ *errmsg = "tls fetch client certificate from chain failed"; + return NGX_ERROR; + } + + if (SSL_use_certificate(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "lua tls set client certificate failed"; ++ *errmsg = "tls set client certificate failed"; + return NGX_ERROR; + } + +@@ -1676,21 +1677,21 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, i); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "lua tls fetch client intermediate certificate " +- "from chain failed"; ++ *errmsg = "tls fetch client intermediate certificate from " ++ "chain failed"; + return NGX_ERROR; + } + + if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "lua tls set client intermediate certificate failed"; ++ *errmsg = "tls set client intermediate certificate failed"; + return NGX_ERROR; + } + } + + if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { + ERR_clear_error(); +- *errmsg = "lua ssl set client private key failed"; ++ *errmsg = "tls set client private key failed"; + return NGX_ERROR; + } + } +@@ -1709,7 +1710,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + } + + #else +- *errmsg = "OpenSSL has no SNI support"; ++ *errmsg = "no TLS extension support"; + return NGX_ERROR; + #endif + } +@@ -1752,7 +1753,6 @@ new_ssl_name: + u->ssl_name.data = ngx_alloc(server_name->len, ngx_cycle->log); + if (u->ssl_name.data == NULL) { + u->ssl_name.len = 0; +- + *errmsg = "no memory"; + return NGX_ERROR; + } +@@ -1773,7 +1773,7 @@ new_ssl_name: + rc = ngx_ssl_handshake(c); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "ngx_ssl_handshake returned %d", rc); ++ "ngx_ssl_handshake returned: %d", rc); + + if (rc == NGX_AGAIN) { + if (c->write->timer_set) { +@@ -1805,7 +1805,6 @@ new_ssl_name: + + if (rc == NGX_ERROR) { + *errmsg = u->error_ret; +- + return NGX_ERROR; + } + +@@ -1919,7 +1918,6 @@ failed: + } + + +- + int + ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, +-- +2.32.0 (Apple Git-132) + + +From 0bcf4d1a955db9218e8b0e50685c1d0de8c90b9a Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Tue, 24 Nov 2020 01:49:28 -0800 +Subject: [PATCH 09/17] fixed style according to @spacewander's review + +--- + src/ngx_http_lua_socket_tcp.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index bd7cc7ca..1aa37627 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -1536,15 +1536,15 @@ static const char * + ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, unsigned int ops) + { +- if (ops & SOCKET_OP_CONNECT && u->conn_waiting) { ++ if ((ops & SOCKET_OP_CONNECT) && u->conn_waiting) { + return "socket busy connecting"; + } + +- if (ops & SOCKET_OP_READ && u->read_waiting) { ++ if ((ops & SOCKET_OP_READ) && u->read_waiting) { + return "socket busy reading"; + } + +- if (ops & SOCKET_OP_WRITE ++ if ((ops & SOCKET_OP_WRITE) + && (u->write_waiting + || (u->raw_downstream + && (r->connection->buffered & NGX_HTTP_LOWLEVEL_BUFFERED)))) +-- +2.32.0 (Apple Git-132) + + +From 9b010940f77bbd486c1192eed23af7c35baf4cdb Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 13:42:06 +0800 +Subject: [PATCH 10/17] resize tcp_socket_metatable to 7 + +--- + src/ngx_http_lua_socket_tcp.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 1aa37627..7cdc45c4 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -160,6 +160,8 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, ++ SOCKET_CLIENT_CERT_INDEX = 6, ++ SOCKET_CLIENT_KEY_INDEX = 7, + }; + + +@@ -424,7 +426,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 5 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 7 /* narr */, 1 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 36245613be1031b22b0e6b2eec398dac288fe9a5 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:12:13 +0800 +Subject: [PATCH 11/17] change errms tls to ssl + +--- + src/ngx_http_lua_socket_tcp.c | 24 ++++++++++++------------ + 1 file changed, 12 insertions(+), 12 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 7cdc45c4..af986364 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -1574,7 +1574,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + X509 *x509; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tcp socket tls handshake"); ++ "lua tcp socket ssl handshake"); + + if (u == NULL + || u->peer.connection == NULL +@@ -1640,12 +1640,12 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (sess != NULL) { + if (ngx_ssl_set_session(c, sess) != NGX_OK) { +- *errmsg = "tls set session failed"; ++ *errmsg = "ssl set session failed"; + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua tls set session: %p", sess); ++ "lua ssl set session: %p", sess); + + } else { + u->ssl_session_reuse = enable_session_reuse; +@@ -1663,13 +1663,13 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, 0); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "tls fetch client certificate from chain failed"; ++ *errmsg = "ssl fetch client certificate from chain failed"; + return NGX_ERROR; + } + + if (SSL_use_certificate(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client certificate failed"; ++ *errmsg = "ssl set client certificate failed"; + return NGX_ERROR; + } + +@@ -1679,28 +1679,28 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, i); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "tls fetch client intermediate certificate from " ++ *errmsg = "ssl fetch client intermediate certificate from " + "chain failed"; + return NGX_ERROR; + } + + if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client intermediate certificate failed"; ++ *errmsg = "ssl set client intermediate certificate failed"; + return NGX_ERROR; + } + } + + if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client private key failed"; ++ *errmsg = "ssl set client private key failed"; + return NGX_ERROR; + } + } + + if (server_name != NULL && server_name->data != NULL) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tls server name: \"%V\"", server_name); ++ "lua ssl server name: \"%V\"", server_name); + + #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME + if (SSL_set_tlsext_host_name(c->ssl->connection, +@@ -1926,7 +1926,7 @@ ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + const char **errmsg, int *openssl_error_code) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua cosocket get TLS handshake result for upstream: %p", u); ++ "lua cosocket get SSL handshake result for upstream: %p", u); + + if (u->error_ret != NULL) { + *errmsg = u->error_ret; +@@ -1962,7 +1962,7 @@ ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + u->ssl_session_ret = ssl_session; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua tls save session: %p", ssl_session); ++ "lua ssl save session: %p", ssl_session); + } + + return 0; +@@ -1973,7 +1973,7 @@ void + ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, +- "lua tls free session: %p", sess); ++ "lua ssl free session: %p", sess); + + ngx_ssl_free_session(sess); + } +-- +2.32.0 (Apple Git-132) + + +From 1f12b89485da6b7ac5dd23810bf094f214dc324e Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:38:49 +0800 +Subject: [PATCH 12/17] rename function name from tls to ssl + +--- + src/ngx_http_lua_socket_tcp.c | 28 ++++++++++++++-------------- + 1 file changed, 14 insertions(+), 14 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index af986364..76e98597 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -22,8 +22,8 @@ + static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) +-static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); +-static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++static void ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c); ++static int ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); + #endif + static int ngx_http_lua_socket_tcp_receive(lua_State *L); +@@ -1559,7 +1559,7 @@ ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + + + int +-ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, ++ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, + int ocsp_status_req, STACK_OF(X509) *chain, EVP_PKEY *pkey, +@@ -1614,7 +1614,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + u->ssl_session_reuse = enable_session_reuse; + +- (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); ++ (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, NULL); + + return NGX_OK; + } +@@ -1785,13 +1785,13 @@ new_ssl_name: + ngx_add_timer(c->read, u->connect_timeout); + + u->conn_waiting = 1; +- u->write_prepare_retvals = ngx_http_lua_tls_handshake_retval_handler; ++ u->write_prepare_retvals = ngx_http_lua_ssl_handshake_retval_handler; + + ngx_http_lua_cleanup_pending_operation(coctx); + coctx->cleanup = ngx_http_lua_coctx_cleanup; + coctx->data = u; + +- c->ssl->handler = ngx_http_lua_tls_handshake_handler; ++ c->ssl->handler = ngx_http_lua_ssl_handshake_handler; + + if (ctx->entered_content_phase) { + r->write_event_handler = ngx_http_lua_content_wev_handler; +@@ -1803,7 +1803,7 @@ new_ssl_name: + return NGX_AGAIN; + } + +- ngx_http_lua_tls_handshake_handler(c); ++ ngx_http_lua_ssl_handshake_handler(c); + + if (rc == NGX_ERROR) { + *errmsg = u->error_ret; +@@ -1815,7 +1815,7 @@ new_ssl_name: + + + static void +-ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) ++ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + { + int waiting; + ngx_int_t rc; +@@ -1860,7 +1860,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " + "certificate verify error: (%d: %s)", + rc, u->error_ret); + } +@@ -1877,7 +1877,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " + "certificate does not match host \"%V\"", + &u->ssl_name); + } +@@ -1892,7 +1892,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + ngx_http_lua_socket_handle_conn_success(r, u); + + } else { +- (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); ++ (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, NULL); + } + + if (waiting) { +@@ -1921,7 +1921,7 @@ failed: + + + int +-ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, ++ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, + const char **errmsg, int *openssl_error_code) + { +@@ -1942,7 +1942,7 @@ ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + + + static int +-ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L) + { + ngx_connection_t *c; +@@ -1970,7 +1970,7 @@ ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + + + void +-ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) ++ngx_http_lua_ffi_ssl_free_session(ngx_ssl_session_t *sess) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "lua ssl free session: %p", sess); +-- +2.32.0 (Apple Git-132) + + +From 84242561aa54ffed3bfab433cfef6f7797e01a47 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:46:38 +0800 +Subject: [PATCH 13/17] rename to SOCKET_CLIENT_PRIV_INDEX + +--- + src/ngx_http_lua_socket_tcp.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 76e98597..90da45fc 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -160,8 +160,8 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, +- SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_KEY_INDEX = 7, ++ SOCKET_CLIENT_CERT_INDEX = 6, ++ SOCKET_CLIENT_PRIV_INDEX = 7, + }; + + +-- +2.32.0 (Apple Git-132) + + +From 555166646c525167f9e1e5bb81b6cb100a4834f9 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:49:18 +0800 +Subject: [PATCH 14/17] rename to SOCKET_CLIENT_PKEY_INDEX + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 90da45fc..494486de 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -161,7 +161,7 @@ enum { + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, + SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_PRIV_INDEX = 7, ++ SOCKET_CLIENT_PKEY_INDEX = 7, + }; + + +-- +2.32.0 (Apple Git-132) + + +From e9b54c43c05b064b831fe67d0e0aaff45b2ec505 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 17:17:09 +0800 +Subject: [PATCH 15/17] need not to change tcp_socket_metatable + +--- + src/ngx_http_lua_socket_tcp.c | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 494486de..152d8cbd 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -160,8 +160,6 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, +- SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_PKEY_INDEX = 7, + }; + + +@@ -426,7 +424,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 7 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 5 /* narr */, 1 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 6c47356ddc327a8692260bd6f43ea67cf2787a73 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Wed, 26 Jan 2022 19:55:29 +0800 +Subject: [PATCH 16/17] increase nrec to 3 in the socket object + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 152d8cbd..8d71f8b4 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -424,7 +424,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 5 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 5 /* narr */, 3 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 1d538552c7629310d850d4360408ddb555afcbcc Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 09:18:52 +0800 +Subject: [PATCH 17/17] change tcp_socket_metatable nrec to 15 + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c b/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +index 8d71f8b4..5dcdef0e 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_tcp.c +@@ -312,7 +312,7 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + /* {{{tcp object metatable */ + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); +- lua_createtable(L, 0 /* narr */, 14 /* nrec */); ++ lua_createtable(L, 0 /* narr */, 15 /* nrec */); + + lua_pushcfunction(L, ngx_http_lua_socket_tcp_connect); + lua_setfield(L, -2, "connect"); +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/ngx_lua-0.10.20_02-dyn_upstream_keepalive.patch b/build/openresty/patches/ngx_lua-0.10.20_02-dyn_upstream_keepalive.patch new file mode 100644 index 000000000000..effdd5b517b5 --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.20_02-dyn_upstream_keepalive.patch @@ -0,0 +1,1319 @@ +From 2d12ac3e4045258b7a174b0505d92f63c26d82fc Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:44 -0700 +Subject: [PATCH 1/3] feature: implemented keepalive pooling in + 'balancer_by_lua*'. + +--- + src/ngx_http_lua_balancer.c | 738 ++++++++++++++++++++++++++++++------ + src/ngx_http_lua_common.h | 4 + + src/ngx_http_lua_module.c | 3 + + 3 files changed, 629 insertions(+), 116 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c b/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +index f71a3e00..0d403716 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +@@ -16,46 +16,102 @@ + #include "ngx_http_lua_directive.h" + + ++typedef struct { ++ ngx_uint_t size; ++ ngx_uint_t connections; ++ ++ uint32_t crc32; ++ ++ lua_State *lua_vm; ++ ++ ngx_queue_t cache; ++ ngx_queue_t free; ++} ngx_http_lua_balancer_keepalive_pool_t; ++ ++ ++typedef struct { ++ ngx_queue_t queue; ++ ngx_connection_t *connection; ++ ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++} ngx_http_lua_balancer_keepalive_item_t; ++ ++ + struct ngx_http_lua_balancer_peer_data_s { +- /* the round robin data must be first */ +- ngx_http_upstream_rr_peer_data_t rrp; ++ ngx_uint_t cpool_size; ++ ngx_uint_t keepalive_requests; ++ ngx_msec_t keepalive_timeout; ++ ++ ngx_uint_t more_tries; ++ ngx_uint_t total_tries; + +- ngx_http_lua_srv_conf_t *conf; +- ngx_http_request_t *request; ++ int last_peer_state; + +- ngx_uint_t more_tries; +- ngx_uint_t total_tries; ++ uint32_t cpool_crc32; + +- struct sockaddr *sockaddr; +- socklen_t socklen; ++ void *data; + +- ngx_str_t *host; +- in_port_t port; ++ ngx_event_get_peer_pt original_get_peer; ++ ngx_event_free_peer_pt original_free_peer; + +- int last_peer_state; ++#if (NGX_HTTP_SSL) ++ ngx_event_set_peer_session_pt original_set_session; ++ ngx_event_save_peer_session_pt original_save_session; ++#endif ++ ++ ngx_http_request_t *request; ++ ngx_http_lua_srv_conf_t *conf; ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++ ++ ngx_str_t *host; ++ ++ struct sockaddr *sockaddr; ++ socklen_t socklen; ++ ++ unsigned keepalive:1; + + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) +- unsigned cloned_upstream_conf; /* :1 */ ++ unsigned cloned_upstream_conf:1; + #endif + }; + + +-#if (NGX_HTTP_SSL) +-static ngx_int_t ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, +- void *data); +-static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, +- void *data); +-#endif ++static ngx_int_t ngx_http_lua_balancer_by_chunk(lua_State *L, ++ ngx_http_request_t *r); + static ngx_int_t ngx_http_lua_balancer_init(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us); + static ngx_int_t ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); + static ngx_int_t ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, + void *data); +-static ngx_int_t ngx_http_lua_balancer_by_chunk(lua_State *L, +- ngx_http_request_t *r); + static void ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); ++static ngx_int_t ngx_http_lua_balancer_create_keepalive_pool(lua_State *L, ++ ngx_log_t *log, uint32_t cpool_crc32, ngx_uint_t cpool_size, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool); ++static void ngx_http_lua_balancer_get_keepalive_pool(lua_State *L, ++ uint32_t cpool_crc32, ngx_http_lua_balancer_keepalive_pool_t **cpool); ++static void ngx_http_lua_balancer_free_keepalive_pool(ngx_log_t *log, ++ ngx_http_lua_balancer_keepalive_pool_t *cpool); ++static void ngx_http_lua_balancer_close(ngx_connection_t *c); ++static void ngx_http_lua_balancer_dummy_handler(ngx_event_t *ev); ++static void ngx_http_lua_balancer_close_handler(ngx_event_t *ev); ++#if (NGX_HTTP_SSL) ++static ngx_int_t ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, ++ void *data); ++static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, ++ void *data); ++#endif ++ ++ ++#define ngx_http_lua_balancer_keepalive_is_enabled(bp) \ ++ (bp->keepalive) ++ ++#define ngx_http_lua_balancer_peer_set(bp) \ ++ (bp->sockaddr && bp->socklen) ++ ++ ++static char ngx_http_lua_balancer_keepalive_pools_table_key; + + + ngx_int_t +@@ -102,6 +158,61 @@ ngx_http_lua_balancer_handler_inline(ngx_http_request_t *r, + } + + ++static ngx_int_t ++ngx_http_lua_balancer_by_chunk(lua_State *L, ngx_http_request_t *r) ++{ ++ u_char *err_msg; ++ size_t len; ++ ngx_int_t rc; ++ ++ /* init nginx context in Lua VM */ ++ ngx_http_lua_set_req(L, r); ++ ++#ifndef OPENRESTY_LUAJIT ++ ngx_http_lua_create_new_globals_table(L, 0 /* narr */, 1 /* nrec */); ++ ++ /* {{{ make new env inheriting main thread's globals table */ ++ lua_createtable(L, 0, 1 /* nrec */); /* the metatable for the new env */ ++ ngx_http_lua_get_globals_table(L); ++ lua_setfield(L, -2, "__index"); ++ lua_setmetatable(L, -2); /* setmetatable({}, {__index = _G}) */ ++ /* }}} */ ++ ++ lua_setfenv(L, -2); /* set new running env for the code closure */ ++#endif /* OPENRESTY_LUAJIT */ ++ ++ lua_pushcfunction(L, ngx_http_lua_traceback); ++ lua_insert(L, 1); /* put it under chunk and args */ ++ ++ /* protected call user code */ ++ rc = lua_pcall(L, 0, 1, 1); ++ ++ lua_remove(L, 1); /* remove traceback function */ ++ ++ dd("rc == %d", (int) rc); ++ ++ if (rc != 0) { ++ /* error occurred when running loaded code */ ++ err_msg = (u_char *) lua_tolstring(L, -1, &len); ++ ++ if (err_msg == NULL) { ++ err_msg = (u_char *) "unknown reason"; ++ len = sizeof("unknown reason") - 1; ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "failed to run balancer_by_lua*: %*s", len, err_msg); ++ ++ lua_settop(L, 0); /* clear remaining elems on stack */ ++ ++ return NGX_ERROR; ++ } ++ ++ lua_settop(L, 0); /* clear remaining elems on stack */ ++ return rc; ++} ++ ++ + char * + ngx_http_lua_balancer_by_lua_block(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +@@ -125,16 +236,16 @@ char * + ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) + { +- u_char *cache_key = NULL; +- u_char *name; +- ngx_str_t *value; +- ngx_http_lua_srv_conf_t *lscf = conf; +- ++ u_char *cache_key = NULL; ++ u_char *name; ++ ngx_str_t *value; + ngx_http_upstream_srv_conf_t *uscf; ++ ngx_http_lua_srv_conf_t *lscf = conf; + + dd("enter"); + +- /* must specify a content handler */ ++ /* content handler setup */ ++ + if (cmd->post == NULL) { + return NGX_CONF_ERROR; + } +@@ -178,11 +289,19 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + lscf->balancer.src_key = cache_key; + ++ /* balancer setup */ ++ + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); ++ ++ lscf->balancer.original_init_upstream = uscf->peer.init_upstream; ++ ++ } else { ++ lscf->balancer.original_init_upstream = ++ ngx_http_upstream_init_round_robin; + } + + uscf->peer.init_upstream = ngx_http_lua_balancer_init; +@@ -198,14 +317,18 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + + static ngx_int_t +-ngx_http_lua_balancer_init(ngx_conf_t *cf, +- ngx_http_upstream_srv_conf_t *us) ++ngx_http_lua_balancer_init(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) + { +- if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { ++ ngx_http_lua_srv_conf_t *lscf; ++ ++ lscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); ++ ++ if (lscf->balancer.original_init_upstream(cf, us) != NGX_OK) { + return NGX_ERROR; + } + +- /* this callback is called upon individual requests */ ++ lscf->balancer.original_init_peer = us->peer.init; ++ + us->peer.init = ngx_http_lua_balancer_init_peer; + + return NGX_OK; +@@ -216,33 +339,38 @@ static ngx_int_t + ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) + { +- ngx_http_lua_srv_conf_t *bcf; ++ ngx_http_lua_srv_conf_t *lscf; + ngx_http_lua_balancer_peer_data_t *bp; + +- bp = ngx_pcalloc(r->pool, sizeof(ngx_http_lua_balancer_peer_data_t)); +- if (bp == NULL) { ++ lscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); ++ ++ if (lscf->balancer.original_init_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + +- r->upstream->peer.data = &bp->rrp; +- +- if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { ++ bp = ngx_pcalloc(r->pool, sizeof(ngx_http_lua_balancer_peer_data_t)); ++ if (bp == NULL) { + return NGX_ERROR; + } + ++ bp->conf = lscf; ++ bp->request = r; ++ bp->data = r->upstream->peer.data; ++ bp->original_get_peer = r->upstream->peer.get; ++ bp->original_free_peer = r->upstream->peer.free; ++ ++ r->upstream->peer.data = bp; + r->upstream->peer.get = ngx_http_lua_balancer_get_peer; + r->upstream->peer.free = ngx_http_lua_balancer_free_peer; + + #if (NGX_HTTP_SSL) ++ bp->original_set_session = r->upstream->peer.set_session; ++ bp->original_save_session = r->upstream->peer.save_session; ++ + r->upstream->peer.set_session = ngx_http_lua_balancer_set_session; + r->upstream->peer.save_session = ngx_http_lua_balancer_save_session; + #endif + +- bcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); +- +- bp->conf = bcf; +- bp->request = r; +- + return NGX_OK; + } + +@@ -250,25 +378,26 @@ ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + static ngx_int_t + ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + { +- lua_State *L; +- ngx_int_t rc; +- ngx_http_request_t *r; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_srv_conf_t *lscf; +- ngx_http_lua_main_conf_t *lmcf; +- ngx_http_lua_balancer_peer_data_t *bp = data; ++ lua_State *L; ++ ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_request_t *r; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_srv_conf_t *lscf; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ngx_http_lua_balancer_peer_data_t *bp = data; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, +- "lua balancer peer, tries: %ui", pc->tries); +- +- lscf = bp->conf; ++ "lua balancer: get peer, tries: %ui", pc->tries); + + r = bp->request; ++ lscf = bp->conf; + + ngx_http_lua_assert(lscf->balancer.handler && r); + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); +- + if (ctx == NULL) { + ctx = ngx_http_lua_create_ctx(r); + if (ctx == NULL) { +@@ -286,9 +415,15 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + + ctx->context = NGX_HTTP_LUA_CONTEXT_BALANCER; + ++ bp->cpool = NULL; + bp->sockaddr = NULL; + bp->socklen = 0; + bp->more_tries = 0; ++ bp->cpool_crc32 = 0; ++ bp->cpool_size = 0; ++ bp->keepalive_requests = 0; ++ bp->keepalive_timeout = 0; ++ bp->keepalive = 0; + bp->total_tries++; + + lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +@@ -300,7 +435,6 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + lmcf->balancer_peer_data = bp; + + rc = lscf->balancer.handler(r, lscf, L); +- + if (rc == NGX_ERROR) { + return NGX_ERROR; + } +@@ -322,105 +456,414 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + } + } + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + pc->sockaddr = bp->sockaddr; + pc->socklen = bp->socklen; ++ pc->name = bp->host; + pc->cached = 0; + pc->connection = NULL; +- pc->name = bp->host; +- +- bp->rrp.peers->single = 0; + + if (bp->more_tries) { + r->upstream->peer.tries += bp->more_tries; + } + +- dd("tries: %d", (int) r->upstream->peer.tries); ++ if (ngx_http_lua_balancer_keepalive_is_enabled(bp)) { ++ ngx_http_lua_balancer_get_keepalive_pool(L, bp->cpool_crc32, ++ &bp->cpool); ++ ++ if (bp->cpool == NULL ++ && ngx_http_lua_balancer_create_keepalive_pool(L, pc->log, ++ bp->cpool_crc32, ++ bp->cpool_size, ++ &bp->cpool) ++ != NGX_OK) ++ { ++ return NGX_ERROR; ++ } ++ ++ ngx_http_lua_assert(bp->cpool); ++ ++ if (!ngx_queue_empty(&bp->cpool->cache)) { ++ q = ngx_queue_head(&bp->cpool->cache); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ c = item->connection; ++ ++ ngx_queue_remove(q); ++ ngx_queue_insert_head(&bp->cpool->free, q); ++ ++ c->idle = 0; ++ c->sent = 0; ++ c->log = pc->log; ++ c->read->log = pc->log; ++ c->write->log = pc->log; ++ c->pool->log = pc->log; ++ ++ if (c->read->timer_set) { ++ ngx_del_timer(c->read); ++ } ++ ++ pc->cached = 1; ++ pc->connection = c; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive reusing connection %p, " ++ "requests: %ui, cpool: %p", ++ c, c->requests, bp->cpool); ++ ++ return NGX_DONE; ++ } ++ ++ bp->cpool->connections++; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive no free connection, " ++ "cpool: %p", bp->cpool); ++ } + + return NGX_OK; + } + +- return ngx_http_upstream_get_round_robin_peer(pc, &bp->rrp); ++ return bp->original_get_peer(pc, bp->data); + } + + +-static ngx_int_t +-ngx_http_lua_balancer_by_chunk(lua_State *L, ngx_http_request_t *r) ++static void ++ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, void *data, ++ ngx_uint_t state) + { +- u_char *err_msg; +- size_t len; +- ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_upstream_t *u; ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++ ngx_http_lua_balancer_peer_data_t *bp = data; + +- /* init nginx context in Lua VM */ +- ngx_http_lua_set_req(L, r); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: free peer, tries: %ui", pc->tries); + +-#ifndef OPENRESTY_LUAJIT +- ngx_http_lua_create_new_globals_table(L, 0 /* narr */, 1 /* nrec */); ++ u = bp->request->upstream; ++ c = pc->connection; + +- /* {{{ make new env inheriting main thread's globals table */ +- lua_createtable(L, 0, 1 /* nrec */); /* the metatable for the new env */ +- ngx_http_lua_get_globals_table(L); +- lua_setfield(L, -2, "__index"); +- lua_setmetatable(L, -2); /* setmetatable({}, {__index = _G}) */ +- /* }}} */ ++ if (ngx_http_lua_balancer_peer_set(bp)) { ++ bp->last_peer_state = (int) state; + +- lua_setfenv(L, -2); /* set new running env for the code closure */ +-#endif /* OPENRESTY_LUAJIT */ ++ if (pc->tries) { ++ pc->tries--; ++ } + +- lua_pushcfunction(L, ngx_http_lua_traceback); +- lua_insert(L, 1); /* put it under chunk and args */ ++ if (ngx_http_lua_balancer_keepalive_is_enabled(bp)) { ++ cpool = bp->cpool; + +- /* protected call user code */ +- rc = lua_pcall(L, 0, 1, 1); ++ if (state & NGX_PEER_FAILED ++ || c == NULL ++ || c->read->eof ++ || c->read->error ++ || c->read->timedout ++ || c->write->error ++ || c->write->timedout) ++ { ++ goto invalid; ++ } + +- lua_remove(L, 1); /* remove traceback function */ ++ if (bp->keepalive_requests ++ && c->requests >= bp->keepalive_requests) ++ { ++ goto invalid; ++ } + +- dd("rc == %d", (int) rc); ++ if (!u->keepalive) { ++ goto invalid; ++ } + +- if (rc != 0) { +- /* error occurred when running loaded code */ +- err_msg = (u_char *) lua_tolstring(L, -1, &len); ++ if (!u->request_body_sent) { ++ goto invalid; ++ } + +- if (err_msg == NULL) { +- err_msg = (u_char *) "unknown reason"; +- len = sizeof("unknown reason") - 1; ++ if (ngx_terminate || ngx_exiting) { ++ goto invalid; ++ } ++ ++ if (ngx_handle_read_event(c->read, 0) != NGX_OK) { ++ goto invalid; ++ } ++ ++ if (ngx_queue_empty(&cpool->free)) { ++ q = ngx_queue_last(&cpool->cache); ++ ngx_queue_remove(q); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ ++ ngx_http_lua_balancer_close(item->connection); ++ ++ } else { ++ q = ngx_queue_head(&cpool->free); ++ ngx_queue_remove(q); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ } ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive saving connection %p, " ++ "cpool: %p, connections: %ui", ++ c, cpool, cpool->connections); ++ ++ ngx_queue_insert_head(&cpool->cache, q); ++ ++ item->connection = c; ++ ++ pc->connection = NULL; ++ ++ if (bp->keepalive_timeout) { ++ c->read->delayed = 0; ++ ngx_add_timer(c->read, bp->keepalive_timeout); ++ ++ } else if (c->read->timer_set) { ++ ngx_del_timer(c->read); ++ } ++ ++ if (c->write->timer_set) { ++ ngx_del_timer(c->write); ++ } ++ ++ c->write->handler = ngx_http_lua_balancer_dummy_handler; ++ c->read->handler = ngx_http_lua_balancer_close_handler; ++ ++ c->data = item; ++ c->idle = 1; ++ c->log = ngx_cycle->log; ++ c->read->log = ngx_cycle->log; ++ c->write->log = ngx_cycle->log; ++ c->pool->log = ngx_cycle->log; ++ ++ if (c->read->ready) { ++ ngx_http_lua_balancer_close_handler(c->read); ++ } ++ ++ return; ++ ++invalid: ++ ++ cpool->connections--; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive not saving connection %p, " ++ "cpool: %p, connections: %ui", ++ c, cpool, cpool->connections); ++ ++ if (cpool->connections == 0) { ++ ngx_http_lua_balancer_free_keepalive_pool(pc->log, cpool); ++ } + } + +- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, +- "failed to run balancer_by_lua*: %*s", len, err_msg); ++ return; ++ } + +- lua_settop(L, 0); /* clear remaining elems on stack */ ++ bp->original_free_peer(pc, bp->data, state); ++} ++ ++ ++static ngx_int_t ++ngx_http_lua_balancer_create_keepalive_pool(lua_State *L, ngx_log_t *log, ++ uint32_t cpool_crc32, ngx_uint_t cpool_size, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool) ++{ ++ size_t size; ++ ngx_uint_t i; ++ ngx_http_lua_balancer_keepalive_pool_t *upool; ++ ngx_http_lua_balancer_keepalive_item_t *items; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ ngx_http_lua_assert(lua_istable(L, -1)); ++ ++ size = sizeof(ngx_http_lua_balancer_keepalive_pool_t) ++ + sizeof(ngx_http_lua_balancer_keepalive_item_t) * cpool_size; + ++ upool = lua_newuserdata(L, size); /* pools upool */ ++ if (upool == NULL) { + return NGX_ERROR; + } + +- lua_settop(L, 0); /* clear remaining elems on stack */ +- return rc; ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, ++ "lua balancer: keepalive create pool, crc32: %ui, " ++ "size: %ui", cpool_crc32, cpool_size); ++ ++ upool->lua_vm = L; ++ upool->crc32 = cpool_crc32; ++ upool->size = cpool_size; ++ upool->connections = 0; ++ ++ ngx_queue_init(&upool->cache); ++ ngx_queue_init(&upool->free); ++ ++ lua_rawseti(L, -2, cpool_crc32); /* pools */ ++ lua_pop(L, 1); /* orig stack */ ++ ++ items = (ngx_http_lua_balancer_keepalive_item_t *) (&upool->free + 1); ++ ++ ngx_http_lua_assert((void *) items == ngx_align_ptr(items, NGX_ALIGNMENT)); ++ ++ for (i = 0; i < cpool_size; i++) { ++ ngx_queue_insert_head(&upool->free, &items[i].queue); ++ items[i].cpool = upool; ++ } ++ ++ *cpool = upool; ++ ++ return NGX_OK; + } + + + static void +-ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, void *data, +- ngx_uint_t state) ++ngx_http_lua_balancer_get_keepalive_pool(lua_State *L, uint32_t cpool_crc32, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool) + { +- ngx_http_lua_balancer_peer_data_t *bp = data; ++ ngx_http_lua_balancer_keepalive_pool_t *upool; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ if (lua_isnil(L, -1)) { ++ lua_pop(L, 1); /* orig stack */ ++ ++ /* create upstream connection pools table */ ++ lua_createtable(L, 0, 0); /* pools */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_pushvalue(L, -2); /* pools pools_table_key pools */ ++ lua_rawset(L, LUA_REGISTRYINDEX); /* pools */ ++ } + +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, +- "lua balancer free peer, tries: %ui", pc->tries); ++ ngx_http_lua_assert(lua_istable(L, -1)); + +- if (bp->sockaddr && bp->socklen) { +- bp->last_peer_state = (int) state; ++ lua_rawgeti(L, -1, cpool_crc32); /* pools upool? */ ++ upool = lua_touserdata(L, -1); ++ lua_pop(L, 2); /* orig stack */ + +- if (pc->tries) { +- pc->tries--; ++ *cpool = upool; ++} ++ ++ ++static void ++ngx_http_lua_balancer_free_keepalive_pool(ngx_log_t *log, ++ ngx_http_lua_balancer_keepalive_pool_t *cpool) ++{ ++ lua_State *L; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, ++ "lua balancer: keepalive free pool %p, crc32: %ui", ++ cpool, cpool->crc32); ++ ++ ngx_http_lua_assert(cpool->connections == 0); ++ ++ L = cpool->lua_vm; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ if (lua_isnil(L, -1)) { ++ lua_pop(L, 1); /* orig stack */ ++ return; ++ } ++ ++ ngx_http_lua_assert(lua_istable(L, -1)); ++ ++ lua_pushnil(L); /* pools nil */ ++ lua_rawseti(L, -2, cpool->crc32); /* pools */ ++ lua_pop(L, 1); /* orig stack */ ++} ++ ++ ++static void ++ngx_http_lua_balancer_close(ngx_connection_t *c) ++{ ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ++ item = c->data; ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ c->ssl->no_wait_shutdown = 1; ++ c->ssl->no_send_shutdown = 1; ++ ++ if (ngx_ssl_shutdown(c) == NGX_AGAIN) { ++ c->ssl->handler = ngx_http_lua_balancer_close; ++ return; ++ } ++ } ++#endif ++ ++ ngx_destroy_pool(c->pool); ++ ngx_close_connection(c); ++ ++ item->cpool->connections--; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "lua balancer: keepalive closing connection %p, cpool: %p, " ++ "connections: %ui", ++ c, item->cpool, item->cpool->connections); ++} ++ ++ ++static void ++ngx_http_lua_balancer_dummy_handler(ngx_event_t *ev) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, ++ "lua balancer: dummy handler"); ++} ++ ++ ++static void ++ngx_http_lua_balancer_close_handler(ngx_event_t *ev) ++{ ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ++ int n; ++ char buf[1]; ++ ngx_connection_t *c; ++ ++ c = ev->data; ++ ++ if (c->close || c->read->timedout) { ++ goto close; ++ } ++ ++ n = recv(c->fd, buf, 1, MSG_PEEK); ++ ++ if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { ++ ev->ready = 0; ++ ++ if (ngx_handle_read_event(c->read, 0) != NGX_OK) { ++ goto close; + } + + return; + } + +- /* fallback */ ++close: ++ ++ item = c->data; ++ c->log = ev->log; ++ ++ ngx_http_lua_balancer_close(c); + +- ngx_http_upstream_free_round_robin_peer(pc, data, state); ++ ngx_queue_remove(&item->queue); ++ ngx_queue_insert_head(&item->cpool->free, &item->queue); ++ ++ if (item->cpool->connections == 0) { ++ ngx_http_lua_balancer_free_keepalive_pool(ev->log, item->cpool); ++ } + } + + +@@ -431,12 +874,12 @@ ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, void *data) + { + ngx_http_lua_balancer_peer_data_t *bp = data; + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + /* TODO */ + return NGX_OK; + } + +- return ngx_http_upstream_set_round_robin_peer_session(pc, &bp->rrp); ++ return bp->original_set_session(pc, bp->data); + } + + +@@ -445,13 +888,12 @@ ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, void *data) + { + ngx_http_lua_balancer_peer_data_t *bp = data; + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + /* TODO */ + return; + } + +- ngx_http_upstream_save_round_robin_peer_session(pc, &bp->rrp); +- return; ++ bp->original_save_session(pc, bp->data); + } + + #endif +@@ -459,14 +901,14 @@ ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, void *data) + + int + ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, +- const u_char *addr, size_t addr_len, int port, char **err) ++ const u_char *addr, size_t addr_len, int port, unsigned int cpool_crc32, ++ unsigned int cpool_size, char **err) + { +- ngx_url_t url; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- +- ngx_http_lua_main_conf_t *lmcf; +- ngx_http_lua_balancer_peer_data_t *bp; ++ ngx_url_t url; ++ ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { + *err = "no request found"; +@@ -536,6 +978,70 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ bp->cpool_crc32 = (uint32_t) cpool_crc32; ++ bp->cpool_size = (ngx_uint_t) cpool_size; ++ ++ return NGX_OK; ++} ++ ++ ++int ++ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, ++ unsigned long timeout, unsigned int max_requests, char **err) ++{ ++ ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_peer_data_t *bp; ++ ++ if (r == NULL) { ++ *err = "no request found"; ++ return NGX_ERROR; ++ } ++ ++ u = r->upstream; ++ ++ if (u == NULL) { ++ *err = "no upstream found"; ++ return NGX_ERROR; ++ } ++ ++ ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); ++ if (ctx == NULL) { ++ *err = "no ctx found"; ++ return NGX_ERROR; ++ } ++ ++ if ((ctx->context & NGX_HTTP_LUA_CONTEXT_BALANCER) == 0) { ++ *err = "API disabled in the current context"; ++ return NGX_ERROR; ++ } ++ ++ lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); ++ ++ /* we cannot read r->upstream->peer.data here directly because ++ * it could be overridden by other modules like ++ * ngx_http_upstream_keepalive_module. ++ */ ++ bp = lmcf->balancer_peer_data; ++ if (bp == NULL) { ++ *err = "no upstream peer data found"; ++ return NGX_ERROR; ++ } ++ ++ if (!ngx_http_lua_balancer_peer_set(bp)) { ++ *err = "no current peer set"; ++ return NGX_ERROR; ++ } ++ ++ if (!bp->cpool_crc32) { ++ bp->cpool_crc32 = ngx_crc32_long(bp->host->data, bp->host->len); ++ } ++ ++ bp->keepalive_timeout = (ngx_msec_t) timeout; ++ bp->keepalive_requests = (ngx_uint_t) max_requests; ++ bp->keepalive = 1; ++ + return NGX_OK; + } + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_common.h b/ngx_lua-0.10.20/src/ngx_http_lua_common.h +index 781a2454..9ce6836a 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h +@@ -320,6 +320,10 @@ union ngx_http_lua_srv_conf_u { + #endif + + struct { ++ ngx_http_upstream_init_pt original_init_upstream; ++ ngx_http_upstream_init_peer_pt original_init_peer; ++ uintptr_t data; ++ + ngx_http_lua_srv_conf_handler_pt handler; + ngx_str_t src; + u_char *src_key; +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_module.c b/ngx_lua-0.10.20/src/ngx_http_lua_module.c +index 9816d864..5d7cedfd 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_module.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_module.c +@@ -1068,6 +1068,9 @@ ngx_http_lua_create_srv_conf(ngx_conf_t *cf) + * lscf->srv.ssl_session_fetch_src = { 0, NULL }; + * lscf->srv.ssl_session_fetch_src_key = NULL; + * ++ * lscf->balancer.original_init_upstream = NULL; ++ * lscf->balancer.original_init_peer = NULL; ++ * lscf->balancer.data = NULL; + * lscf->balancer.handler = NULL; + * lscf->balancer.src = { 0, NULL }; + * lscf->balancer.src_key = NULL; +-- +2.26.2 + + +From 4c5cb29a265b2f9524434322adf15d07deec6c7f Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:54 -0700 +Subject: [PATCH 2/3] feature: we now avoid the need for 'upstream' blocks to + define a stub 'server' directive when using 'balancer_by_lua*'. + +--- + src/ngx_http_lua_balancer.c | 42 +++++++++++++++++++++++++++++++++++-- + 1 file changed, 40 insertions(+), 2 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c b/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +index 0d403716..5c862d22 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +@@ -111,7 +111,8 @@ static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, + (bp->sockaddr && bp->socklen) + + +-static char ngx_http_lua_balancer_keepalive_pools_table_key; ++static char ngx_http_lua_balancer_keepalive_pools_table_key; ++static struct sockaddr *ngx_http_lua_balancer_default_server_sockaddr; + + + ngx_int_t +@@ -239,7 +240,9 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + u_char *cache_key = NULL; + u_char *name; + ngx_str_t *value; ++ ngx_url_t url; + ngx_http_upstream_srv_conf_t *uscf; ++ ngx_http_upstream_server_t *us; + ngx_http_lua_srv_conf_t *lscf = conf; + + dd("enter"); +@@ -293,6 +296,29 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + ++ if (uscf->servers->nelts == 0) { ++ us = ngx_array_push(uscf->servers); ++ if (us == NULL) { ++ return NGX_CONF_ERROR; ++ } ++ ++ ngx_memzero(us, sizeof(ngx_http_upstream_server_t)); ++ ngx_memzero(&url, sizeof(ngx_url_t)); ++ ++ ngx_str_set(&url.url, "0.0.0.1"); ++ url.default_port = 80; ++ ++ if (ngx_parse_url(cf->pool, &url) != NGX_OK) { ++ return NGX_CONF_ERROR; ++ } ++ ++ us->name = url.url; ++ us->addrs = url.addrs; ++ us->naddrs = url.naddrs; ++ ++ ngx_http_lua_balancer_default_server_sockaddr = us->addrs[0].sockaddr; ++ } ++ + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); +@@ -525,7 +551,19 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + return NGX_OK; + } + +- return bp->original_get_peer(pc, bp->data); ++ rc = bp->original_get_peer(pc, bp->data); ++ if (rc == NGX_ERROR) { ++ return rc; ++ } ++ ++ if (pc->sockaddr == ngx_http_lua_balancer_default_server_sockaddr) { ++ ngx_log_error(NGX_LOG_ERR, pc->log, 0, ++ "lua balancer: no peer set"); ++ ++ return NGX_ERROR; ++ } ++ ++ return rc; + } + + +-- +2.26.2 + + +From 941cd893573561574bc6a326d6306f1a30127293 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:58 -0700 +Subject: [PATCH 3/3] refactor: used a simpler way to stash the balancer peer + data. + +--- + src/ngx_http_lua_balancer.c | 91 +++++++++---------------------------- + src/ngx_http_lua_common.h | 7 --- + 2 files changed, 22 insertions(+), 76 deletions(-) + +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c b/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +index 5c862d22..3ea1f067 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_balancer.c +@@ -411,9 +411,9 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_srv_conf_t *lscf; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_keepalive_item_t *item; + ngx_http_lua_balancer_peer_data_t *bp = data; ++ void *pdata; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "lua balancer: get peer, tries: %ui", pc->tries); +@@ -452,15 +452,13 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + bp->keepalive = 0; + bp->total_tries++; + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* balancer_by_lua does not support yielding and +- * there cannot be any conflicts among concurrent requests, +- * thus it is safe to store the peer data in the main conf. +- */ +- lmcf->balancer_peer_data = bp; ++ pdata = r->upstream->peer.data; ++ r->upstream->peer.data = bp; + + rc = lscf->balancer.handler(r, lscf, L); ++ ++ r->upstream->peer.data = pdata; ++ + if (rc == NGX_ERROR) { + return NGX_ERROR; + } +@@ -945,7 +943,6 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + ngx_url_t url; + ngx_http_upstream_t *u; + ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -971,18 +968,6 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* we cannot read r->upstream->peer.data here directly because +- * it could be overridden by other modules like +- * ngx_http_upstream_keepalive_module. +- */ +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } +- + ngx_memzero(&url, sizeof(ngx_url_t)); + + url.url.data = ngx_palloc(r->pool, addr_len); +@@ -1006,6 +991,8 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; ++ + if (url.addrs && url.addrs[0].sockaddr) { + bp->sockaddr = url.addrs[0].sockaddr; + bp->socklen = url.addrs[0].socklen; +@@ -1029,7 +1016,6 @@ ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, + { + ngx_http_upstream_t *u; + ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -1055,17 +1041,7 @@ ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* we cannot read r->upstream->peer.data here directly because +- * it could be overridden by other modules like +- * ngx_http_upstream_keepalive_module. +- */ +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + if (!ngx_http_lua_balancer_peer_set(bp)) { + *err = "no current peer set"; +@@ -1089,14 +1065,13 @@ ngx_http_lua_ffi_balancer_set_timeouts(ngx_http_request_t *r, + long connect_timeout, long send_timeout, long read_timeout, + char **err) + { +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; + + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) + ngx_http_upstream_conf_t *ucf; +-#endif +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; ++#endif + + if (r == NULL) { + *err = "no request found"; +@@ -1121,15 +1096,9 @@ ngx_http_lua_ffi_balancer_set_timeouts(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } +- + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; ++ + if (!bp->cloned_upstream_conf) { + /* we clone the upstream conf for the current request so that + * we do not affect other requests at all. */ +@@ -1184,12 +1153,10 @@ ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + int count, char **err) + { + #if (nginx_version >= 1007005) +- ngx_uint_t max_tries, total; ++ ngx_uint_t max_tries, total; + #endif +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- +- ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -1215,13 +1182,7 @@ ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + #if (nginx_version >= 1007005) + max_tries = r->upstream->conf->next_upstream_tries; +@@ -1247,12 +1208,10 @@ int + ngx_http_lua_ffi_balancer_get_last_failure(ngx_http_request_t *r, + int *status, char **err) + { +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- ngx_http_upstream_state_t *state; +- ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; ++ ngx_http_upstream_state_t *state; + ngx_http_lua_balancer_peer_data_t *bp; +- ngx_http_lua_main_conf_t *lmcf; + + if (r == NULL) { + *err = "no request found"; +@@ -1277,13 +1236,7 @@ ngx_http_lua_ffi_balancer_get_last_failure(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + if (r->upstream_states && r->upstream_states->nelts > 1) { + state = r->upstream_states->elts; +diff --git a/ngx_lua-0.10.20/src/ngx_http_lua_common.h b/ngx_lua-0.10.20/src/ngx_http_lua_common.h +index 9ce6836a..9a4342df 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h +@@ -239,13 +239,6 @@ struct ngx_http_lua_main_conf_s { + ngx_http_lua_main_conf_handler_pt exit_worker_handler; + ngx_str_t exit_worker_src; + +- ngx_http_lua_balancer_peer_data_t *balancer_peer_data; +- /* neither yielding nor recursion is possible in +- * balancer_by_lua*, so there cannot be any races among +- * concurrent requests and it is safe to store the peer +- * data pointer in the main conf. +- */ +- + ngx_chain_t *body_filter_chain; + /* neither yielding nor recursion is possible in + * body_filter_by_lua*, so there cannot be any races among +-- +2.26.2 diff --git a/build/openresty/patches/ngx_lua-0.10.20_03-ngx-worker-pids.patch b/build/openresty/patches/ngx_lua-0.10.20_03-ngx-worker-pids.patch new file mode 100644 index 000000000000..27b2f253123b --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.20_03-ngx-worker-pids.patch @@ -0,0 +1,57 @@ +From edfa0f984ec60bd0658b80643c2fd253f3c5ad0b Mon Sep 17 00:00:00 2001 +From: attenuation +Date: Sun, 21 Aug 2022 21:59:28 +0800 +Subject: [PATCH] feat: add ngx_http_lua_ffi_worker_pids to get all workers pid + map + +--- + +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_worker.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_worker.c +index 0ca2d414e3..52ec34a844 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_worker.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_worker.c +@@ -8,6 +8,7 @@ + #define DDEBUG 0 + #endif + #include "ddebug.h" ++#include + + + #define NGX_PROCESS_PRIVILEGED_AGENT 99 +@@ -20,6 +21,36 @@ ngx_http_lua_ffi_worker_pid(void) + } + + ++int ++ngx_http_lua_ffi_worker_pids(int *pids, size_t *pids_len) ++{ ++ ngx_int_t i, n; ++ ++ n = 0; ++ for (i = 0; i < NGX_MAX_PROCESSES; i++) { ++ if (i != ngx_process_slot && ngx_processes[i].pid == 0) { ++ break; ++ } ++ ++ if (i == ngx_process_slot && ngx_processes[i].pid == 0) { ++ pids[n++] = ngx_pid; ++ } ++ ++ if (ngx_processes[i].pid > 0) { ++ pids[n++] = ngx_processes[i].pid; ++ } ++ } ++ ++ if (n == 0) { ++ return NGX_ERROR; ++ } ++ ++ *pids_len = n; ++ ++ return NGX_OK; ++} ++ ++ + int + ngx_http_lua_ffi_worker_id(void) + { \ No newline at end of file diff --git a/build/openresty/patches/ngx_lua-0.10.20_04-crash-when-buffering-with-invalid-if-match-header.patch b/build/openresty/patches/ngx_lua-0.10.20_04-crash-when-buffering-with-invalid-if-match-header.patch new file mode 100644 index 000000000000..cf4d3ed17e9c --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.20_04-crash-when-buffering-with-invalid-if-match-header.patch @@ -0,0 +1,239 @@ +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_accessby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_accessby.c +index 58c2514..d40eab1 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_accessby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_accessby.c +@@ -240,7 +240,7 @@ ngx_http_lua_access_by_chunk(lua_State *L, ngx_http_request_t *r) + ngx_event_t *rev; + ngx_connection_t *c; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ngx_http_lua_loc_conf_t *llcf; + +@@ -291,9 +291,9 @@ ngx_http_lua_access_by_chunk(lua_State *L, ngx_http_request_t *r) + + /* }}} */ + +- /* {{{ register request cleanup hooks */ ++ /* {{{ register nginx pool cleanup hooks */ + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_bodyfilterby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_bodyfilterby.c +index 7560869..8e308ae 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_bodyfilterby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_bodyfilterby.c +@@ -233,7 +233,7 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in) + ngx_http_lua_ctx_t *ctx; + ngx_int_t rc; + uint16_t old_context; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + ngx_chain_t *out; + ngx_http_lua_main_conf_t *lmcf; + +@@ -273,7 +273,7 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in) + } + + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h +index 7a66cb1..0bfed59 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_common.h +@@ -540,7 +540,7 @@ typedef struct ngx_http_lua_ctx_s { + ngx_chain_t *busy_bufs; + ngx_chain_t *free_recv_bufs; + +- ngx_http_cleanup_pt *cleanup; ++ ngx_pool_cleanup_pt *cleanup; + + ngx_http_cleanup_t *free_cleanup; /* free list of cleanup records */ + +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_contentby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_contentby.c +index 76e6a07..5e2ae55 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_contentby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_contentby.c +@@ -29,7 +29,7 @@ ngx_http_lua_content_by_chunk(lua_State *L, ngx_http_request_t *r) + lua_State *co; + ngx_event_t *rev; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ngx_http_lua_loc_conf_t *llcf; + +@@ -83,7 +83,7 @@ ngx_http_lua_content_by_chunk(lua_State *L, ngx_http_request_t *r) + + /* {{{ register request cleanup hooks */ + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_directive.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_directive.c +index 1ec641e..e276663 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_directive.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_directive.c +@@ -1265,7 +1265,7 @@ ngx_http_lua_set_by_lua_init(ngx_http_request_t *r) + { + lua_State *L; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { +@@ -1280,7 +1280,7 @@ ngx_http_lua_set_by_lua_init(ngx_http_request_t *r) + } + + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_headerfilterby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_headerfilterby.c +index 4741c72..9f49a8e 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_headerfilterby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_headerfilterby.c +@@ -230,7 +230,7 @@ ngx_http_lua_header_filter(ngx_http_request_t *r) + ngx_http_lua_loc_conf_t *llcf; + ngx_http_lua_ctx_t *ctx; + ngx_int_t rc; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + uint16_t old_context; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +@@ -259,7 +259,7 @@ ngx_http_lua_header_filter(ngx_http_request_t *r) + } + + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_rewriteby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_rewriteby.c +index d1eabec..4109f28 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_rewriteby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_rewriteby.c +@@ -241,7 +241,7 @@ ngx_http_lua_rewrite_by_chunk(lua_State *L, ngx_http_request_t *r) + ngx_event_t *rev; + ngx_connection_t *c; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ngx_http_lua_loc_conf_t *llcf; + +@@ -291,9 +291,9 @@ ngx_http_lua_rewrite_by_chunk(lua_State *L, ngx_http_request_t *r) + + /* }}} */ + +- /* {{{ register request cleanup hooks */ ++ /* {{{ register nginx pool cleanup hooks */ + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_udp.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_udp.c +index fd3e074..74fcac1 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_udp.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_socket_udp.c +@@ -591,7 +591,7 @@ ngx_http_lua_socket_resolve_retval_handler(ngx_http_request_t *r, + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_co_ctx_t *coctx; + ngx_connection_t *c; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + ngx_http_upstream_resolved_t *ur; + ngx_int_t rc; + ngx_http_lua_udp_connection_t *uc; +@@ -625,7 +625,7 @@ ngx_http_lua_socket_resolve_retval_handler(ngx_http_request_t *r, + } + + if (u->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + u->ft_type |= NGX_HTTP_LUA_SOCKET_FT_ERROR; + lua_pushnil(L); +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_certby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_certby.c +index 6ed2f3f..2ca8ac3 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_certby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_certby.c +@@ -443,7 +443,7 @@ ngx_http_lua_ssl_cert_by_chunk(lua_State *L, ngx_http_request_t *r) + ngx_int_t rc; + lua_State *co; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + +@@ -497,7 +497,7 @@ ngx_http_lua_ssl_cert_by_chunk(lua_State *L, ngx_http_request_t *r) + + /* register request cleanup hooks */ + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + rc = NGX_ERROR; + ngx_http_lua_finalize_request(r, rc); +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_session_fetchby.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_session_fetchby.c +index 8d8c42b..86e6502 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_session_fetchby.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_ssl_session_fetchby.c +@@ -468,7 +468,7 @@ ngx_http_lua_ssl_sess_fetch_by_chunk(lua_State *L, ngx_http_request_t *r) + ngx_int_t rc; + lua_State *co; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + +@@ -522,7 +522,7 @@ ngx_http_lua_ssl_sess_fetch_by_chunk(lua_State *L, ngx_http_request_t *r) + + /* register request cleanup hooks */ + if (ctx->cleanup == NULL) { +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + rc = NGX_ERROR; + ngx_http_lua_finalize_request(r, rc); +diff --git a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_timer.c b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_timer.c +index 353007d..7d03db1 100644 +--- a/bundle/ngx_lua-0.10.20/src/ngx_http_lua_timer.c ++++ b/bundle/ngx_lua-0.10.20/src/ngx_http_lua_timer.c +@@ -509,7 +509,7 @@ ngx_http_lua_timer_handler(ngx_event_t *ev) + ngx_connection_t *c = NULL; + ngx_http_request_t *r = NULL; + ngx_http_lua_ctx_t *ctx; +- ngx_http_cleanup_t *cln; ++ ngx_pool_cleanup_t *cln; + ngx_pool_cleanup_t *pcln; + + ngx_http_lua_timer_ctx_t tctx; +@@ -608,7 +608,7 @@ ngx_http_lua_timer_handler(ngx_event_t *ev) + + L = ngx_http_lua_get_lua_vm(r, ctx); + +- cln = ngx_http_cleanup_add(r, 0); ++ cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + errmsg = "could not add request cleanup"; + goto failed; diff --git a/build/openresty/patches/ngx_lua-0.10.21_01-cosocket-mtls.patch b/build/openresty/patches/ngx_lua-0.10.21_01-cosocket-mtls.patch new file mode 100644 index 000000000000..0a27abf866b8 --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.21_01-cosocket-mtls.patch @@ -0,0 +1,1433 @@ +From 287d58810c450f912a8d31a94a1c86ccc039c0e1 Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 16:39:05 -0700 +Subject: [PATCH 04/17] cosocket: add function `tcpsock:tlshandshake`, retired + the Lua C API based `tcpsock:sslhandshake` implementation. + +--- + src/ngx_http_lua_socket_tcp.c | 387 +++++++++++++++------------------- + src/ngx_http_lua_socket_tcp.h | 3 + + 2 files changed, 177 insertions(+), 213 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 26467fdd..4ef22c11 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -23,6 +23,9 @@ static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) + static int ngx_http_lua_socket_tcp_sslhandshake(lua_State *L); ++static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); ++static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); + #endif + static int ngx_http_lua_socket_tcp_receive(lua_State *L); + static int ngx_http_lua_socket_tcp_receiveany(lua_State *L); +@@ -149,12 +152,6 @@ static void ngx_http_lua_socket_shutdown_pool_helper( + ngx_http_lua_socket_pool_t *spool); + static int ngx_http_lua_socket_prepare_error_retvals(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L, ngx_uint_t ft_type); +-#if (NGX_HTTP_SSL) +-static int ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, +- ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); +-static void ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c); +-static int ngx_http_lua_ssl_free_session(lua_State *L); +-#endif + static void ngx_http_lua_socket_tcp_close_connection(ngx_connection_t *c); + + +@@ -324,13 +321,6 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + lua_pushcfunction(L, ngx_http_lua_socket_tcp_connect); + lua_setfield(L, -2, "connect"); + +-#if (NGX_HTTP_SSL) +- +- lua_pushcfunction(L, ngx_http_lua_socket_tcp_sslhandshake); +- lua_setfield(L, -2, "sslhandshake"); +- +-#endif +- + lua_pushcfunction(L, ngx_http_lua_socket_tcp_receive); + lua_setfield(L, -2, "receive"); + +@@ -404,19 +394,6 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + lua_setfield(L, -2, "__gc"); + lua_rawset(L, LUA_REGISTRYINDEX); + /* }}} */ +- +-#if (NGX_HTTP_SSL) +- +- /* {{{ssl session userdata metatable */ +- lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( +- ssl_session_metatable_key)); +- lua_createtable(L, 0 /* narr */, 1 /* nrec */); /* metatable */ +- lua_pushcfunction(L, ngx_http_lua_ssl_free_session); +- lua_setfield(L, -2, "__gc"); +- lua_rawset(L, LUA_REGISTRYINDEX); +- /* }}} */ +- +-#endif + } + + +@@ -1559,64 +1536,69 @@ ngx_http_lua_socket_conn_error_retval_handler(ngx_http_request_t *r, + + #if (NGX_HTTP_SSL) + +-static int +-ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) ++static const char * ++ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, unsigned int ops) + { +- int n, top; +- ngx_int_t rc; +- ngx_str_t name = ngx_null_string; +- ngx_connection_t *c; +- ngx_ssl_session_t **psession; +- ngx_http_request_t *r; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_co_ctx_t *coctx; +- +- ngx_http_lua_socket_tcp_upstream_t *u; +- +- /* Lua function arguments: self [,session] [,host] [,verify] +- [,send_status_req] */ ++ if (ops & SOCKET_OP_CONNECT && u->conn_waiting) { ++ return "socket busy connecting"; ++ } + +- n = lua_gettop(L); +- if (n < 1 || n > 5) { +- return luaL_error(L, "ngx.socket sslhandshake: expecting 1 ~ 5 " +- "arguments (including the object), but seen %d", n); ++ if (ops & SOCKET_OP_READ && u->read_waiting) { ++ return "socket busy reading"; + } + +- r = ngx_http_lua_get_req(L); +- if (r == NULL) { +- return luaL_error(L, "no request found"); ++ if (ops & SOCKET_OP_WRITE ++ && (u->write_waiting ++ || (u->raw_downstream ++ && (r->connection->buffered & NGX_HTTP_LOWLEVEL_BUFFERED)))) ++ { ++ return "socket busy writing"; + } + +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tcp socket ssl handshake"); ++ return NULL; ++} + +- luaL_checktype(L, 1, LUA_TTABLE); ++int ++ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, ++ int enable_session_reuse, ngx_str_t *server_name, int verify, ++ int ocsp_status_req, const char **errmsg) ++{ ++ ngx_int_t rc; ++ ngx_connection_t *c; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_co_ctx_t *coctx; ++ const char *busy_rc; + +- lua_rawgeti(L, 1, SOCKET_CTX_INDEX); +- u = lua_touserdata(L, -1); ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua tcp socket tls handshake"); + + if (u == NULL + || u->peer.connection == NULL + || u->read_closed + || u->write_closed) + { +- lua_pushnil(L); +- lua_pushliteral(L, "closed"); +- return 2; ++ *errmsg = "closed"; ++ return NGX_ERROR; + } + + if (u->request != r) { +- return luaL_error(L, "bad request"); ++ *errmsg = "bad request"; ++ return NGX_ERROR; + } + +- ngx_http_lua_socket_check_busy_connecting(r, u, L); +- ngx_http_lua_socket_check_busy_reading(r, u, L); +- ngx_http_lua_socket_check_busy_writing(r, u, L); ++ busy_rc = ngx_http_lua_socket_tcp_check_busy(r, u, SOCKET_OP_CONNECT ++ | SOCKET_OP_READ ++ | SOCKET_OP_WRITE); ++ if (busy_rc != NULL) { ++ *errmsg = busy_rc; ++ return NGX_ERROR; ++ } + + if (u->raw_downstream || u->body_downstream) { +- lua_pushnil(L); +- lua_pushliteral(L, "not supported for downstream"); +- return 2; ++ *errmsg = "not supported for downstream"; ++ return NGX_ERROR; + } + + c = u->peer.connection; +@@ -1624,122 +1606,96 @@ ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) + u->ssl_session_reuse = 1; + + if (c->ssl && c->ssl->handshaked) { +- switch (lua_type(L, 2)) { +- case LUA_TUSERDATA: +- lua_pushvalue(L, 2); +- break; ++ if (sess != NULL) { ++ return NGX_DONE; ++ } + +- case LUA_TBOOLEAN: +- if (!lua_toboolean(L, 2)) { +- /* avoid generating the ssl session */ +- lua_pushboolean(L, 1); +- break; +- } +- /* fall through */ ++ u->ssl_session_reuse = enable_session_reuse; + +- default: +- ngx_http_lua_ssl_handshake_retval_handler(r, u, L); +- break; +- } ++ (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); + +- return 1; ++ return NGX_OK; + } + + if (ngx_ssl_create_connection(u->conf->ssl, c, + NGX_SSL_BUFFER|NGX_SSL_CLIENT) + != NGX_OK) + { +- lua_pushnil(L); +- lua_pushliteral(L, "failed to create ssl connection"); +- return 2; ++ *errmsg = "failed to create ssl connection"; ++ return NGX_ERROR; + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { +- return luaL_error(L, "no ctx found"); ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "no ngx_lua ctx found while TLS handshaking"); ++ ++ ngx_http_lua_assert(NULL); ++ ++ *errmsg = "no ctx found"; ++ return NGX_ERROR; + } + + coctx = ctx->cur_co_ctx; + + c->sendfile = 0; + +- if (n >= 2) { +- if (lua_type(L, 2) == LUA_TBOOLEAN) { +- u->ssl_session_reuse = lua_toboolean(L, 2); +- +- } else { +- psession = lua_touserdata(L, 2); +- +- if (psession != NULL && *psession != NULL) { +- if (ngx_ssl_set_session(c, *psession) != NGX_OK) { +- lua_pushnil(L); +- lua_pushliteral(L, "lua ssl set session failed"); +- return 2; +- } +- +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua ssl set session: %p", *psession); +- } ++ if (sess != NULL) { ++ if (ngx_ssl_set_session(c, sess) != NGX_OK) { ++ *errmsg = "lua tls set session failed"; ++ return NGX_ERROR; + } + +- if (n >= 3) { +- name.data = (u_char *) lua_tolstring(L, 3, &name.len); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "lua tls set session: %p", sess); + +- if (name.data) { +- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua ssl server name: \"%*s\"", name.len, +- name.data); ++ } else { ++ u->ssl_session_reuse = enable_session_reuse; ++ } + +-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME ++ if (server_name != NULL && server_name->data != NULL) { ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua tls server name: \"%V\"", server_name); + +- if (SSL_set_tlsext_host_name(c->ssl->connection, +- (char *) name.data) +- == 0) +- { +- lua_pushnil(L); +- lua_pushliteral(L, "SSL_set_tlsext_host_name failed"); +- return 2; +- } ++#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME ++ if (SSL_set_tlsext_host_name(c->ssl->connection, ++ (char *) server_name->data) ++ == 0) ++ { ++ *errmsg = "SSL_set_tlsext_host_name failed"; ++ return NGX_ERROR; ++ } + + #else +- +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua socket SNI disabled because the current " +- "version of OpenSSL lacks the support"); +- ++ *errmsg = "OpenSSL has no SNI support"; ++ return NGX_ERROR; + #endif +- } ++ } + +- if (n >= 4) { +- u->ssl_verify = lua_toboolean(L, 4); ++ u->ssl_verify = verify; + +- if (n >= 5) { +- if (lua_toboolean(L, 5)) { ++ if (ocsp_status_req) { + #ifdef NGX_HTTP_LUA_USE_OCSP +- SSL_set_tlsext_status_type(c->ssl->connection, +- TLSEXT_STATUSTYPE_ocsp); ++ SSL_set_tlsext_status_type(c->ssl->connection, ++ TLSEXT_STATUSTYPE_ocsp); ++ + #else +- return luaL_error(L, "no OCSP support"); ++ *errmsg = "no OCSP support"; ++ return NGX_ERROR; + #endif +- } +- } +- } +- } + } + +- dd("found sni name: %.*s %p", (int) name.len, name.data, name.data); +- +- if (name.len == 0) { ++ if (server_name->len == 0) { + u->ssl_name.len = 0; + + } else { + if (u->ssl_name.data) { + /* buffer already allocated */ + +- if (u->ssl_name.len >= name.len) { ++ if (u->ssl_name.len >= server_name->len) { + /* reuse it */ +- ngx_memcpy(u->ssl_name.data, name.data, name.len); +- u->ssl_name.len = name.len; ++ ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ u->ssl_name.len = server_name->len; + + } else { + ngx_free(u->ssl_name.data); +@@ -1750,17 +1706,16 @@ ngx_http_lua_socket_tcp_sslhandshake(lua_State *L) + + new_ssl_name: + +- u->ssl_name.data = ngx_alloc(name.len, ngx_cycle->log); ++ u->ssl_name.data = ngx_alloc(server_name->len, ngx_cycle->log); + if (u->ssl_name.data == NULL) { + u->ssl_name.len = 0; + +- lua_pushnil(L); +- lua_pushliteral(L, "no memory"); +- return 2; ++ *errmsg = "no memory"; ++ return NGX_ERROR; + } + +- ngx_memcpy(u->ssl_name.data, name.data, name.len); +- u->ssl_name.len = name.len; ++ ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ u->ssl_name.len = server_name->len; + } + } + +@@ -1774,7 +1729,8 @@ new_ssl_name: + + rc = ngx_ssl_handshake(c); + +- dd("ngx_ssl_handshake returned %d", (int) rc); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "ngx_ssl_handshake returned %d", rc); + + if (rc == NGX_AGAIN) { + if (c->write->timer_set) { +@@ -1784,13 +1740,13 @@ new_ssl_name: + ngx_add_timer(c->read, u->connect_timeout); + + u->conn_waiting = 1; +- u->write_prepare_retvals = ngx_http_lua_ssl_handshake_retval_handler; ++ u->write_prepare_retvals = ngx_http_lua_tls_handshake_retval_handler; + + ngx_http_lua_cleanup_pending_operation(coctx); + coctx->cleanup = ngx_http_lua_coctx_cleanup; + coctx->data = u; + +- c->ssl->handler = ngx_http_lua_ssl_handshake_handler; ++ c->ssl->handler = ngx_http_lua_tls_handshake_handler; + + if (ctx->entered_content_phase) { + r->write_event_handler = ngx_http_lua_content_wev_handler; +@@ -1799,21 +1755,25 @@ new_ssl_name: + r->write_event_handler = ngx_http_core_run_phases; + } + +- return lua_yield(L, 0); ++ return NGX_AGAIN; ++ } ++ ++ ngx_http_lua_tls_handshake_handler(c); ++ ++ if (rc == NGX_ERROR) { ++ *errmsg = u->error_ret; ++ ++ return NGX_ERROR; + } + +- top = lua_gettop(L); +- ngx_http_lua_ssl_handshake_handler(c); +- return lua_gettop(L) - top; ++ return NGX_OK; + } + + + static void +-ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) ++ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + { +- const char *err; + int waiting; +- lua_State *L; + ngx_int_t rc; + ngx_connection_t *dc; /* downstream connection */ + ngx_http_request_t *r; +@@ -1836,11 +1796,9 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + waiting = u->conn_waiting; + + dc = r->connection; +- L = u->write_co_ctx->co; + + if (c->read->timedout) { +- lua_pushnil(L); +- lua_pushliteral(L, "timeout"); ++ u->error_ret = "timeout"; + goto failed; + } + +@@ -1849,19 +1807,18 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + } + + if (c->ssl->handshaked) { +- + if (u->ssl_verify) { + rc = SSL_get_verify_result(c->ssl->connection); + + if (rc != X509_V_OK) { +- lua_pushnil(L); +- err = lua_pushfstring(L, "%d: %s", (int) rc, +- X509_verify_cert_error_string(rc)); ++ u->error_ret = X509_verify_cert_error_string(rc); ++ u->openssl_error_code_ret = rc; + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " +- "certificate verify error: (%s)", err); ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ "certificate verify error: (%d: %s)", ++ rc, u->error_ret); + } + + goto failed; +@@ -1872,12 +1829,11 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + if (u->ssl_name.len + && ngx_ssl_check_host(c, &u->ssl_name) != NGX_OK) + { +- lua_pushnil(L); +- lua_pushliteral(L, "certificate host mismatch"); ++ u->error_ret = "certificate host mismatch"; + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " + "certificate does not match host \"%V\"", + &u->ssl_name); + } +@@ -1892,7 +1848,7 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + ngx_http_lua_socket_handle_conn_success(r, u); + + } else { +- (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, L); ++ (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); + } + + if (waiting) { +@@ -1902,60 +1858,84 @@ ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + return; + } + +- lua_pushnil(L); +- lua_pushliteral(L, "handshake failed"); ++ u->error_ret = "handshake failed"; + + failed: + + if (waiting) { + u->write_prepare_retvals = +- ngx_http_lua_socket_conn_error_retval_handler; +- ngx_http_lua_socket_handle_conn_error(r, u, +- NGX_HTTP_LUA_SOCKET_FT_SSL); ++ ngx_http_lua_socket_conn_error_retval_handler; ++ ngx_http_lua_socket_handle_conn_error(r, u, NGX_HTTP_LUA_SOCKET_FT_SSL); + ngx_http_run_posted_requests(dc); + + } else { +- (void) ngx_http_lua_socket_conn_error_retval_handler(r, u, L); ++ u->ft_type |= NGX_HTTP_LUA_SOCKET_FT_SSL; ++ ++ (void) ngx_http_lua_socket_conn_error_retval_handler(r, u, NULL); ++ } ++} ++ ++ ++ ++int ++ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, ++ ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, ++ const char **errmsg, int *openssl_error_code) ++{ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "lua cosocket get TLS handshake result for upstream: %p", u); ++ ++ if (u->error_ret != NULL) { ++ *errmsg = u->error_ret; ++ *openssl_error_code = u->openssl_error_code_ret; ++ ++ return NGX_ERROR; + } ++ ++ *sess = u->ssl_session_ret; ++ ++ return NGX_OK; + } + + + static int +-ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, ++ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L) + { + ngx_connection_t *c; +- ngx_ssl_session_t *ssl_session, **ud; ++ ngx_ssl_session_t *ssl_session; + + if (!u->ssl_session_reuse) { +- lua_pushboolean(L, 1); +- return 1; ++ return 0; + } + +- ud = lua_newuserdata(L, sizeof(ngx_ssl_session_t *)); +- + c = u->peer.connection; + + ssl_session = ngx_ssl_get_session(c); + if (ssl_session == NULL) { +- *ud = NULL; ++ u->ssl_session_ret = NULL; + + } else { +- *ud = ssl_session; ++ u->ssl_session_ret = ssl_session; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua ssl save session: %p", ssl_session); +- +- /* set up the __gc metamethod */ +- lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( +- ssl_session_metatable_key)); +- lua_rawget(L, LUA_REGISTRYINDEX); +- lua_setmetatable(L, -2); ++ "lua tls save session: %p", ssl_session); + } + +- return 1; ++ return 0; ++} ++ ++ ++void ++ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) ++{ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, ++ "lua tls free session: %p", sess); ++ ++ ngx_ssl_free_session(sess); + } + ++ + #endif /* NGX_HTTP_SSL */ + + +@@ -2008,12 +1988,14 @@ ngx_http_lua_socket_prepare_error_retvals(ngx_http_request_t *r, + u_char errstr[NGX_MAX_ERROR_STR]; + u_char *p; + +- if (ft_type & (NGX_HTTP_LUA_SOCKET_FT_RESOLVER +- | NGX_HTTP_LUA_SOCKET_FT_SSL)) +- { ++ if (ft_type & NGX_HTTP_LUA_SOCKET_FT_RESOLVER) { + return 2; + } + ++ if (ft_type & NGX_HTTP_LUA_SOCKET_FT_SSL) { ++ return 0; ++ } ++ + lua_pushnil(L); + + if (ft_type & NGX_HTTP_LUA_SOCKET_FT_TIMEOUT) { +@@ -6101,27 +6083,6 @@ ngx_http_lua_coctx_cleanup(void *data) + } + + +-#if (NGX_HTTP_SSL) +- +-static int +-ngx_http_lua_ssl_free_session(lua_State *L) +-{ +- ngx_ssl_session_t **psession; +- +- psession = lua_touserdata(L, 1); +- if (psession && *psession != NULL) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, +- "lua ssl free session: %p", *psession); +- +- ngx_ssl_free_session(*psession); +- } +- +- return 0; +-} +- +-#endif /* NGX_HTTP_SSL */ +- +- + void + ngx_http_lua_cleanup_conn_pools(lua_State *L) + { +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.h b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.h +index a0a5a518..ee9411bc 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.h ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.h +@@ -120,6 +120,9 @@ struct ngx_http_lua_socket_tcp_upstream_s { + + #if (NGX_HTTP_SSL) + ngx_str_t ssl_name; ++ ngx_ssl_session_t *ssl_session_ret; ++ const char *error_ret; ++ int openssl_error_code_ret; + #endif + + unsigned ft_type:16; +-- +2.32.0 (Apple Git-132) + + +From f5ba21d6f742e6b169d972a81b6124b27c076016 Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 16:54:32 -0700 +Subject: [PATCH 05/17] change: better error when request context couldn't be + found. + +--- + src/ngx_http_lua_socket_tcp.c | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 4ef22c11..abd487fa 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -1627,13 +1627,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { +- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, +- "no ngx_lua ctx found while TLS handshaking"); +- +- ngx_http_lua_assert(NULL); +- +- *errmsg = "no ctx found"; +- return NGX_ERROR; ++ return NGX_HTTP_LUA_FFI_NO_REQ_CTX; + } + + coctx = ctx->cur_co_ctx; +-- +2.32.0 (Apple Git-132) + + +From 78a450d571febf7ba918ecc13369144925d02bcb Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 17:24:07 -0700 +Subject: [PATCH 06/17] feature: TCP cosocket client certificate support. + closes #534 + +--- + src/ngx_http_lua_socket_tcp.c | 60 +++++++++++++++++++++++++++++++---- + 1 file changed, 54 insertions(+), 6 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index abd487fa..61671b70 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -22,7 +22,6 @@ + static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) +-static int ngx_http_lua_socket_tcp_sslhandshake(lua_State *L); + static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); + static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); +@@ -219,9 +218,6 @@ static char ngx_http_lua_upstream_udata_metatable_key; + static char ngx_http_lua_downstream_udata_metatable_key; + static char ngx_http_lua_pool_udata_metatable_key; + static char ngx_http_lua_pattern_udata_metatable_key; +-#if (NGX_HTTP_SSL) +-static char ngx_http_lua_ssl_session_metatable_key; +-#endif + + + #define ngx_http_lua_tcp_socket_metatable_literal_key "__tcp_cosocket_mt" +@@ -1563,13 +1559,16 @@ int + ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, +- int ocsp_status_req, const char **errmsg) ++ int ocsp_status_req, STACK_OF(X509) *chain, EVP_PKEY *pkey, ++ const char **errmsg) + { +- ngx_int_t rc; ++ ngx_int_t rc, i; + ngx_connection_t *c; + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_co_ctx_t *coctx; + const char *busy_rc; ++ ngx_ssl_conn_t *ssl_conn; ++ X509 *x509; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "lua tcp socket tls handshake"); +@@ -1625,6 +1624,8 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ ssl_conn = c->ssl->connection; ++ + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return NGX_HTTP_LUA_FFI_NO_REQ_CTX; +@@ -1647,6 +1648,53 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + u->ssl_session_reuse = enable_session_reuse; + } + ++ if (chain != NULL) { ++ ngx_http_lua_assert(pkey != NULL); /* ensured by resty.core */ ++ ++ if (sk_X509_num(chain) < 1) { ++ ERR_clear_error(); ++ *errmsg = "invalid client certificate chain"; ++ return NGX_ERROR; ++ } ++ ++ x509 = sk_X509_value(chain, 0); ++ if (x509 == NULL) { ++ ERR_clear_error(); ++ *errmsg = "lua tls fetch client certificate from chain failed"; ++ return NGX_ERROR; ++ } ++ ++ if (SSL_use_certificate(ssl_conn, x509) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua tls set client certificate failed"; ++ return NGX_ERROR; ++ } ++ ++ /* read rest of the chain */ ++ ++ for (i = 1; i < sk_X509_num(chain); i++) { ++ x509 = sk_X509_value(chain, i); ++ if (x509 == NULL) { ++ ERR_clear_error(); ++ *errmsg = "lua tls fetch client intermediate certificate " ++ "from chain failed"; ++ return NGX_ERROR; ++ } ++ ++ if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua tls set client intermediate certificate failed"; ++ return NGX_ERROR; ++ } ++ } ++ ++ if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { ++ ERR_clear_error(); ++ *errmsg = "lua ssl set client private key failed"; ++ return NGX_ERROR; ++ } ++ } ++ + if (server_name != NULL && server_name->data != NULL) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "lua tls server name: \"%V\"", server_name); +-- +2.32.0 (Apple Git-132) + + +From 6cc0c89e946ef42adfbc55e8a461ccc2f367254a Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Wed, 18 Sep 2019 17:25:20 -0700 +Subject: [PATCH 07/17] style: style fixes. + +--- + src/ngx_http_lua_socket_tcp.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 61671b70..a7d410c9 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -1736,7 +1736,8 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (u->ssl_name.len >= server_name->len) { + /* reuse it */ +- ngx_memcpy(u->ssl_name.data, server_name->data, server_name->len); ++ ngx_memcpy(u->ssl_name.data, server_name->data, ++ server_name->len); + u->ssl_name.len = server_name->len; + + } else { +-- +2.32.0 (Apple Git-132) + + +From 21cd7779252732a02fa0e596b66a1d4663d2fd64 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Mon, 6 Jan 2020 17:56:10 -0800 +Subject: [PATCH 08/17] cleanup + +--- + src/ngx_http_lua_socket_tcp.c | 24 +++++++++++------------- + 1 file changed, 11 insertions(+), 13 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index a7d410c9..bd7cc7ca 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -1555,6 +1555,7 @@ ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + return NULL; + } + ++ + int + ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, +@@ -1596,7 +1597,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + } + + if (u->raw_downstream || u->body_downstream) { +- *errmsg = "not supported for downstream"; ++ *errmsg = "not supported for downstream sockets"; + return NGX_ERROR; + } + +@@ -1637,7 +1638,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (sess != NULL) { + if (ngx_ssl_set_session(c, sess) != NGX_OK) { +- *errmsg = "lua tls set session failed"; ++ *errmsg = "tls set session failed"; + return NGX_ERROR; + } + +@@ -1660,13 +1661,13 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, 0); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "lua tls fetch client certificate from chain failed"; ++ *errmsg = "tls fetch client certificate from chain failed"; + return NGX_ERROR; + } + + if (SSL_use_certificate(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "lua tls set client certificate failed"; ++ *errmsg = "tls set client certificate failed"; + return NGX_ERROR; + } + +@@ -1676,21 +1677,21 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, i); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "lua tls fetch client intermediate certificate " +- "from chain failed"; ++ *errmsg = "tls fetch client intermediate certificate from " ++ "chain failed"; + return NGX_ERROR; + } + + if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "lua tls set client intermediate certificate failed"; ++ *errmsg = "tls set client intermediate certificate failed"; + return NGX_ERROR; + } + } + + if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { + ERR_clear_error(); +- *errmsg = "lua ssl set client private key failed"; ++ *errmsg = "tls set client private key failed"; + return NGX_ERROR; + } + } +@@ -1709,7 +1710,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + } + + #else +- *errmsg = "OpenSSL has no SNI support"; ++ *errmsg = "no TLS extension support"; + return NGX_ERROR; + #endif + } +@@ -1752,7 +1753,6 @@ new_ssl_name: + u->ssl_name.data = ngx_alloc(server_name->len, ngx_cycle->log); + if (u->ssl_name.data == NULL) { + u->ssl_name.len = 0; +- + *errmsg = "no memory"; + return NGX_ERROR; + } +@@ -1773,7 +1773,7 @@ new_ssl_name: + rc = ngx_ssl_handshake(c); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "ngx_ssl_handshake returned %d", rc); ++ "ngx_ssl_handshake returned: %d", rc); + + if (rc == NGX_AGAIN) { + if (c->write->timer_set) { +@@ -1805,7 +1805,6 @@ new_ssl_name: + + if (rc == NGX_ERROR) { + *errmsg = u->error_ret; +- + return NGX_ERROR; + } + +@@ -1919,7 +1918,6 @@ failed: + } + + +- + int + ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, +-- +2.32.0 (Apple Git-132) + + +From 0bcf4d1a955db9218e8b0e50685c1d0de8c90b9a Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Tue, 24 Nov 2020 01:49:28 -0800 +Subject: [PATCH 09/17] fixed style according to @spacewander's review + +--- + src/ngx_http_lua_socket_tcp.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index bd7cc7ca..1aa37627 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -1536,15 +1536,15 @@ static const char * + ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, unsigned int ops) + { +- if (ops & SOCKET_OP_CONNECT && u->conn_waiting) { ++ if ((ops & SOCKET_OP_CONNECT) && u->conn_waiting) { + return "socket busy connecting"; + } + +- if (ops & SOCKET_OP_READ && u->read_waiting) { ++ if ((ops & SOCKET_OP_READ) && u->read_waiting) { + return "socket busy reading"; + } + +- if (ops & SOCKET_OP_WRITE ++ if ((ops & SOCKET_OP_WRITE) + && (u->write_waiting + || (u->raw_downstream + && (r->connection->buffered & NGX_HTTP_LOWLEVEL_BUFFERED)))) +-- +2.32.0 (Apple Git-132) + + +From 9b010940f77bbd486c1192eed23af7c35baf4cdb Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 13:42:06 +0800 +Subject: [PATCH 10/17] resize tcp_socket_metatable to 7 + +--- + src/ngx_http_lua_socket_tcp.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 1aa37627..7cdc45c4 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -160,6 +160,8 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, ++ SOCKET_CLIENT_CERT_INDEX = 6, ++ SOCKET_CLIENT_KEY_INDEX = 7, + }; + + +@@ -424,7 +426,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 5 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 7 /* narr */, 1 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 36245613be1031b22b0e6b2eec398dac288fe9a5 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:12:13 +0800 +Subject: [PATCH 11/17] change errms tls to ssl + +--- + src/ngx_http_lua_socket_tcp.c | 24 ++++++++++++------------ + 1 file changed, 12 insertions(+), 12 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 7cdc45c4..af986364 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -1574,7 +1574,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + X509 *x509; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tcp socket tls handshake"); ++ "lua tcp socket ssl handshake"); + + if (u == NULL + || u->peer.connection == NULL +@@ -1640,12 +1640,12 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + if (sess != NULL) { + if (ngx_ssl_set_session(c, sess) != NGX_OK) { +- *errmsg = "tls set session failed"; ++ *errmsg = "ssl set session failed"; + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua tls set session: %p", sess); ++ "lua ssl set session: %p", sess); + + } else { + u->ssl_session_reuse = enable_session_reuse; +@@ -1663,13 +1663,13 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, 0); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "tls fetch client certificate from chain failed"; ++ *errmsg = "ssl fetch client certificate from chain failed"; + return NGX_ERROR; + } + + if (SSL_use_certificate(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client certificate failed"; ++ *errmsg = "ssl set client certificate failed"; + return NGX_ERROR; + } + +@@ -1679,28 +1679,28 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + x509 = sk_X509_value(chain, i); + if (x509 == NULL) { + ERR_clear_error(); +- *errmsg = "tls fetch client intermediate certificate from " ++ *errmsg = "ssl fetch client intermediate certificate from " + "chain failed"; + return NGX_ERROR; + } + + if (SSL_add1_chain_cert(ssl_conn, x509) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client intermediate certificate failed"; ++ *errmsg = "ssl set client intermediate certificate failed"; + return NGX_ERROR; + } + } + + if (SSL_use_PrivateKey(ssl_conn, pkey) == 0) { + ERR_clear_error(); +- *errmsg = "tls set client private key failed"; ++ *errmsg = "ssl set client private key failed"; + return NGX_ERROR; + } + } + + if (server_name != NULL && server_name->data != NULL) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua tls server name: \"%V\"", server_name); ++ "lua ssl server name: \"%V\"", server_name); + + #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME + if (SSL_set_tlsext_host_name(c->ssl->connection, +@@ -1926,7 +1926,7 @@ ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + const char **errmsg, int *openssl_error_code) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, +- "lua cosocket get TLS handshake result for upstream: %p", u); ++ "lua cosocket get SSL handshake result for upstream: %p", u); + + if (u->error_ret != NULL) { + *errmsg = u->error_ret; +@@ -1962,7 +1962,7 @@ ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + u->ssl_session_ret = ssl_session; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, +- "lua tls save session: %p", ssl_session); ++ "lua ssl save session: %p", ssl_session); + } + + return 0; +@@ -1973,7 +1973,7 @@ void + ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, +- "lua tls free session: %p", sess); ++ "lua ssl free session: %p", sess); + + ngx_ssl_free_session(sess); + } +-- +2.32.0 (Apple Git-132) + + +From 1f12b89485da6b7ac5dd23810bf094f214dc324e Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:38:49 +0800 +Subject: [PATCH 12/17] rename function name from tls to ssl + +--- + src/ngx_http_lua_socket_tcp.c | 28 ++++++++++++++-------------- + 1 file changed, 14 insertions(+), 14 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index af986364..76e98597 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -22,8 +22,8 @@ + static int ngx_http_lua_socket_tcp(lua_State *L); + static int ngx_http_lua_socket_tcp_connect(lua_State *L); + #if (NGX_HTTP_SSL) +-static void ngx_http_lua_tls_handshake_handler(ngx_connection_t *c); +-static int ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++static void ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c); ++static int ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L); + #endif + static int ngx_http_lua_socket_tcp_receive(lua_State *L); +@@ -1559,7 +1559,7 @@ ngx_http_lua_socket_tcp_check_busy(ngx_http_request_t *r, + + + int +-ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, ++ngx_http_lua_ffi_socket_tcp_sslhandshake(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t *sess, + int enable_session_reuse, ngx_str_t *server_name, int verify, + int ocsp_status_req, STACK_OF(X509) *chain, EVP_PKEY *pkey, +@@ -1614,7 +1614,7 @@ ngx_http_lua_ffi_socket_tcp_tlshandshake(ngx_http_request_t *r, + + u->ssl_session_reuse = enable_session_reuse; + +- (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); ++ (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, NULL); + + return NGX_OK; + } +@@ -1785,13 +1785,13 @@ new_ssl_name: + ngx_add_timer(c->read, u->connect_timeout); + + u->conn_waiting = 1; +- u->write_prepare_retvals = ngx_http_lua_tls_handshake_retval_handler; ++ u->write_prepare_retvals = ngx_http_lua_ssl_handshake_retval_handler; + + ngx_http_lua_cleanup_pending_operation(coctx); + coctx->cleanup = ngx_http_lua_coctx_cleanup; + coctx->data = u; + +- c->ssl->handler = ngx_http_lua_tls_handshake_handler; ++ c->ssl->handler = ngx_http_lua_ssl_handshake_handler; + + if (ctx->entered_content_phase) { + r->write_event_handler = ngx_http_lua_content_wev_handler; +@@ -1803,7 +1803,7 @@ new_ssl_name: + return NGX_AGAIN; + } + +- ngx_http_lua_tls_handshake_handler(c); ++ ngx_http_lua_ssl_handshake_handler(c); + + if (rc == NGX_ERROR) { + *errmsg = u->error_ret; +@@ -1815,7 +1815,7 @@ new_ssl_name: + + + static void +-ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) ++ngx_http_lua_ssl_handshake_handler(ngx_connection_t *c) + { + int waiting; + ngx_int_t rc; +@@ -1860,7 +1860,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " + "certificate verify error: (%d: %s)", + rc, u->error_ret); + } +@@ -1877,7 +1877,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + + llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module); + if (llcf->log_socket_errors) { +- ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua tls " ++ ngx_log_error(NGX_LOG_ERR, dc->log, 0, "lua ssl " + "certificate does not match host \"%V\"", + &u->ssl_name); + } +@@ -1892,7 +1892,7 @@ ngx_http_lua_tls_handshake_handler(ngx_connection_t *c) + ngx_http_lua_socket_handle_conn_success(r, u); + + } else { +- (void) ngx_http_lua_tls_handshake_retval_handler(r, u, NULL); ++ (void) ngx_http_lua_ssl_handshake_retval_handler(r, u, NULL); + } + + if (waiting) { +@@ -1921,7 +1921,7 @@ failed: + + + int +-ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, ++ngx_http_lua_ffi_socket_tcp_get_sslhandshake_result(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, ngx_ssl_session_t **sess, + const char **errmsg, int *openssl_error_code) + { +@@ -1942,7 +1942,7 @@ ngx_http_lua_ffi_socket_tcp_get_tlshandshake_result(ngx_http_request_t *r, + + + static int +-ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, ++ngx_http_lua_ssl_handshake_retval_handler(ngx_http_request_t *r, + ngx_http_lua_socket_tcp_upstream_t *u, lua_State *L) + { + ngx_connection_t *c; +@@ -1970,7 +1970,7 @@ ngx_http_lua_tls_handshake_retval_handler(ngx_http_request_t *r, + + + void +-ngx_http_lua_ffi_tls_free_session(ngx_ssl_session_t *sess) ++ngx_http_lua_ffi_ssl_free_session(ngx_ssl_session_t *sess) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "lua ssl free session: %p", sess); +-- +2.32.0 (Apple Git-132) + + +From 84242561aa54ffed3bfab433cfef6f7797e01a47 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:46:38 +0800 +Subject: [PATCH 13/17] rename to SOCKET_CLIENT_PRIV_INDEX + +--- + src/ngx_http_lua_socket_tcp.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 76e98597..90da45fc 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -160,8 +160,8 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, +- SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_KEY_INDEX = 7, ++ SOCKET_CLIENT_CERT_INDEX = 6, ++ SOCKET_CLIENT_PRIV_INDEX = 7, + }; + + +-- +2.32.0 (Apple Git-132) + + +From 555166646c525167f9e1e5bb81b6cb100a4834f9 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 14:49:18 +0800 +Subject: [PATCH 14/17] rename to SOCKET_CLIENT_PKEY_INDEX + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 90da45fc..494486de 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -161,7 +161,7 @@ enum { + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, + SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_PRIV_INDEX = 7, ++ SOCKET_CLIENT_PKEY_INDEX = 7, + }; + + +-- +2.32.0 (Apple Git-132) + + +From e9b54c43c05b064b831fe67d0e0aaff45b2ec505 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Fri, 21 Jan 2022 17:17:09 +0800 +Subject: [PATCH 15/17] need not to change tcp_socket_metatable + +--- + src/ngx_http_lua_socket_tcp.c | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 494486de..152d8cbd 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -160,8 +160,6 @@ enum { + SOCKET_CONNECT_TIMEOUT_INDEX = 2, + SOCKET_SEND_TIMEOUT_INDEX = 4, + SOCKET_READ_TIMEOUT_INDEX = 5, +- SOCKET_CLIENT_CERT_INDEX = 6, +- SOCKET_CLIENT_PKEY_INDEX = 7, + }; + + +@@ -426,7 +424,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 7 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 5 /* narr */, 1 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 6c47356ddc327a8692260bd6f43ea67cf2787a73 Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Wed, 26 Jan 2022 19:55:29 +0800 +Subject: [PATCH 16/17] increase nrec to 3 in the socket object + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 152d8cbd..8d71f8b4 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -424,7 +424,7 @@ ngx_http_lua_socket_tcp(lua_State *L) + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + +- lua_createtable(L, 5 /* narr */, 1 /* nrec */); ++ lua_createtable(L, 5 /* narr */, 3 /* nrec */); + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); + lua_rawget(L, LUA_REGISTRYINDEX); +-- +2.32.0 (Apple Git-132) + + +From 1d538552c7629310d850d4360408ddb555afcbcc Mon Sep 17 00:00:00 2001 +From: chronolaw +Date: Sat, 29 Jan 2022 09:18:52 +0800 +Subject: [PATCH 17/17] change tcp_socket_metatable nrec to 15 + +--- + src/ngx_http_lua_socket_tcp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +index 8d71f8b4..5dcdef0e 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_socket_tcp.c +@@ -312,7 +312,7 @@ ngx_http_lua_inject_socket_tcp_api(ngx_log_t *log, lua_State *L) + /* {{{tcp object metatable */ + lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( + tcp_socket_metatable_key)); +- lua_createtable(L, 0 /* narr */, 14 /* nrec */); ++ lua_createtable(L, 0 /* narr */, 15 /* nrec */); + + lua_pushcfunction(L, ngx_http_lua_socket_tcp_connect); + lua_setfield(L, -2, "connect"); +-- +2.32.0 (Apple Git-132) + diff --git a/build/openresty/patches/ngx_lua-0.10.21_02-dyn_upstream_keepalive.patch b/build/openresty/patches/ngx_lua-0.10.21_02-dyn_upstream_keepalive.patch new file mode 100644 index 000000000000..84304498b42e --- /dev/null +++ b/build/openresty/patches/ngx_lua-0.10.21_02-dyn_upstream_keepalive.patch @@ -0,0 +1,1319 @@ +From 2d12ac3e4045258b7a174b0505d92f63c26d82fc Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:44 -0700 +Subject: [PATCH 1/3] feature: implemented keepalive pooling in + 'balancer_by_lua*'. + +--- + src/ngx_http_lua_balancer.c | 738 ++++++++++++++++++++++++++++++------ + src/ngx_http_lua_common.h | 4 + + src/ngx_http_lua_module.c | 3 + + 3 files changed, 629 insertions(+), 116 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +index f71a3e00..0d403716 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +@@ -16,46 +16,102 @@ + #include "ngx_http_lua_directive.h" + + ++typedef struct { ++ ngx_uint_t size; ++ ngx_uint_t connections; ++ ++ uint32_t crc32; ++ ++ lua_State *lua_vm; ++ ++ ngx_queue_t cache; ++ ngx_queue_t free; ++} ngx_http_lua_balancer_keepalive_pool_t; ++ ++ ++typedef struct { ++ ngx_queue_t queue; ++ ngx_connection_t *connection; ++ ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++} ngx_http_lua_balancer_keepalive_item_t; ++ ++ + struct ngx_http_lua_balancer_peer_data_s { +- /* the round robin data must be first */ +- ngx_http_upstream_rr_peer_data_t rrp; ++ ngx_uint_t cpool_size; ++ ngx_uint_t keepalive_requests; ++ ngx_msec_t keepalive_timeout; ++ ++ ngx_uint_t more_tries; ++ ngx_uint_t total_tries; + +- ngx_http_lua_srv_conf_t *conf; +- ngx_http_request_t *request; ++ int last_peer_state; + +- ngx_uint_t more_tries; +- ngx_uint_t total_tries; ++ uint32_t cpool_crc32; + +- struct sockaddr *sockaddr; +- socklen_t socklen; ++ void *data; + +- ngx_str_t *host; +- in_port_t port; ++ ngx_event_get_peer_pt original_get_peer; ++ ngx_event_free_peer_pt original_free_peer; + +- int last_peer_state; ++#if (NGX_HTTP_SSL) ++ ngx_event_set_peer_session_pt original_set_session; ++ ngx_event_save_peer_session_pt original_save_session; ++#endif ++ ++ ngx_http_request_t *request; ++ ngx_http_lua_srv_conf_t *conf; ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++ ++ ngx_str_t *host; ++ ++ struct sockaddr *sockaddr; ++ socklen_t socklen; ++ ++ unsigned keepalive:1; + + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) +- unsigned cloned_upstream_conf; /* :1 */ ++ unsigned cloned_upstream_conf:1; + #endif + }; + + +-#if (NGX_HTTP_SSL) +-static ngx_int_t ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, +- void *data); +-static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, +- void *data); +-#endif ++static ngx_int_t ngx_http_lua_balancer_by_chunk(lua_State *L, ++ ngx_http_request_t *r); + static ngx_int_t ngx_http_lua_balancer_init(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us); + static ngx_int_t ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); + static ngx_int_t ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, + void *data); +-static ngx_int_t ngx_http_lua_balancer_by_chunk(lua_State *L, +- ngx_http_request_t *r); + static void ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); ++static ngx_int_t ngx_http_lua_balancer_create_keepalive_pool(lua_State *L, ++ ngx_log_t *log, uint32_t cpool_crc32, ngx_uint_t cpool_size, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool); ++static void ngx_http_lua_balancer_get_keepalive_pool(lua_State *L, ++ uint32_t cpool_crc32, ngx_http_lua_balancer_keepalive_pool_t **cpool); ++static void ngx_http_lua_balancer_free_keepalive_pool(ngx_log_t *log, ++ ngx_http_lua_balancer_keepalive_pool_t *cpool); ++static void ngx_http_lua_balancer_close(ngx_connection_t *c); ++static void ngx_http_lua_balancer_dummy_handler(ngx_event_t *ev); ++static void ngx_http_lua_balancer_close_handler(ngx_event_t *ev); ++#if (NGX_HTTP_SSL) ++static ngx_int_t ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, ++ void *data); ++static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, ++ void *data); ++#endif ++ ++ ++#define ngx_http_lua_balancer_keepalive_is_enabled(bp) \ ++ (bp->keepalive) ++ ++#define ngx_http_lua_balancer_peer_set(bp) \ ++ (bp->sockaddr && bp->socklen) ++ ++ ++static char ngx_http_lua_balancer_keepalive_pools_table_key; + + + ngx_int_t +@@ -102,6 +158,61 @@ ngx_http_lua_balancer_handler_inline(ngx_http_request_t *r, + } + + ++static ngx_int_t ++ngx_http_lua_balancer_by_chunk(lua_State *L, ngx_http_request_t *r) ++{ ++ u_char *err_msg; ++ size_t len; ++ ngx_int_t rc; ++ ++ /* init nginx context in Lua VM */ ++ ngx_http_lua_set_req(L, r); ++ ++#ifndef OPENRESTY_LUAJIT ++ ngx_http_lua_create_new_globals_table(L, 0 /* narr */, 1 /* nrec */); ++ ++ /* {{{ make new env inheriting main thread's globals table */ ++ lua_createtable(L, 0, 1 /* nrec */); /* the metatable for the new env */ ++ ngx_http_lua_get_globals_table(L); ++ lua_setfield(L, -2, "__index"); ++ lua_setmetatable(L, -2); /* setmetatable({}, {__index = _G}) */ ++ /* }}} */ ++ ++ lua_setfenv(L, -2); /* set new running env for the code closure */ ++#endif /* OPENRESTY_LUAJIT */ ++ ++ lua_pushcfunction(L, ngx_http_lua_traceback); ++ lua_insert(L, 1); /* put it under chunk and args */ ++ ++ /* protected call user code */ ++ rc = lua_pcall(L, 0, 1, 1); ++ ++ lua_remove(L, 1); /* remove traceback function */ ++ ++ dd("rc == %d", (int) rc); ++ ++ if (rc != 0) { ++ /* error occurred when running loaded code */ ++ err_msg = (u_char *) lua_tolstring(L, -1, &len); ++ ++ if (err_msg == NULL) { ++ err_msg = (u_char *) "unknown reason"; ++ len = sizeof("unknown reason") - 1; ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "failed to run balancer_by_lua*: %*s", len, err_msg); ++ ++ lua_settop(L, 0); /* clear remaining elems on stack */ ++ ++ return NGX_ERROR; ++ } ++ ++ lua_settop(L, 0); /* clear remaining elems on stack */ ++ return rc; ++} ++ ++ + char * + ngx_http_lua_balancer_by_lua_block(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +@@ -125,16 +236,16 @@ char * + ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) + { +- u_char *cache_key = NULL; +- u_char *name; +- ngx_str_t *value; +- ngx_http_lua_srv_conf_t *lscf = conf; +- ++ u_char *cache_key = NULL; ++ u_char *name; ++ ngx_str_t *value; + ngx_http_upstream_srv_conf_t *uscf; ++ ngx_http_lua_srv_conf_t *lscf = conf; + + dd("enter"); + +- /* must specify a content handler */ ++ /* content handler setup */ ++ + if (cmd->post == NULL) { + return NGX_CONF_ERROR; + } +@@ -178,11 +289,19 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + lscf->balancer.src_key = cache_key; + ++ /* balancer setup */ ++ + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); ++ ++ lscf->balancer.original_init_upstream = uscf->peer.init_upstream; ++ ++ } else { ++ lscf->balancer.original_init_upstream = ++ ngx_http_upstream_init_round_robin; + } + + uscf->peer.init_upstream = ngx_http_lua_balancer_init; +@@ -198,14 +317,18 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + + static ngx_int_t +-ngx_http_lua_balancer_init(ngx_conf_t *cf, +- ngx_http_upstream_srv_conf_t *us) ++ngx_http_lua_balancer_init(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) + { +- if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { ++ ngx_http_lua_srv_conf_t *lscf; ++ ++ lscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); ++ ++ if (lscf->balancer.original_init_upstream(cf, us) != NGX_OK) { + return NGX_ERROR; + } + +- /* this callback is called upon individual requests */ ++ lscf->balancer.original_init_peer = us->peer.init; ++ + us->peer.init = ngx_http_lua_balancer_init_peer; + + return NGX_OK; +@@ -216,33 +339,38 @@ static ngx_int_t + ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) + { +- ngx_http_lua_srv_conf_t *bcf; ++ ngx_http_lua_srv_conf_t *lscf; + ngx_http_lua_balancer_peer_data_t *bp; + +- bp = ngx_pcalloc(r->pool, sizeof(ngx_http_lua_balancer_peer_data_t)); +- if (bp == NULL) { ++ lscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); ++ ++ if (lscf->balancer.original_init_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + +- r->upstream->peer.data = &bp->rrp; +- +- if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { ++ bp = ngx_pcalloc(r->pool, sizeof(ngx_http_lua_balancer_peer_data_t)); ++ if (bp == NULL) { + return NGX_ERROR; + } + ++ bp->conf = lscf; ++ bp->request = r; ++ bp->data = r->upstream->peer.data; ++ bp->original_get_peer = r->upstream->peer.get; ++ bp->original_free_peer = r->upstream->peer.free; ++ ++ r->upstream->peer.data = bp; + r->upstream->peer.get = ngx_http_lua_balancer_get_peer; + r->upstream->peer.free = ngx_http_lua_balancer_free_peer; + + #if (NGX_HTTP_SSL) ++ bp->original_set_session = r->upstream->peer.set_session; ++ bp->original_save_session = r->upstream->peer.save_session; ++ + r->upstream->peer.set_session = ngx_http_lua_balancer_set_session; + r->upstream->peer.save_session = ngx_http_lua_balancer_save_session; + #endif + +- bcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_lua_module); +- +- bp->conf = bcf; +- bp->request = r; +- + return NGX_OK; + } + +@@ -250,25 +378,26 @@ ngx_http_lua_balancer_init_peer(ngx_http_request_t *r, + static ngx_int_t + ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + { +- lua_State *L; +- ngx_int_t rc; +- ngx_http_request_t *r; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_srv_conf_t *lscf; +- ngx_http_lua_main_conf_t *lmcf; +- ngx_http_lua_balancer_peer_data_t *bp = data; ++ lua_State *L; ++ ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_request_t *r; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_srv_conf_t *lscf; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ngx_http_lua_balancer_peer_data_t *bp = data; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, +- "lua balancer peer, tries: %ui", pc->tries); +- +- lscf = bp->conf; ++ "lua balancer: get peer, tries: %ui", pc->tries); + + r = bp->request; ++ lscf = bp->conf; + + ngx_http_lua_assert(lscf->balancer.handler && r); + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); +- + if (ctx == NULL) { + ctx = ngx_http_lua_create_ctx(r); + if (ctx == NULL) { +@@ -286,9 +415,15 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + + ctx->context = NGX_HTTP_LUA_CONTEXT_BALANCER; + ++ bp->cpool = NULL; + bp->sockaddr = NULL; + bp->socklen = 0; + bp->more_tries = 0; ++ bp->cpool_crc32 = 0; ++ bp->cpool_size = 0; ++ bp->keepalive_requests = 0; ++ bp->keepalive_timeout = 0; ++ bp->keepalive = 0; + bp->total_tries++; + + lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +@@ -300,7 +435,6 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + lmcf->balancer_peer_data = bp; + + rc = lscf->balancer.handler(r, lscf, L); +- + if (rc == NGX_ERROR) { + return NGX_ERROR; + } +@@ -322,105 +456,414 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + } + } + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + pc->sockaddr = bp->sockaddr; + pc->socklen = bp->socklen; ++ pc->name = bp->host; + pc->cached = 0; + pc->connection = NULL; +- pc->name = bp->host; +- +- bp->rrp.peers->single = 0; + + if (bp->more_tries) { + r->upstream->peer.tries += bp->more_tries; + } + +- dd("tries: %d", (int) r->upstream->peer.tries); ++ if (ngx_http_lua_balancer_keepalive_is_enabled(bp)) { ++ ngx_http_lua_balancer_get_keepalive_pool(L, bp->cpool_crc32, ++ &bp->cpool); ++ ++ if (bp->cpool == NULL ++ && ngx_http_lua_balancer_create_keepalive_pool(L, pc->log, ++ bp->cpool_crc32, ++ bp->cpool_size, ++ &bp->cpool) ++ != NGX_OK) ++ { ++ return NGX_ERROR; ++ } ++ ++ ngx_http_lua_assert(bp->cpool); ++ ++ if (!ngx_queue_empty(&bp->cpool->cache)) { ++ q = ngx_queue_head(&bp->cpool->cache); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ c = item->connection; ++ ++ ngx_queue_remove(q); ++ ngx_queue_insert_head(&bp->cpool->free, q); ++ ++ c->idle = 0; ++ c->sent = 0; ++ c->log = pc->log; ++ c->read->log = pc->log; ++ c->write->log = pc->log; ++ c->pool->log = pc->log; ++ ++ if (c->read->timer_set) { ++ ngx_del_timer(c->read); ++ } ++ ++ pc->cached = 1; ++ pc->connection = c; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive reusing connection %p, " ++ "requests: %ui, cpool: %p", ++ c, c->requests, bp->cpool); ++ ++ return NGX_DONE; ++ } ++ ++ bp->cpool->connections++; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive no free connection, " ++ "cpool: %p", bp->cpool); ++ } + + return NGX_OK; + } + +- return ngx_http_upstream_get_round_robin_peer(pc, &bp->rrp); ++ return bp->original_get_peer(pc, bp->data); + } + + +-static ngx_int_t +-ngx_http_lua_balancer_by_chunk(lua_State *L, ngx_http_request_t *r) ++static void ++ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, void *data, ++ ngx_uint_t state) + { +- u_char *err_msg; +- size_t len; +- ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_upstream_t *u; ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ngx_http_lua_balancer_keepalive_pool_t *cpool; ++ ngx_http_lua_balancer_peer_data_t *bp = data; + +- /* init nginx context in Lua VM */ +- ngx_http_lua_set_req(L, r); ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: free peer, tries: %ui", pc->tries); + +-#ifndef OPENRESTY_LUAJIT +- ngx_http_lua_create_new_globals_table(L, 0 /* narr */, 1 /* nrec */); ++ u = bp->request->upstream; ++ c = pc->connection; + +- /* {{{ make new env inheriting main thread's globals table */ +- lua_createtable(L, 0, 1 /* nrec */); /* the metatable for the new env */ +- ngx_http_lua_get_globals_table(L); +- lua_setfield(L, -2, "__index"); +- lua_setmetatable(L, -2); /* setmetatable({}, {__index = _G}) */ +- /* }}} */ ++ if (ngx_http_lua_balancer_peer_set(bp)) { ++ bp->last_peer_state = (int) state; + +- lua_setfenv(L, -2); /* set new running env for the code closure */ +-#endif /* OPENRESTY_LUAJIT */ ++ if (pc->tries) { ++ pc->tries--; ++ } + +- lua_pushcfunction(L, ngx_http_lua_traceback); +- lua_insert(L, 1); /* put it under chunk and args */ ++ if (ngx_http_lua_balancer_keepalive_is_enabled(bp)) { ++ cpool = bp->cpool; + +- /* protected call user code */ +- rc = lua_pcall(L, 0, 1, 1); ++ if (state & NGX_PEER_FAILED ++ || c == NULL ++ || c->read->eof ++ || c->read->error ++ || c->read->timedout ++ || c->write->error ++ || c->write->timedout) ++ { ++ goto invalid; ++ } + +- lua_remove(L, 1); /* remove traceback function */ ++ if (bp->keepalive_requests ++ && c->requests >= bp->keepalive_requests) ++ { ++ goto invalid; ++ } + +- dd("rc == %d", (int) rc); ++ if (!u->keepalive) { ++ goto invalid; ++ } + +- if (rc != 0) { +- /* error occurred when running loaded code */ +- err_msg = (u_char *) lua_tolstring(L, -1, &len); ++ if (!u->request_body_sent) { ++ goto invalid; ++ } + +- if (err_msg == NULL) { +- err_msg = (u_char *) "unknown reason"; +- len = sizeof("unknown reason") - 1; ++ if (ngx_terminate || ngx_exiting) { ++ goto invalid; ++ } ++ ++ if (ngx_handle_read_event(c->read, 0) != NGX_OK) { ++ goto invalid; ++ } ++ ++ if (ngx_queue_empty(&cpool->free)) { ++ q = ngx_queue_last(&cpool->cache); ++ ngx_queue_remove(q); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ ++ ngx_http_lua_balancer_close(item->connection); ++ ++ } else { ++ q = ngx_queue_head(&cpool->free); ++ ngx_queue_remove(q); ++ ++ item = ngx_queue_data(q, ngx_http_lua_balancer_keepalive_item_t, ++ queue); ++ } ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive saving connection %p, " ++ "cpool: %p, connections: %ui", ++ c, cpool, cpool->connections); ++ ++ ngx_queue_insert_head(&cpool->cache, q); ++ ++ item->connection = c; ++ ++ pc->connection = NULL; ++ ++ if (bp->keepalive_timeout) { ++ c->read->delayed = 0; ++ ngx_add_timer(c->read, bp->keepalive_timeout); ++ ++ } else if (c->read->timer_set) { ++ ngx_del_timer(c->read); ++ } ++ ++ if (c->write->timer_set) { ++ ngx_del_timer(c->write); ++ } ++ ++ c->write->handler = ngx_http_lua_balancer_dummy_handler; ++ c->read->handler = ngx_http_lua_balancer_close_handler; ++ ++ c->data = item; ++ c->idle = 1; ++ c->log = ngx_cycle->log; ++ c->read->log = ngx_cycle->log; ++ c->write->log = ngx_cycle->log; ++ c->pool->log = ngx_cycle->log; ++ ++ if (c->read->ready) { ++ ngx_http_lua_balancer_close_handler(c->read); ++ } ++ ++ return; ++ ++invalid: ++ ++ cpool->connections--; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, ++ "lua balancer: keepalive not saving connection %p, " ++ "cpool: %p, connections: %ui", ++ c, cpool, cpool->connections); ++ ++ if (cpool->connections == 0) { ++ ngx_http_lua_balancer_free_keepalive_pool(pc->log, cpool); ++ } + } + +- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, +- "failed to run balancer_by_lua*: %*s", len, err_msg); ++ return; ++ } + +- lua_settop(L, 0); /* clear remaining elems on stack */ ++ bp->original_free_peer(pc, bp->data, state); ++} ++ ++ ++static ngx_int_t ++ngx_http_lua_balancer_create_keepalive_pool(lua_State *L, ngx_log_t *log, ++ uint32_t cpool_crc32, ngx_uint_t cpool_size, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool) ++{ ++ size_t size; ++ ngx_uint_t i; ++ ngx_http_lua_balancer_keepalive_pool_t *upool; ++ ngx_http_lua_balancer_keepalive_item_t *items; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ ngx_http_lua_assert(lua_istable(L, -1)); ++ ++ size = sizeof(ngx_http_lua_balancer_keepalive_pool_t) ++ + sizeof(ngx_http_lua_balancer_keepalive_item_t) * cpool_size; + ++ upool = lua_newuserdata(L, size); /* pools upool */ ++ if (upool == NULL) { + return NGX_ERROR; + } + +- lua_settop(L, 0); /* clear remaining elems on stack */ +- return rc; ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, ++ "lua balancer: keepalive create pool, crc32: %ui, " ++ "size: %ui", cpool_crc32, cpool_size); ++ ++ upool->lua_vm = L; ++ upool->crc32 = cpool_crc32; ++ upool->size = cpool_size; ++ upool->connections = 0; ++ ++ ngx_queue_init(&upool->cache); ++ ngx_queue_init(&upool->free); ++ ++ lua_rawseti(L, -2, cpool_crc32); /* pools */ ++ lua_pop(L, 1); /* orig stack */ ++ ++ items = (ngx_http_lua_balancer_keepalive_item_t *) (&upool->free + 1); ++ ++ ++ ++ for (i = 0; i < cpool_size; i++) { ++ ngx_queue_insert_head(&upool->free, &items[i].queue); ++ items[i].cpool = upool; ++ } ++ ++ *cpool = upool; ++ ++ return NGX_OK; + } + + + static void +-ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, void *data, +- ngx_uint_t state) ++ngx_http_lua_balancer_get_keepalive_pool(lua_State *L, uint32_t cpool_crc32, ++ ngx_http_lua_balancer_keepalive_pool_t **cpool) + { +- ngx_http_lua_balancer_peer_data_t *bp = data; ++ ngx_http_lua_balancer_keepalive_pool_t *upool; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ if (lua_isnil(L, -1)) { ++ lua_pop(L, 1); /* orig stack */ ++ ++ /* create upstream connection pools table */ ++ lua_createtable(L, 0, 0); /* pools */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_pushvalue(L, -2); /* pools pools_table_key pools */ ++ lua_rawset(L, LUA_REGISTRYINDEX); /* pools */ ++ } + +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, +- "lua balancer free peer, tries: %ui", pc->tries); ++ ngx_http_lua_assert(lua_istable(L, -1)); + +- if (bp->sockaddr && bp->socklen) { +- bp->last_peer_state = (int) state; ++ lua_rawgeti(L, -1, cpool_crc32); /* pools upool? */ ++ upool = lua_touserdata(L, -1); ++ lua_pop(L, 2); /* orig stack */ + +- if (pc->tries) { +- pc->tries--; ++ *cpool = upool; ++} ++ ++ ++static void ++ngx_http_lua_balancer_free_keepalive_pool(ngx_log_t *log, ++ ngx_http_lua_balancer_keepalive_pool_t *cpool) ++{ ++ lua_State *L; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, ++ "lua balancer: keepalive free pool %p, crc32: %ui", ++ cpool, cpool->crc32); ++ ++ ngx_http_lua_assert(cpool->connections == 0); ++ ++ L = cpool->lua_vm; ++ ++ /* get upstream connection pools table */ ++ lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask( ++ balancer_keepalive_pools_table_key)); ++ lua_rawget(L, LUA_REGISTRYINDEX); /* pools? */ ++ ++ if (lua_isnil(L, -1)) { ++ lua_pop(L, 1); /* orig stack */ ++ return; ++ } ++ ++ ngx_http_lua_assert(lua_istable(L, -1)); ++ ++ lua_pushnil(L); /* pools nil */ ++ lua_rawseti(L, -2, cpool->crc32); /* pools */ ++ lua_pop(L, 1); /* orig stack */ ++} ++ ++ ++static void ++ngx_http_lua_balancer_close(ngx_connection_t *c) ++{ ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ++ item = c->data; ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ c->ssl->no_wait_shutdown = 1; ++ c->ssl->no_send_shutdown = 1; ++ ++ if (ngx_ssl_shutdown(c) == NGX_AGAIN) { ++ c->ssl->handler = ngx_http_lua_balancer_close; ++ return; ++ } ++ } ++#endif ++ ++ ngx_destroy_pool(c->pool); ++ ngx_close_connection(c); ++ ++ item->cpool->connections--; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "lua balancer: keepalive closing connection %p, cpool: %p, " ++ "connections: %ui", ++ c, item->cpool, item->cpool->connections); ++} ++ ++ ++static void ++ngx_http_lua_balancer_dummy_handler(ngx_event_t *ev) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, ++ "lua balancer: dummy handler"); ++} ++ ++ ++static void ++ngx_http_lua_balancer_close_handler(ngx_event_t *ev) ++{ ++ ngx_http_lua_balancer_keepalive_item_t *item; ++ ++ int n; ++ char buf[1]; ++ ngx_connection_t *c; ++ ++ c = ev->data; ++ ++ if (c->close || c->read->timedout) { ++ goto close; ++ } ++ ++ n = recv(c->fd, buf, 1, MSG_PEEK); ++ ++ if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { ++ ev->ready = 0; ++ ++ if (ngx_handle_read_event(c->read, 0) != NGX_OK) { ++ goto close; + } + + return; + } + +- /* fallback */ ++close: ++ ++ item = c->data; ++ c->log = ev->log; ++ ++ ngx_http_lua_balancer_close(c); + +- ngx_http_upstream_free_round_robin_peer(pc, data, state); ++ ngx_queue_remove(&item->queue); ++ ngx_queue_insert_head(&item->cpool->free, &item->queue); ++ ++ if (item->cpool->connections == 0) { ++ ngx_http_lua_balancer_free_keepalive_pool(ev->log, item->cpool); ++ } + } + + +@@ -431,12 +874,12 @@ ngx_http_lua_balancer_set_session(ngx_peer_connection_t *pc, void *data) + { + ngx_http_lua_balancer_peer_data_t *bp = data; + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + /* TODO */ + return NGX_OK; + } + +- return ngx_http_upstream_set_round_robin_peer_session(pc, &bp->rrp); ++ return bp->original_set_session(pc, bp->data); + } + + +@@ -445,13 +888,12 @@ ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, void *data) + { + ngx_http_lua_balancer_peer_data_t *bp = data; + +- if (bp->sockaddr && bp->socklen) { ++ if (ngx_http_lua_balancer_peer_set(bp)) { + /* TODO */ + return; + } + +- ngx_http_upstream_save_round_robin_peer_session(pc, &bp->rrp); +- return; ++ bp->original_save_session(pc, bp->data); + } + + #endif +@@ -459,14 +901,14 @@ ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, void *data) + + int + ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, +- const u_char *addr, size_t addr_len, int port, char **err) ++ const u_char *addr, size_t addr_len, int port, unsigned int cpool_crc32, ++ unsigned int cpool_size, char **err) + { +- ngx_url_t url; +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- +- ngx_http_lua_main_conf_t *lmcf; +- ngx_http_lua_balancer_peer_data_t *bp; ++ ngx_url_t url; ++ ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { + *err = "no request found"; +@@ -536,6 +978,70 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ bp->cpool_crc32 = (uint32_t) cpool_crc32; ++ bp->cpool_size = (ngx_uint_t) cpool_size; ++ ++ return NGX_OK; ++} ++ ++ ++int ++ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, ++ unsigned long timeout, unsigned int max_requests, char **err) ++{ ++ ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_balancer_peer_data_t *bp; ++ ++ if (r == NULL) { ++ *err = "no request found"; ++ return NGX_ERROR; ++ } ++ ++ u = r->upstream; ++ ++ if (u == NULL) { ++ *err = "no upstream found"; ++ return NGX_ERROR; ++ } ++ ++ ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); ++ if (ctx == NULL) { ++ *err = "no ctx found"; ++ return NGX_ERROR; ++ } ++ ++ if ((ctx->context & NGX_HTTP_LUA_CONTEXT_BALANCER) == 0) { ++ *err = "API disabled in the current context"; ++ return NGX_ERROR; ++ } ++ ++ lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); ++ ++ /* we cannot read r->upstream->peer.data here directly because ++ * it could be overridden by other modules like ++ * ngx_http_upstream_keepalive_module. ++ */ ++ bp = lmcf->balancer_peer_data; ++ if (bp == NULL) { ++ *err = "no upstream peer data found"; ++ return NGX_ERROR; ++ } ++ ++ if (!ngx_http_lua_balancer_peer_set(bp)) { ++ *err = "no current peer set"; ++ return NGX_ERROR; ++ } ++ ++ if (!bp->cpool_crc32) { ++ bp->cpool_crc32 = ngx_crc32_long(bp->host->data, bp->host->len); ++ } ++ ++ bp->keepalive_timeout = (ngx_msec_t) timeout; ++ bp->keepalive_requests = (ngx_uint_t) max_requests; ++ bp->keepalive = 1; ++ + return NGX_OK; + } + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h +index 781a2454..9ce6836a 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h +@@ -328,6 +328,10 @@ union ngx_http_lua_srv_conf_u { + #endif + + struct { ++ ngx_http_upstream_init_pt original_init_upstream; ++ ngx_http_upstream_init_peer_pt original_init_peer; ++ uintptr_t data; ++ + ngx_http_lua_srv_conf_handler_pt handler; + ngx_str_t src; + u_char *src_key; +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_module.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_module.c +index 9816d864..5d7cedfd 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_module.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_module.c +@@ -1117,6 +1117,9 @@ ngx_http_lua_create_srv_conf(ngx_conf_t *cf) + * lscf->srv.ssl_session_fetch_src = { 0, NULL }; + * lscf->srv.ssl_session_fetch_src_key = NULL; + * ++ * lscf->balancer.original_init_upstream = NULL; ++ * lscf->balancer.original_init_peer = NULL; ++ * lscf->balancer.data = NULL; + * lscf->balancer.handler = NULL; + * lscf->balancer.src = { 0, NULL }; + * lscf->balancer.src_key = NULL; +-- +2.26.2 + + +From 4c5cb29a265b2f9524434322adf15d07deec6c7f Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:54 -0700 +Subject: [PATCH 2/3] feature: we now avoid the need for 'upstream' blocks to + define a stub 'server' directive when using 'balancer_by_lua*'. + +--- + src/ngx_http_lua_balancer.c | 42 +++++++++++++++++++++++++++++++++++-- + 1 file changed, 40 insertions(+), 2 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +index 0d403716..5c862d22 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +@@ -111,7 +111,8 @@ static void ngx_http_lua_balancer_save_session(ngx_peer_connection_t *pc, + (bp->sockaddr && bp->socklen) + + +-static char ngx_http_lua_balancer_keepalive_pools_table_key; ++static char ngx_http_lua_balancer_keepalive_pools_table_key; ++static struct sockaddr *ngx_http_lua_balancer_default_server_sockaddr; + + + ngx_int_t +@@ -239,7 +240,9 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + u_char *cache_key = NULL; + u_char *name; + ngx_str_t *value; ++ ngx_url_t url; + ngx_http_upstream_srv_conf_t *uscf; ++ ngx_http_upstream_server_t *us; + ngx_http_lua_srv_conf_t *lscf = conf; + + dd("enter"); +@@ -293,6 +296,29 @@ ngx_http_lua_balancer_by_lua(ngx_conf_t *cf, ngx_command_t *cmd, + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + ++ if (uscf->servers->nelts == 0) { ++ us = ngx_array_push(uscf->servers); ++ if (us == NULL) { ++ return NGX_CONF_ERROR; ++ } ++ ++ ngx_memzero(us, sizeof(ngx_http_upstream_server_t)); ++ ngx_memzero(&url, sizeof(ngx_url_t)); ++ ++ ngx_str_set(&url.url, "0.0.0.1"); ++ url.default_port = 80; ++ ++ if (ngx_parse_url(cf->pool, &url) != NGX_OK) { ++ return NGX_CONF_ERROR; ++ } ++ ++ us->name = url.url; ++ us->addrs = url.addrs; ++ us->naddrs = url.naddrs; ++ ++ ngx_http_lua_balancer_default_server_sockaddr = us->addrs[0].sockaddr; ++ } ++ + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); +@@ -525,7 +551,19 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + return NGX_OK; + } + +- return bp->original_get_peer(pc, bp->data); ++ rc = bp->original_get_peer(pc, bp->data); ++ if (rc == NGX_ERROR) { ++ return rc; ++ } ++ ++ if (pc->sockaddr == ngx_http_lua_balancer_default_server_sockaddr) { ++ ngx_log_error(NGX_LOG_ERR, pc->log, 0, ++ "lua balancer: no peer set"); ++ ++ return NGX_ERROR; ++ } ++ ++ return rc; + } + + +-- +2.26.2 + + +From 941cd893573561574bc6a326d6306f1a30127293 Mon Sep 17 00:00:00 2001 +From: Thibault Charbonnier +Date: Tue, 17 Sep 2019 11:43:58 -0700 +Subject: [PATCH 3/3] refactor: used a simpler way to stash the balancer peer + data. + +--- + src/ngx_http_lua_balancer.c | 91 +++++++++---------------------------- + src/ngx_http_lua_common.h | 7 --- + 2 files changed, 22 insertions(+), 76 deletions(-) + +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +index 5c862d22..3ea1f067 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_balancer.c +@@ -411,9 +411,9 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_srv_conf_t *lscf; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_keepalive_item_t *item; + ngx_http_lua_balancer_peer_data_t *bp = data; ++ void *pdata; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "lua balancer: get peer, tries: %ui", pc->tries); +@@ -452,15 +452,13 @@ ngx_http_lua_balancer_get_peer(ngx_peer_connection_t *pc, void *data) + bp->keepalive = 0; + bp->total_tries++; + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* balancer_by_lua does not support yielding and +- * there cannot be any conflicts among concurrent requests, +- * thus it is safe to store the peer data in the main conf. +- */ +- lmcf->balancer_peer_data = bp; ++ pdata = r->upstream->peer.data; ++ r->upstream->peer.data = bp; + + rc = lscf->balancer.handler(r, lscf, L); ++ ++ r->upstream->peer.data = pdata; ++ + if (rc == NGX_ERROR) { + return NGX_ERROR; + } +@@ -945,7 +943,6 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + ngx_url_t url; + ngx_http_upstream_t *u; + ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -971,18 +968,6 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* we cannot read r->upstream->peer.data here directly because +- * it could be overridden by other modules like +- * ngx_http_upstream_keepalive_module. +- */ +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } +- + ngx_memzero(&url, sizeof(ngx_url_t)); + + url.url.data = ngx_palloc(r->pool, addr_len); +@@ -1006,6 +991,8 @@ ngx_http_lua_ffi_balancer_set_current_peer(ngx_http_request_t *r, + return NGX_ERROR; + } + ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; ++ + if (url.addrs && url.addrs[0].sockaddr) { + bp->sockaddr = url.addrs[0].sockaddr; + bp->socklen = url.addrs[0].socklen; +@@ -1029,7 +1016,6 @@ ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, + { + ngx_http_upstream_t *u; + ngx_http_lua_ctx_t *ctx; +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -1055,17 +1041,7 @@ ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- /* we cannot read r->upstream->peer.data here directly because +- * it could be overridden by other modules like +- * ngx_http_upstream_keepalive_module. +- */ +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + if (!ngx_http_lua_balancer_peer_set(bp)) { + *err = "no current peer set"; +@@ -1089,14 +1065,13 @@ ngx_http_lua_ffi_balancer_set_timeouts(ngx_http_request_t *r, + long connect_timeout, long send_timeout, long read_timeout, + char **err) + { +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; + + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) + ngx_http_upstream_conf_t *ucf; +-#endif +- ngx_http_lua_main_conf_t *lmcf; + ngx_http_lua_balancer_peer_data_t *bp; ++#endif + + if (r == NULL) { + *err = "no request found"; +@@ -1121,15 +1096,9 @@ ngx_http_lua_ffi_balancer_set_timeouts(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } +- + #if !(HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS) ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; ++ + if (!bp->cloned_upstream_conf) { + /* we clone the upstream conf for the current request so that + * we do not affect other requests at all. */ +@@ -1184,12 +1153,10 @@ ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + int count, char **err) + { + #if (nginx_version >= 1007005) +- ngx_uint_t max_tries, total; ++ ngx_uint_t max_tries, total; + #endif +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- +- ngx_http_lua_main_conf_t *lmcf; ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; + ngx_http_lua_balancer_peer_data_t *bp; + + if (r == NULL) { +@@ -1215,13 +1182,7 @@ ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + #if (nginx_version >= 1007005) + max_tries = r->upstream->conf->next_upstream_tries; +@@ -1247,12 +1208,10 @@ int + ngx_http_lua_ffi_balancer_get_last_failure(ngx_http_request_t *r, + int *status, char **err) + { +- ngx_http_lua_ctx_t *ctx; +- ngx_http_upstream_t *u; +- ngx_http_upstream_state_t *state; +- ++ ngx_http_lua_ctx_t *ctx; ++ ngx_http_upstream_t *u; ++ ngx_http_upstream_state_t *state; + ngx_http_lua_balancer_peer_data_t *bp; +- ngx_http_lua_main_conf_t *lmcf; + + if (r == NULL) { + *err = "no request found"; +@@ -1277,13 +1236,7 @@ ngx_http_lua_ffi_balancer_get_last_failure(ngx_http_request_t *r, + return NGX_ERROR; + } + +- lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module); +- +- bp = lmcf->balancer_peer_data; +- if (bp == NULL) { +- *err = "no upstream peer data found"; +- return NGX_ERROR; +- } ++ bp = (ngx_http_lua_balancer_peer_data_t *) u->peer.data; + + if (r->upstream_states && r->upstream_states->nelts > 1) { + state = r->upstream_states->elts; +diff --git a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h +index 9ce6836a..9a4342df 100644 +--- a/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h ++++ b/bundle/ngx_lua-0.10.21/src/ngx_http_lua_common.h +@@ -240,13 +240,6 @@ struct ngx_http_lua_main_conf_s { + ngx_http_lua_main_conf_handler_pt exit_worker_handler; + ngx_str_t exit_worker_src; + +- ngx_http_lua_balancer_peer_data_t *balancer_peer_data; +- /* neither yielding nor recursion is possible in +- * balancer_by_lua*, so there cannot be any races among +- * concurrent requests and it is safe to store the peer +- * data pointer in the main conf. +- */ +- + ngx_chain_t *body_filter_chain; + /* neither yielding nor recursion is possible in + * body_filter_by_lua*, so there cannot be any races among +-- +2.26.2 diff --git a/build/openresty/patches/ngx_stream_lua-0.0.10_01-expose_request_struct.patch b/build/openresty/patches/ngx_stream_lua-0.0.10_01-expose_request_struct.patch new file mode 100644 index 000000000000..29b9d8ab2a72 --- /dev/null +++ b/build/openresty/patches/ngx_stream_lua-0.0.10_01-expose_request_struct.patch @@ -0,0 +1,26 @@ +From 0acb7f5ad0fbc9ee037f0c5d689f98861fe9e49b Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Tue, 10 Dec 2019 11:51:53 -0800 +Subject: [PATCH] Sync with meta-lua-nginx-module + 1330009671cd86eaf045f9f2c5cda3727a94570f. + +--- + ngx_stream_lua-0.0.10/src/api/ngx_stream_lua_api.h | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/ngx_stream_lua-0.0.10/src/api/ngx_stream_lua_api.h b/ngx_stream_lua-0.0.10/src/api/ngx_stream_lua_api.h +index 0e5a18f..040ef84 100644 +--- a/bundle/ngx_stream_lua-0.0.10/src/api/ngx_stream_lua_api.h ++++ b/bundle/ngx_stream_lua-0.0.10/src/api/ngx_stream_lua_api.h +@@ -21,6 +21,9 @@ + + + ++#include ++#include "../ngx_stream_lua_request.h" ++ + + #include + #include +-- +2.20.1 diff --git a/build/openresty/patches/ngx_stream_lua-0.0.10_02-ngx-worker-pids.patch b/build/openresty/patches/ngx_stream_lua-0.0.10_02-ngx-worker-pids.patch new file mode 100644 index 000000000000..96fd7ff50331 --- /dev/null +++ b/build/openresty/patches/ngx_stream_lua-0.0.10_02-ngx-worker-pids.patch @@ -0,0 +1,49 @@ +From 9ce0848cff7c3c5eb0a7d5adfe2de22ea98e1e63 Mon Sep 17 00:00:00 2001 +From: attenuation +Date: Sun, 21 Aug 2022 22:12:22 +0800 +Subject: [PATCH] feat: add ngx_stream_lua_ffi_worker_pids to get all workers + pid map + +--- + +diff --git a/bundle/ngx_stream_lua-0.0.10/src/ngx_stream_lua_worker.c b/bundle/ngx_stream_lua-0.0.10/src/ngx_stream_lua_worker.c +index e4a9c915..4fb23314 100644 +--- a/bundle/ngx_stream_lua-0.0.10/src/ngx_stream_lua_worker.c ++++ b/bundle/ngx_stream_lua-0.0.10/src/ngx_stream_lua_worker.c +@@ -28,6 +28,36 @@ ngx_stream_lua_ffi_worker_pid(void) + } + + ++int ++ngx_stream_lua_ffi_worker_pids(int *pids, size_t *pids_len) ++{ ++ ngx_int_t i, n; ++ ++ n = 0; ++ for (i = 0; i < NGX_MAX_PROCESSES; i++) { ++ if (i != ngx_process_slot && ngx_processes[i].pid == 0) { ++ break; ++ } ++ ++ if (i == ngx_process_slot && ngx_processes[i].pid == 0) { ++ pids[n++] = ngx_pid; ++ } ++ ++ if (ngx_processes[i].pid > 0) { ++ pids[n++] = ngx_processes[i].pid; ++ } ++ } ++ ++ if (n == 0) { ++ return NGX_ERROR; ++ } ++ ++ *pids_len = n; ++ ++ return NGX_OK; ++} ++ ++ + int + ngx_stream_lua_ffi_worker_id(void) + { \ No newline at end of file diff --git a/build/openresty/patches/ngx_stream_lua-0.0.11_01-expose_request_struct.patch b/build/openresty/patches/ngx_stream_lua-0.0.11_01-expose_request_struct.patch new file mode 100644 index 000000000000..0c307318acc4 --- /dev/null +++ b/build/openresty/patches/ngx_stream_lua-0.0.11_01-expose_request_struct.patch @@ -0,0 +1,26 @@ +From 0acb7f5ad0fbc9ee037f0c5d689f98861fe9e49b Mon Sep 17 00:00:00 2001 +From: Datong Sun +Date: Tue, 10 Dec 2019 11:51:53 -0800 +Subject: [PATCH] Sync with meta-lua-nginx-module + 1330009671cd86eaf045f9f2c5cda3727a94570f. + +--- + ngx_stream_lua-0.0.11/src/api/ngx_stream_lua_api.h | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/bundle/ngx_stream_lua-0.0.11/src/api/ngx_stream_lua_api.h b/bundle/ngx_stream_lua-0.0.11/src/api/ngx_stream_lua_api.h +index 0e5a18f..040ef84 100644 +--- a/bundle/ngx_stream_lua-0.0.11/src/api/ngx_stream_lua_api.h ++++ b/bundle/ngx_stream_lua-0.0.11/src/api/ngx_stream_lua_api.h +@@ -21,6 +21,9 @@ + + + ++#include ++#include "../ngx_stream_lua_request.h" ++ + + #include + #include +-- +2.20.1 diff --git a/build/openresty/patches/openresty-custom_prefix_and_cc.patch b/build/openresty/patches/openresty-custom_prefix_and_cc.patch new file mode 100644 index 000000000000..f90925125df6 --- /dev/null +++ b/build/openresty/patches/openresty-custom_prefix_and_cc.patch @@ -0,0 +1,107 @@ +diff --git a/configure b/configure +index d461294..2e8d3e2 100755 +--- a/configure ++++ b/configure +@@ -128,7 +128,7 @@ my $ngx_sbin; + my %resty_opts; + my $dry_run; + my @ngx_rpaths; +-my $cc; ++my $cc = $ENV{CC}; + my $cores; + my $luajit_xcflags = ''; + my $user_luajit_xcflags; +@@ -356,6 +356,9 @@ for my $opt (@ARGV) { + push @ngx_opts, "--with-$lib-opt=-g $opt"; + $with_ext_lib_opts{$lib} = 1; + ++ } elsif ($opt =~ /^--with-install-prefix=(.*)/) { ++ $resty_opts{install_prefix} = $1; ++ + } elsif ($opt =~ /^--sbin-path=(.*)/) { + $ngx_sbin = $1; + push @ngx_opts, $opt; +@@ -696,7 +699,12 @@ _END_ + #unshift @ngx_ld_opts, "-L$lib"; + #unshift @ngx_cc_opts, "-I$inc"; + +- push @ngx_rpaths, "$luajit_prefix/lib"; ++ my $real_luajit_prefix = $luajit_prefix; ++ if ($opts->{install_prefix}) { ++ $real_luajit_prefix = "$opts->{install_prefix}/openresty/luajit"; ++ } ++ ++ push @ngx_rpaths, "$real_luajit_prefix/lib"; + + } elsif ($opts->{luajit}) { + my $luajit_src = auto_complete 'LuaJIT'; +@@ -862,7 +870,12 @@ _END_ + #unshift @ngx_cc_opts, "-I$inc"; + + if ($platform ne 'msys') { +- push @ngx_rpaths, File::Spec->catfile($luajit_prefix, "lib"); ++ my $real_luajit_prefix = $luajit_prefix; ++ if ($opts->{install_prefix}) { ++ $real_luajit_prefix = "$opts->{install_prefix}/openresty/luajit"; ++ } ++ ++ push @ngx_rpaths, File::Spec->catfile($real_luajit_prefix, "lib"); + } + + cd '..'; +@@ -871,8 +884,13 @@ _END_ + if ($opts->{luajit} || $opts->{luajit_path}) { + # build lua modules + +- $lualib_prefix = File::Spec->catfile($prefix, "lualib"); +- my $site_lualib_prefix = File::Spec->catfile($prefix, "site/lualib"); ++ my $openresty_prefix = $prefix; ++ if ($opts->{install_prefix}) { ++ $openresty_prefix = "$opts->{install_prefix}/openresty"; ++ } ++ ++ $lualib_prefix = File::Spec->catfile($openresty_prefix, "lualib"); ++ my $site_lualib_prefix = File::Spec->catfile($openresty_prefix, "site/lualib"); + + { + my $ngx_lua_dir = auto_complete 'ngx_lua'; +@@ -926,6 +944,11 @@ _EOC_ + close $in; + } + ++ # set it back ++ $lualib_prefix = File::Spec->catfile($prefix, "lualib"); ++ $site_lualib_prefix = File::Spec->catfile($prefix, "site/lualib"); ++ ++ + unless ($opts->{no_lua_cjson}) { + my $dir = auto_complete 'lua-cjson'; + if (!defined $dir) { +@@ -1173,10 +1196,16 @@ _EOC_ + open my $in, $resty_bin + or die "Cannot open $resty_bin for reading: $!\n"; + my ($new, $found); ++ ++ my $real_ngx_sbin = $ngx_sbin; ++ if ($opts->{install_prefix}) { ++ $real_ngx_sbin = "$opts->{install_prefix}/openresty/nginx/sbin/nginx"; ++ } ++ + while (<$in>) { + if (/^my \$nginx_path;$/) { + $found = 1; +- $new .= qq/my \$nginx_path = '$ngx_sbin';\n/; ++ $new .= qq/my \$nginx_path = '$real_ngx_sbin';\n/; + + } else { + $new .= $_; +@@ -1354,6 +1383,9 @@ _EOC_ + --with-libpq=DIR specify the libpq (or postgresql) installation prefix + --with-pg_config=PATH specify the path of the pg_config utility + ++ --with-install-prefix=DIR specify the install prefix on target that differs from ++ --prefix that injects hardcoded paths in compiled binary ++ + Options directly inherited from nginx + + --sbin-path=PATH set nginx binary pathname diff --git a/build/openresty/pcre/BUILD.bazel b/build/openresty/pcre/BUILD.bazel new file mode 100644 index 000000000000..78a14d48ef7e --- /dev/null +++ b/build/openresty/pcre/BUILD.bazel @@ -0,0 +1,16 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +exports_files( + [ + "BUILD.pcre.bazel", + ], + visibility = ["//visibility:public"], +) + +build_test( + name = "build", + targets = [ + "@pcre//:pcre", + ], + visibility = ["//:__pkg__"], +) diff --git a/build/openresty/pcre/BUILD.pcre.bazel b/build/openresty/pcre/BUILD.pcre.bazel new file mode 100644 index 000000000000..229005a870f3 --- /dev/null +++ b/build/openresty/pcre/BUILD.pcre.bazel @@ -0,0 +1,36 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# pcre cmake detects cross compile automatically +cmake( + name = "pcre", + build_args = [ + "--", # <- Pass remaining options to the native tool. + "-j" + KONG_VAR["NPROC"], + ], + cache_entries = { + "CMAKE_C_FLAGS": "${CMAKE_C_FLAGS:-} -fPIC", + "PCRE_BUILD_PCREGREP": "OFF", # we don't need the cli binary + "PCRE_BUILD_TESTS": "OFF", # test doesn't compile on aarch64-linux-gnu (cross) + "CMAKE_INSTALL_LIBDIR": "lib", # force distros that uses lib64 (rhel family) to use lib + }, + lib_source = ":all_srcs", + out_static_libs = ["libpcre.a"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "pcre_dir", + srcs = [ + ":pcre", + ], + output_group = "gen_dir", +) diff --git a/build/openresty/pcre/README.md b/build/openresty/pcre/README.md new file mode 100644 index 000000000000..d8923b5e28d3 --- /dev/null +++ b/build/openresty/pcre/README.md @@ -0,0 +1,5 @@ +This target is modified from https://github.com/bazelbuild/rules_foreign_cc/tree/main/examples/third_party +with following changes: + +- Read version from `requirements.txt` +- Updated `build_file` to new path under //build/openresty diff --git a/build/openresty/pcre/pcre_repositories.bzl b/build/openresty/pcre/pcre_repositories.bzl new file mode 100644 index 000000000000..54448927f566 --- /dev/null +++ b/build/openresty/pcre/pcre_repositories.bzl @@ -0,0 +1,20 @@ +"""A module defining the third party dependency PCRE""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def pcre_repositories(): + version = KONG_VAR["PCRE"] + + maybe( + http_archive, + name = "pcre", + build_file = "//build/openresty/pcre:BUILD.pcre.bazel", + strip_prefix = "pcre-" + version, + sha256 = "4e6ce03e0336e8b4a3d6c2b70b1c5e18590a5673a98186da90d4f33c23defc09", + urls = [ + "https://mirror.bazel.build/downloads.sourceforge.net/project/pcre/pcre/" + version + "/pcre-" + version + ".tar.gz", + "https://downloads.sourceforge.net/project/pcre/pcre/" + version + "/pcre-" + version + ".tar.gz", + ], + ) diff --git a/build/openresty/repositories.bzl b/build/openresty/repositories.bzl new file mode 100644 index 000000000000..5516fed29255 --- /dev/null +++ b/build/openresty/repositories.bzl @@ -0,0 +1,88 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@kong_bindings//:variables.bzl", "KONG_VAR") +load("//build/openresty/pcre:pcre_repositories.bzl", "pcre_repositories") +load("//build/openresty/openssl:openssl_repositories.bzl", "openssl_repositories") +load("//build/openresty/atc_router:atc_router_repositories.bzl", "atc_router_repositories") + +# This is a dummy file to export the module's repository. +_NGINX_MODULE_DUMMY_FILE = """ +filegroup( + name = "all_srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) +""" + +def openresty_repositories(): + pcre_repositories() + openssl_repositories() + atc_router_repositories() + + openresty_version = KONG_VAR["OPENRESTY"] + + maybe( + openresty_http_archive_wrapper, + name = "openresty", + build_file = "//build/openresty:BUILD.openresty.bazel", + sha256 = "0c5093b64f7821e85065c99e5d4e6cc31820cfd7f37b9a0dec84209d87a2af99", + strip_prefix = "openresty-" + openresty_version, + urls = [ + "https://openresty.org/download/openresty-" + openresty_version + ".tar.gz", + "https://github.com/Kong/openresty-release-mirror/releases/download/" + openresty_version + "/openresty-" + openresty_version + ".tar.gz", + ], + patches = KONG_VAR["OPENRESTY_PATCHES"], + patch_args = ["-p1"], + ) + + maybe( + new_git_repository, + name = "lua-kong-nginx-module", + branch = KONG_VAR["LUA_KONG_NGINX_MODULE"], + remote = "https://github.com/Kong/lua-kong-nginx-module", + build_file_content = _NGINX_MODULE_DUMMY_FILE, + recursive_init_submodules = True, + ) + + maybe( + new_git_repository, + name = "lua-resty-lmdb", + branch = KONG_VAR["LUA_RESTY_LMDB"], + remote = "https://github.com/Kong/lua-resty-lmdb", + build_file_content = _NGINX_MODULE_DUMMY_FILE, + recursive_init_submodules = True, + patches = ["//build/openresty:lua-resty-lmdb-cross.patch"], + patch_args = ["-p1", "-l"], # -l: ignore whitespace + ) + + maybe( + new_git_repository, + name = "lua-resty-events", + branch = KONG_VAR["LUA_RESTY_EVENTS"], + remote = "https://github.com/Kong/lua-resty-events", + build_file_content = _NGINX_MODULE_DUMMY_FILE, + recursive_init_submodules = True, + ) + +def _openresty_binding_impl(ctx): + ctx.file("BUILD.bazel", _NGINX_MODULE_DUMMY_FILE) + ctx.file("WORKSPACE", "workspace(name = \"openresty_patch\")") + + version = "LuaJIT\\\\ 2.1.0-" + for path in ctx.path("../openresty/bundle").readdir(): + if path.basename.startswith("LuaJIT-2.1-"): + version = version + path.basename.replace("LuaJIT-2.1-", "") + break + + ctx.file("variables.bzl", 'LUAJIT_VERSION = "%s"' % version) + +openresty_binding = repository_rule( + implementation = _openresty_binding_impl, +) + +def openresty_http_archive_wrapper(name, **kwargs): + http_archive(name = name, **kwargs) + openresty_binding(name = name + "_binding") diff --git a/build/package/kong-enterprise-edition.service b/build/package/kong-enterprise-edition.service new file mode 100644 index 000000000000..60bbd97089d8 --- /dev/null +++ b/build/package/kong-enterprise-edition.service @@ -0,0 +1,25 @@ +[Unit] +Description=Kong Enterprise Edition +Documentation=https://docs.konghq.com/enterprise/ +After=syslog.target network.target remote-fs.target nss-lookup.target + +[Service] +ExecStartPre=/usr/local/bin/kong prepare -p /usr/local/kong +ExecStart=/usr/local/openresty/nginx/sbin/nginx -p /usr/local/kong -c nginx.conf +ExecReload=/usr/local/bin/kong prepare -p /usr/local/kong +ExecReload=/usr/local/openresty/nginx/sbin/nginx -p /usr/local/kong -c nginx.conf -s reload +ExecStop=/bin/kill -s QUIT $MAINPID +PrivateTmp=true + +# All environment variables prefixed with `KONG_` and capitalized will override +# the settings specified in the `/etc/kong/kong.conf.default` file. +# +# For example: +# `log_level = debug` in the .conf file -> `KONG_LOG_LEVEL=debug` env var. +Environment=KONG_NGINX_DAEMON=off + +# You can control this limit through /etc/security/limits.conf +LimitNOFILE=infinity + +[Install] +WantedBy=multi-user.target diff --git a/build/package/kong.logrotate b/build/package/kong.logrotate new file mode 100644 index 000000000000..e319135ff5b1 --- /dev/null +++ b/build/package/kong.logrotate @@ -0,0 +1,15 @@ +/usr/local/kong/logs/*.log { + su kong kong + rotate 14 + daily + missingok + compress + delaycompress + notifempty + sharedscripts + postrotate + if [ -f /usr/local/kong/pids/nginx.pid ]; then + kill -USR1 `cat /usr/local/kong/pids/nginx.pid` + fi + endscript +} diff --git a/build/package/kong.service b/build/package/kong.service new file mode 100644 index 000000000000..eeaa6502af97 --- /dev/null +++ b/build/package/kong.service @@ -0,0 +1,25 @@ +[Unit] +Description=Kong +Documentation=https://docs.konghq.com/ +After=syslog.target network.target remote-fs.target nss-lookup.target + +[Service] +ExecStartPre=/usr/local/bin/kong prepare -p /usr/local/kong +ExecStart=/usr/local/openresty/nginx/sbin/nginx -p /usr/local/kong -c nginx.conf +ExecReload=/usr/local/bin/kong prepare -p /usr/local/kong +ExecReload=/usr/local/openresty/nginx/sbin/nginx -p /usr/local/kong -c nginx.conf -s reload +ExecStop=/bin/kill -s QUIT $MAINPID +PrivateTmp=true + +# All environment variables prefixed with `KONG_` and capitalized will override +# the settings specified in the `/etc/kong/kong.conf.default` file. +# +# For example: +# `log_level = debug` in the .conf file -> `KONG_LOG_LEVEL=debug` env var. +Environment=KONG_NGINX_DAEMON=off + +# You can control this limit through /etc/security/limits.conf +LimitNOFILE=infinity + +[Install] +WantedBy=multi-user.target diff --git a/build/package/nfpm.yaml b/build/package/nfpm.yaml new file mode 100644 index 000000000000..58d5cad17aa1 --- /dev/null +++ b/build/package/nfpm.yaml @@ -0,0 +1,78 @@ +name: "${KONG_NAME}" +arch: ${ARCH} +platform: "linux" +version: "${KONG_VERSION}" +section: "default" +priority: "extra" +provides: +- kong +- luarocks +maintainer: "Kong Inc. " +description: | + Kong is a distributed gateway for APIs and Microservices, focused on high performance and reliability. +vendor: "Kong Inc." +license: "ASL 2.0" +contents: +- src: nfpm-prefix/bin + dst: /usr/local/bin +- src: nfpm-prefix/kong + dst: /usr/local/kong +- src: nfpm-prefix/lib + dst: /usr/local/lib +- src: nfpm-prefix/etc/luarocks + dst: /usr/local/etc/luarocks +- src: nfpm-prefix/openresty + dst: /usr/local/openresty +- src: nfpm-prefix/share + dst: /usr/local/share +- src: nfpm-prefix/etc/kong + dst: /etc/kong +- src: bin/kong + dst: /usr/local/bin/kong +- src: bin/kong-health + dst: /usr/local/bin/kong-health +- src: kong/include + dst: /usr/local/kong/include +- src: build/package/kong.service + dst: /lib/systemd/system/kong.service +- src: build/package/kong.logrotate + dst: /etc/kong/kong.logrotate +scripts: + postinstall: ./build/package/postinstall.sh +replaces: +- ${KONG_REPLACES_1} +- ${KONG_REPLACES_2} +conflicts: +- ${KONG_CONFLICTS_1} +- ${KONG_CONFLICTS_2} +overrides: + deb: + depends: + - ca-certificates + - libpcre3 + - perl + - zlib1g-dev + - libyaml-0-2 + rpm: + depends: + - ca-certificates + - pcre + - perl + - perl-Time-HiRes + - zlib + - zlib-devel + - libyaml + # Workaround for https://github.com/goreleaser/nfpm/issues/589 + - ${RPM_EXTRA_DEPS} + - ${RPM_EXTRA_DEPS_2} + - ${RPM_EXTRA_DEPS_3} + apk: + depends: + - ca-certificates + +rpm: + signature: + # PGP secret key (can also be ASCII-armored), the passphrase is taken + # from the environment variable $NFPM_RPM_PASSPHRASE with a fallback + # to $NFPM_PASSPHRASE. + key_file: ${RPM_SIGNING_KEY_FILE} diff --git a/build/package/postinstall.sh b/build/package/postinstall.sh new file mode 100644 index 000000000000..3a1bc9178b3f --- /dev/null +++ b/build/package/postinstall.sh @@ -0,0 +1,30 @@ +create_user() { + groupadd -f kong + useradd -g kong -s /bin/sh -c "Kong default user" kong + + FILES="" + FILES="${FILES} /etc/kong/" + FILES="${FILES} /usr/local/bin/json2lua" + FILES="${FILES} /usr/local/bin/kong" + FILES="${FILES} /usr/local/bin/lapis" + FILES="${FILES} /usr/local/bin/lua2json" + FILES="${FILES} /usr/local/bin/luarocks" + FILES="${FILES} /usr/local/bin/luarocks-admin" + FILES="${FILES} /usr/local/bin/openapi2kong" + FILES="${FILES} /usr/local/etc/luarocks/" + FILES="${FILES} /usr/local/etc/passwdqc/" + FILES="${FILES} /usr/local/kong/" + FILES="${FILES} /usr/local/lib/lua/" + FILES="${FILES} /usr/local/lib/luarocks/" + FILES="${FILES} /usr/local/openresty/" + FILES="${FILES} /usr/local/share/lua/" + + for FILE in ${FILES}; do + chown -R kong:kong ${FILE} + chmod -R g=u ${FILE} + done + + return 0 +} + +create_user > /dev/null 2>&1 diff --git a/build/platforms/distro/BUILD b/build/platforms/distro/BUILD new file mode 100644 index 000000000000..4816ca427a8d --- /dev/null +++ b/build/platforms/distro/BUILD @@ -0,0 +1,37 @@ +constraint_setting(name = "distro") + +constraint_value( + name = "generic", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "alpine", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "rhel9", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "rhel8", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "aws2023", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "aws2", + constraint_setting = ":distro", + visibility = ["//visibility:public"], +) diff --git a/build/repositories.bzl b/build/repositories.bzl new file mode 100644 index 000000000000..7e9478ae8501 --- /dev/null +++ b/build/repositories.bzl @@ -0,0 +1,141 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("//build/luarocks:luarocks_repositories.bzl", "luarocks_repositories") +load("//build/cross_deps:repositories.bzl", "cross_deps_repositories") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +_SRCS_BUILD_FILE_CONTENT = """ +filegroup( + name = "all_srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) +""" + +_DIST_BUILD_FILE_CONTENT = """ +filegroup( + name = "dist_files", + srcs = ["dist"], + visibility = ["//visibility:public"], +) +""" + +def kong_manager_repositories(): + """Defines the kong manager repository""" + + gh_matrix = [ + ["linux", "amd64", "6b3e56ee3253795d9c48e019cfd7b8dfc03b28073a411d1f527f5021764f63cb"], + ["linux", "arm64", "484bfa77456dab0d3a155755334e86dfd510cb628060274384c0a28eba22ed26"], + ["macOS", "amd64", "3187174428dfb73b712f50b550e6a148f3f0ad4b2dbdf352519b159652ed9f50"], + ] + for name, arch, sha in gh_matrix: + http_archive( + name = "gh_%s_%s" % (name, arch), + url = "https://github.com/cli/cli/releases/download/v2.21.2/gh_2.21.2_%s_%s.tar.gz" % (name, arch), + strip_prefix = "gh_2.21.2_%s_%s" % (name, arch), + sha256 = sha, + build_file_content = _SRCS_BUILD_FILE_CONTENT, + ) + +def _copyright_header(ctx): + paths = ctx.execute(["find", ctx.path("."), "-type", "f"]).stdout.split("\n") + + copyright_content = ctx.read(ctx.path(Label("@kong//:distribution/COPYRIGHT-HEADER"))).replace("--", " ") + copyright_content_js = "/*\n" + copyright_content + "*/\n\n" + copyright_content_html = "\n\n" + for path in paths: + if path.endswith(".js") or path.endswith(".map") or path.endswith(".css"): + content = ctx.read(path) + if not content.startswith(copyright_content_js): + ctx.file(path, copyright_content_js + content) + + elif path.endswith(".html"): + content = ctx.read(path) + if not content.startswith(copyright_content_html): + ctx.file(path, copyright_content_html + content) + +def _github_release_impl(ctx): + ctx.file("WORKSPACE", "workspace(name = \"%s\")\n" % ctx.name) + + if ctx.attr.build_file: + ctx.file("BUILD.bazel", ctx.read(ctx.attr.build_file)) + elif ctx.attr.build_file_content: + ctx.file("BUILD.bazel", ctx.attr.build_file_content) + + os_name = ctx.os.name + os_arch = ctx.os.arch + + if os_arch == "aarch64": + os_arch = "arm64" + elif os_arch == "x86_64": + os_arch = "amd64" + elif os_arch != "amd64": + fail("Unsupported arch %s" % os_arch) + + if os_name == "mac os x": + os_name = "macOS" + elif os_name != "linux": + fail("Unsupported OS %s" % os_name) + + if os_name == "macOS" and os_arch == "arm64": + # no binary release of mac m1, need to install from homebrew + # we will just rely on rosseta 2 + os_arch = "amd64" + + gh_bin = "%s" % ctx.path(Label("@gh_%s_%s//:bin/gh" % (os_name, os_arch))) + ret = ctx.execute([gh_bin, "release", "download", ctx.attr.tag, "-p", ctx.attr.pattern, "-R", ctx.attr.repo]) + + if ret.return_code != 0: + gh_token_set = "GITHUB_TOKEN is set, is it valid?" + if not ctx.os.environ.get("GITHUB_TOKEN", ""): + gh_token_set = "GITHUB_TOKEN is not set, is this a private repo?" + fail("Failed to download release (%s): %s, exit: %d" % (gh_token_set, ret.stderr, ret.return_code)) + + ctx.extract(ctx.attr.pattern) + + _copyright_header(ctx) + +github_release = repository_rule( + implementation = _github_release_impl, + attrs = { + "tag": attr.string(mandatory = True), + "pattern": attr.string(mandatory = True), + "repo": attr.string(mandatory = True), + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + }, +) + +def protoc_repositories(): + http_archive( + name = "protoc", + url = "https://github.com/protocolbuffers/protobuf/releases/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip", + sha256 = "2994b7256f7416b90ad831dbf76a27c0934386deb514587109f39141f2636f37", + build_file_content = """ +filegroup( + name = "all_srcs", + srcs = ["include"], + visibility = ["//visibility:public"], +)""", + ) + +def kong_resty_websocket_repositories(): + new_git_repository( + name = "lua-resty-websocket", + branch = KONG_VAR["RESTY_WEBSOCKET_VERSION"], + remote = "https://github.com/Kong/lua-resty-websocket", + build_file_content = _SRCS_BUILD_FILE_CONTENT, + ) + +def build_repositories(): + luarocks_repositories() + + kong_resty_websocket_repositories() + kong_manager_repositories() + + protoc_repositories() + + cross_deps_repositories() diff --git a/build/templates/venv-commons b/build/templates/venv-commons new file mode 100644 index 000000000000..4e0daa93699e --- /dev/null +++ b/build/templates/venv-commons @@ -0,0 +1,75 @@ +#!/bin/bash + +# template variables starts +workspace_path="{{workspace_path}}" +# template variables ends + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 KONG_VENV KONG_VENV_ENV_FILE" + exit 1 +fi + +KONG_VENV=$1 +KONG_VENV_ENV_FILE=$2 + +# clear the file +>| $KONG_VENV_ENV_FILE + +# use env vars to let Fish shell happy, we will unset them later +LUAROCKS_CONFIG="$KONG_VENV/rocks_config" +ROCKS_ROOT="$KONG_VENV" + +chmod -R a+rw "$KONG_VENV" + +mkdir -p "$KONG_VENV/venv/bin" + +echo "#!/bin/bash +$KONG_VENV/openresty/bin/resty -I $KONG_VENV/openresty/site/lualib -I $KONG_VENV/openresty/lualib --nginx $KONG_VENV/openresty/nginx/sbin/nginx \"\$@\" +" >| "$KONG_VENV/venv/bin/resty" +chmod +x "$KONG_VENV/venv/bin/resty" + +echo " +rocks_trees = { + { name = [[system]], root = [[$ROCKS_ROOT]] } +} +" >| "$LUAROCKS_CONFIG" + +# duplicate package.[c]path even though we have set in resty-cli, so luajit and kong can consume +LUA_PATH="\ +$ROCKS_ROOT/share/lua/5.1/?.lua;$ROCKS_ROOT/share/lua/5.1/?.ljbc;\ +$ROCKS_ROOT/share/lua/5.1/?/init.lua;$ROCKS_ROOT/share/lua/5.1/?/init.ljbc;\ +$KONG_VENV/openresty/site/lualib/?.lua;$KONG_VENV/openresty/site/lualib/?.ljbc;\ +$KONG_VENV/openresty/site/lualib/?/init.lua;$KONG_VENV/openresty/site/lualib/?/init.ljbc;\ +$KONG_VENV/openresty/lualib/?.lua;$KONG_VENV/openresty/lualib/?.ljbc;\ +$KONG_VENV/openresty/lualib/?/init.lua;$KONG_VENV/openresty/lualib/?/init.ljbc;\ +$KONG_VENV/openresty/luajit/share/luajit-2.1.0-beta3/?.lua" + +# XXX EE +if [ -d "./plugins-ee" ] ; then + links_dir="${workspace_path}/bazel-bin/build/ee/plugins-ee" + rm -rf "${links_dir}/kong/plugins" + mkdir -p "${links_dir}/kong/plugins" + for plugin in $(ls plugins-ee); do + ln -s "$(realpath "./plugins-ee/${plugin}/kong/plugins/${plugin}")" "${links_dir}/kong/plugins/${plugin}" + done + LUA_PATH="${links_dir}/?.lua;${links_dir}/init/?.lua;$LUA_PATH" +fi +# support custom plugin development +if [ -n $KONG_PLUGIN_PATH ] ; then + LUA_PATH="$KONG_PLUGIN_PATH/?.lua;$KONG_PLUGIN_PATH/?/init.lua;$LUA_PATH" +fi +# default; duplicate of 'lua_package_path' in kong.conf and nginx_kong.lua +LUA_PATH="./?.lua;./?/init.lua;$LUA_PATH;;" + +# write envs to env file +cat >> $KONG_VENV_ENV_FILE </dev/null && stop_services + + unset -f deactivate + unset -f start_services +} + +start_services () { + . $workspace_path/scripts/dependency_services/up.sh + # stop_services is defined by the script above +} + +# actually set env vars +KONG_VENV_ENV_FILE=$(mktemp) +export KONG_VENV_ENV_FILE +bash ${KONG_VENV}-venv/lib/venv-commons $KONG_VENV $KONG_VENV_ENV_FILE +. $KONG_VENV_ENV_FILE + +# set shell prompt +if [ -z "${KONG_VENV_DISABLE_PROMPT-}" ] ; then + if [ -n "${_OLD_KONG_VENV_PS1}" ]; then + # prepend the old PS1 if this script is called multiple times + PS1="(${build_name}) ${_OLD_KONG_VENV_PS1}" + else + _OLD_KONG_VENV_PS1="${PS1-}" + PS1="(${build_name}) ${PS1-}" + fi + export PS1 +fi + +# check wrapper +test -n "$*" && exec "$@" || true diff --git a/build/tests/01-base.sh b/build/tests/01-base.sh new file mode 100755 index 000000000000..324fb11ff869 --- /dev/null +++ b/build/tests/01-base.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +if [ -n "${VERBOSE:-}" ]; then + set -x +fi + +source .requirements +source build/tests/util.sh + +### +# +# user/group +# +### + +# a missing kong user can indicate that the post-install script on rpm/deb +# platforms failed to run properly +msg_test '"kong" user exists' +assert_exec 0 'root' 'getent passwd kong' + +msg_test '"kong" group exists' +assert_exec 0 'root' 'getent group kong' + +### +# +# files and ownership +# +### + +msg_test "/usr/local/kong exists and is owned by kong:root" +assert_exec 0 'kong' "test -O /usr/local/kong || ( rc=\$?; stat '${path}'; exit \$rc )" +assert_exec 0 'root' "test -G /usr/local/kong || ( rc=\$?; stat '${path}'; exit \$rc )" + +msg_test "/usr/local/bin/kong exists and is owned by kong:root" +assert_exec 0 'kong' "test -O /usr/local/kong || ( rc=\$?; stat '${path}'; exit \$rc )" +assert_exec 0 'root' "test -G /usr/local/kong || ( rc=\$?; stat '${path}'; exit \$rc )" + +if alpine; then + # we have never produced real .apk package files for alpine and thus have + # never depended on the kong user/group chown that happens in the + # postinstall script(s) for other package types + # + # if we ever do the work to support real .apk files (with read postinstall + # scripts), we will need to this test + msg_yellow 'skipping file and ownership tests on alpine' +else + for path in \ + /usr/local/bin/luarocks \ + /usr/local/etc/luarocks/ \ + /usr/local/lib/{lua,luarocks}/ \ + /usr/local/openresty/ \ + /usr/local/share/lua/; do + msg_test "${path} exists and is owned by kong:kong" + assert_exec 0 'kong' "test -O ${path} || ( rc=\$?; stat '${path}'; exit \$rc )" + assert_exec 0 'kong' "test -G ${path} || ( rc=\$?; stat '${path}'; exit \$rc )" + done +fi + +msg_test 'default conf file exists and is not empty' +assert_exec 0 'root' "test -s /etc/kong/kong.conf.default" + +msg_test 'default logrotate file exists and is not empty' +assert_exec 0 'root' "test -s /etc/kong/kong.logrotate" + +msg_test 'plugin proto file exists and is not empty' +assert_exec 0 'root' "test -s /usr/local/kong/include/kong/pluginsocket.proto" + +msg_test 'protobuf files exist and are not empty' +assert_exec 0 'root' "for f in /usr/local/kong/include/google/protobuf/*.proto; do test -s \$f; done" + +msg_test 'ssl header files exist and are not empty' +assert_exec 0 'root' "for f in /usr/local/kong/include/openssl/*.h; do test -s \$f; done" + +### +# +# OpenResty binaries/tools +# +### + +msg_test 'openresty binary is expected version' +assert_exec 0 'root' "/usr/local/openresty/bin/openresty -v 2>&1 | grep '${RESTY_VERSION}'" + +# linking to a non-kong-provided luajit library can indicate the package was +# created on a dev workstation where luajit/openresty was installed manually +# and probably shouldn't be shipped to customers +msg_test 'openresty binary is linked to kong-provided luajit library' +assert_exec 0 'root' "ldd /usr/local/openresty/bin/openresty | grep -E 'libluajit-.*openresty/luajit/lib'" + +# if libpcre appears in the ldd output for the openresty binary, static linking +# of it during the compile of openresty may have failed +msg_test 'openresty binary is NOT linked to external PCRE' +assert_exec 0 'root' "ldd /usr/local/openresty/bin/openresty | grep -ov 'libpcre.so'" + +msg_test 'openresty binary compiled with LuaJIT PCRE support' +assert_exec 0 'root' "/usr/local/openresty/bin/openresty -V 2>&1 | grep '\-\-with-pcre-jit'" + +msg_test 'resty CLI can be run by kong user' +assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'print(jit.version)'" + +msg_test 'resty CLI functions and returns valid version of LuaJIT' +assert_exec 0 'root' "/usr/local/openresty/bin/resty -e 'print(jit.version)' | grep -E 'LuaJIT\ ([0-9]\.*){3}\-20[0-9]+'" + +### +# +# SSL verification +# +### + +# check which ssl openresty is using +if docker_exec root '/usr/local/openresty/bin/openresty -V 2>&1' | grep 'BoringSSL'; then + msg_test 'openresty binary uses expected boringssl version' + assert_exec 0 'root' "/usr/local/openresty/bin/openresty -V 2>&1 | grep '${RESTY_BORINGSSL_VERSION}'" +else + msg_test 'openresty binary uses expected openssl version' + assert_exec 0 'root' "/usr/local/openresty/bin/openresty -V 2>&1 | grep '${RESTY_OPENSSL_VERSION}'" +fi + +msg_test 'openresty binary is linked to kong-provided ssl libraries' +assert_exec 0 'root' "ldd /usr/local/openresty/bin/openresty | grep -E 'libssl.so.*kong/lib'" +assert_exec 0 'root' "ldd /usr/local/openresty/bin/openresty | grep -E 'libcrypto.so.*kong/lib'" + +### +# +# LuaRocks +# +### + +msg_test 'lua-resty-websocket lua files exist and contain a version' +assert_exec 0 'root' 'grep _VERSION /usr/local/openresty/lualib/resty/websocket/*.lua' diff --git a/build/tests/02-admin-api.sh b/build/tests/02-admin-api.sh new file mode 100755 index 000000000000..89d80df7cf3c --- /dev/null +++ b/build/tests/02-admin-api.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +if [ -n "${VERBOSE:-}" ]; then + set -x +fi + +source .requirements +source build/tests/util.sh + +service_name="$(random_string)" +route_name="$(random_string)" + +kong_ready + +msg_test "Check admin API is alive" +assert_response "${KONG_ADMIN_URI}" "200" + +msg_test "Create a service" +assert_response "-d name=${service_name} -d url=http://127.0.0.1:8001 ${KONG_ADMIN_URI}/services" "201" + +msg_test "List services" +assert_response "${KONG_ADMIN_URI}/services" "200" + +msg_test "Create a route" +assert_response "-d name=${route_name} -d paths=/anything ${KONG_ADMIN_URI}/services/${service_name}/routes" "201" + +msg_test "List routes" +assert_response "${KONG_ADMIN_URI}/services/${service_name}/routes" "200" + +msg_test "List services" +assert_response "${KONG_ADMIN_URI}/services" "200" + +msg_test "Proxy a request" +assert_response "${KONG_PROXY_URI}/anything" "200" + +if [[ "$EDITION" == "enterprise" ]]; then + it_runs_free_enterprise +fi diff --git a/build/tests/03-enterprise.sh b/build/tests/03-enterprise.sh new file mode 100755 index 000000000000..5979c5d588d7 --- /dev/null +++ b/build/tests/03-enterprise.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +if [ -n "${VERBOSE:-}" ]; then + set -x +fi + +source .requirements +source build/tests/util.sh + +if [[ "$EDITION" != "enterprise" ]]; then + exit 0 +fi + +KONG_LICENSE_URL="https://download.konghq.com/internal/kong-gateway/license.json" +KONG_LICENSE_DATA="$( + curl \ + --silent \ + --location \ + --retry 3 \ + --retry-delay 3 \ + --user "$PULP_USERNAME:$PULP_PASSWORD" \ + --url "$KONG_LICENSE_URL" +)" +export KONG_LICENSE_DATA + +if [[ ! $KONG_LICENSE_DATA == *"signature"* || ! $KONG_LICENSE_DATA == *"payload"* ]]; then + # the check above is a bit lame, but the best we can do without requiring + # yet more additional dependenies like jq or similar. + msg_yellow "failed to download the Kong Enterprise license file! + $KONG_LICENSE_DATA" +fi + +F=$(mktemp) +echo "$KONG_LICENSE_DATA" >$F + +assert_response "-F payload=@$F $KONG_ADMIN_URI/licenses" "201 409" + +rm $F + +sleep 1 + +it_runs_full_enterprise + +### +# +# Enterprise-only +# +### + +# 2.8 does not have expat +for library in \ + jq \ + passwdqc \ + license_utils; do + msg_test "${library} library exists, is not empty, and is readable by kong user" + assert_exec 0 'kong' "test -s /usr/local/kong/lib/lib${library}.so" + + msg_test "resty CLI can ffi load ${library} library" + assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'require(\"ffi\").load \"${library}\"'" +done + +msg_test "nettle libraries exists, are not empty, and are readable by kong user" +assert_exec 0 'kong' "test -s /usr/local/kong/lib/libnettle.so" +assert_exec 0 'kong' "test -s /usr/local/kong/lib/libhogweed.so" + +msg_test "resty CLI can ffi load nettle libraries" +assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'require(\"ffi\").load \"nettle\"'" +assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'require(\"ffi\").load \"hogweed\"'" + +# 2.8 does not have xml2/xslt +# msg_test "xml libraries exists, are not empty, and are readable by kong user" +# assert_exec 0 'kong' "test -s /usr/local/kong/lib/libxml2.so" +# assert_exec 0 'kong' "test -s /usr/local/kong/lib/libxslt.so" + +# msg_test "resty CLI can ffi load xml libraries" +# assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'require(\"ffi\").load \"xml2\"'" +# assert_exec 0 'kong' "/usr/local/openresty/bin/resty -e 'require(\"ffi\").load \"xslt\"'" diff --git a/build/tests/03-http2-admin-api.sh b/build/tests/03-http2-admin-api.sh new file mode 100755 index 000000000000..c60d63fa3334 --- /dev/null +++ b/build/tests/03-http2-admin-api.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +if [ -n "${VERBOSE:-}" ]; then + set -x +fi + +source .requirements +source build/tests/util.sh + +kong_ready + +msg_test "Check if cURL supports HTTP/2" +if ! curl --version | grep -i "http2" > /dev/null; then + err_exit " local cURL does not support HTTP/2" +fi + +msg_test "Check HTTP/2 Admin API response is valid" +admin_api_http2_validity diff --git a/build/tests/util.sh b/build/tests/util.sh new file mode 100755 index 000000000000..b05cf4562f31 --- /dev/null +++ b/build/tests/util.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +KONG_ADMIN_URI=${KONG_ADMIN_URI:-"http://localhost:8001"} +KONG_PROXY_URI=${KONG_PROXY_URI:-"http://localhost:8000"} + +set_x_flag='' +if [ -n "${VERBOSE:-}" ]; then + set -x + set_x_flag='-x' +fi + +msg_test() { + builtin echo -en "\033[1;34m" >&1 + echo -n "===> " + builtin echo -en "\033[1;36m" >&1 + echo -e "$@" + builtin echo -en "\033[0m" >&1 +} + +msg_red() { + builtin echo -en "\033[1;31m" >&2 + echo -e "$@" + builtin echo -en "\033[0m" >&2 +} + +msg_yellow() { + builtin echo -en "\033[1;33m" >&1 + echo -e "$@" + builtin echo -en "\033[0m" >&1 +} + +err_exit() { + msg_red "$@" + exit 1 +} + +random_string() { + echo "a$(shuf -er -n19 {A..Z} {a..z} {0..9} | tr -d '\n')" +} + +kong_ready() { + local TIMEOUT_SECONDS=$((15)) + while [[ "$(curl -s -o /dev/null -w "%{http_code}" localhost:8000)" != 404 ]]; do + sleep 5; + COUNTER=$((COUNTER + 5)) + + if (( COUNTER >= TIMEOUT_SECONDS )) + then + printf '\xe2\x98\x93 ERROR: Timed out waiting for %s' "$KONG" + exit 1 + fi + done +} + +docker_exec() { + local user="${1:-kong}" + + shift + + test -t 1 && USE_TTY='-t' + + # shellcheck disable=SC2086 + docker exec --user="$user" ${USE_TTY} kong sh ${set_x_flag} -c "$@" +} + +_os() { + local os="$1" + + if docker_exec 'root' 'uname -a' | grep -qsi "$os"; then + return + else + docker_exec 'root' "grep -qsi '${os}' /etc/os-release" + return $? + fi +} + +alpine() { + _os 'alpine' +} + +assert_response() { + local endpoint=$1 + local expected_codes=$2 + local resp_code + COUNTER=20 + while : ; do + for code in ${expected_codes}; do + # shellcheck disable=SC2086 + resp_code=$(curl -s -o /dev/null -w "%{http_code}" ${endpoint}) + [ "$resp_code" == "$code" ] && break 2 + done + ((COUNTER-=1)) + [ "$COUNTER" -lt 1 ] && break + sleep 0.5 # 10 seconds max + done + for code in ${expected_codes}; do + [ "$resp_code" == "$code" ] && return + done || err_exit " expected $2, got $resp_code" +} + +assert_exec() { + local expected_code="${1:-0}" + local user="${2:-kong}" + + shift 2 + + ( + docker_exec "$user" "$@" + echo "$?" > /tmp/rc + ) | while read -r line; do printf ' %s\n' "$line"; done + + rc="$(cat /tmp/rc)" + + if ! [ "$rc" == "$expected_code" ]; then + err_exit " expected ${expected_code}, got ${rc}" + fi +} + +it_runs_free_enterprise() { + info=$(curl "$KONG_ADMIN_URI") + msg_test "it does not have ee-only plugins" + [ "$(echo "$info" | jq -r .plugins.available_on_server.canary)" != "true" ] + msg_test "it does not enable vitals" + [ "$(echo "$info" | jq -r .configuration.vitals)" == "false" ] + msg_test "workspaces are not writable" + assert_response "$KONG_ADMIN_URI/workspaces -d name=$(random_string)" "403" +} + +it_runs_full_enterprise() { + info=$(curl "$KONG_ADMIN_URI") + msg_test "it does have ee-only plugins" + [ "$(echo "$info" | jq -r .plugins.available_on_server | jq -r 'has("canary")')" == "true" ] + msg_test "it does enable vitals" + [ "$(echo "$info" | jq -r .configuration.vitals)" == "true" ] + msg_test "workspaces are writable" + assert_response "$KONG_ADMIN_URI/workspaces -d name=$(random_string)" "201" +} diff --git a/build/toolchain/.gitignore b/build/toolchain/.gitignore new file mode 100644 index 000000000000..3d057f524bf0 --- /dev/null +++ b/build/toolchain/.gitignore @@ -0,0 +1 @@ +wrappers-* diff --git a/build/toolchain/BUILD b/build/toolchain/BUILD new file mode 100644 index 000000000000..d684a8f05e75 --- /dev/null +++ b/build/toolchain/BUILD @@ -0,0 +1,98 @@ +load(":cc_toolchain_config.bzl", "cc_toolchain_config") + +package(default_visibility = ["//visibility:public"]) + +filegroup(name = "empty") + +################### +# aarch64-linux-gnu (installed with system) + +toolchain( + name = "gcc_cross_arm64_toolchain", + exec_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:arm64", + ], + toolchain = ":gcc_cross_arm64_cc_toolchain", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +cc_toolchain_config( + name = "gcc_cross_arm64_cc_toolchain_config", + compiler_configuration = {}, + target_cpu = "aarch64", + toolchain_path_prefix = "/usr/aarch64-linux-gnu/", # is this required? + tools_path_prefix = "/usr/bin/aarch64-linux-gnu-", +) + +cc_toolchain( + name = "gcc_cross_arm64_cc_toolchain", + all_files = ":empty", + compiler_files = ":empty", + dwp_files = ":empty", + linker_files = ":empty", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 0, + toolchain_config = ":gcc_cross_arm64_cc_toolchain_config", + toolchain_identifier = "gcc_cross_arm64-toolchain", +) + +################### +# x86_64-linux-musl (downloaded by bazel) + +toolchain( + name = "gcc_cross_musl_x86_64_toolchain", + exec_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "//:musl", + ], + toolchain = ":gcc_cross_musl_x86_64_cc_toolchain", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +cc_toolchain_config( + name = "gcc_cross_musl_x86_64_cc_toolchain_config", + ld = "gcc", + target_cpu = "x86_64", + target_libc = "musl", + toolchain_path_prefix = "gcc-11-x86_64-linux-musl-wrappers/", # is this required? + tools_path_prefix = "gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-", +) + +filegroup( + name = "wrappers", + srcs = glob([ + "gcc-11-x86_64-linux-musl-wrappers/**", + ]), +) + +filegroup( + name = "musl_toolchain_files", + srcs = [ + ":wrappers", + "@gcc-11-x86_64-linux-musl-cross//:toolchain", + ], +) + +cc_toolchain( + name = "gcc_cross_musl_x86_64_cc_toolchain", + all_files = ":musl_toolchain_files", + compiler_files = ":musl_toolchain_files", + dwp_files = ":empty", + linker_files = ":musl_toolchain_files", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 0, + toolchain_config = ":gcc_cross_musl_x86_64_cc_toolchain_config", + toolchain_identifier = "gcc_cross_musl_x86_64-toolchain", +) diff --git a/build/toolchain/cc_toolchain_config.bzl b/build/toolchain/cc_toolchain_config.bzl new file mode 100644 index 000000000000..a01dabb09db2 --- /dev/null +++ b/build/toolchain/cc_toolchain_config.bzl @@ -0,0 +1,213 @@ +# Copyright 2021 The Bazel Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", "tool_path") +load("@kong_bindings//:variables.bzl", "KONG_VAR") + +# Bazel 4.* doesn't support nested starlark functions, so we cannot simplify +#_fmt_flags() by defining it as a nested function. +def _fmt_flags(flags, toolchain_path_prefix): + return [f.format(toolchain_path_prefix = toolchain_path_prefix) for f in flags] + +# Macro for calling cc_toolchain_config from @bazel_tools with setting the +# right paths and flags for the tools. +def _impl(ctx): + target_cpu = ctx.attr.target_cpu + toolchain_path_prefix = ctx.attr.toolchain_path_prefix + tools_path_prefix = ctx.attr.tools_path_prefix + compiler_configuration = ctx.attr.compiler_configuration + + # Unfiltered compiler flags; these are placed at the end of the command + # line, so take precendence over any user supplied flags through --copts or + # such. + unfiltered_compile_flags = [ + # Do not resolve our symlinked resource prefixes to real paths. + "-no-canonical-prefixes", + # Reproducibility + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\"", + "-fdebug-prefix-map={}=__bazel_toolchain__/".format(toolchain_path_prefix), + ] + + # Default compiler flags: + compile_flags = [ + # "--target=" + target_system_name, + # Security + "-U_FORTIFY_SOURCE", # https://github.com/google/sanitizers/issues/247 + "-fstack-protector", + "-fno-omit-frame-pointer", + # Diagnostics + "-fcolor-diagnostics", + "-Wall", + "-Wthread-safety", + "-Wself-assign", + ] + + dbg_compile_flags = ["-g", "-fstandalone-debug"] + + opt_compile_flags = [ + "-g0", + "-O2", + "-D_FORTIFY_SOURCE=1", + "-DNDEBUG", + "-ffunction-sections", + "-fdata-sections", + ] + + link_flags = [ + # "--target=" + target_system_name, + "-lm", + "-no-canonical-prefixes", + ] + + # Similar to link_flags, but placed later in the command line such that + # unused symbols are not stripped. + link_libs = [] + + # Note that for xcompiling from darwin to linux, the native ld64 is + # not an option because it is not a cross-linker, so lld is the + # only option. + + link_flags.extend([ + "-fuse-ld=lld", + "-Wl,--build-id=md5", + "-Wl,--hash-style=gnu", + "-Wl,-z,relro,-z,now", + ]) + + opt_link_flags = ["-Wl,--gc-sections"] + + # Coverage flags: + coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"] + coverage_link_flags = ["-fprofile-instr-generate"] + + ## NOTE: framework paths is missing here; unix_cc_toolchain_config + ## doesn't seem to have a feature for this. + + # C++ built-in include directories: + cxx_builtin_include_directories = [ + "/usr/" + target_cpu + "-linux-gnu/include", + "/usr/lib/gcc-cross/" + target_cpu + "-linux-gnu/11/include", + ] + + # sysroot_path = compiler_configuration["sysroot_path"] + # sysroot_prefix = "" + # if sysroot_path: + # sysroot_prefix = "%sysroot%" + + # cxx_builtin_include_directories.extend([ + # sysroot_prefix + "/include", + # sysroot_prefix + "/usr/include", + # sysroot_prefix + "/usr/local/include", + # ]) + + if "additional_include_dirs" in compiler_configuration: + cxx_builtin_include_directories.extend(compiler_configuration["additional_include_dirs"]) + + ## NOTE: make variables are missing here; unix_cc_toolchain_config doesn't + ## pass these to `create_cc_toolchain_config_info`. + + # The tool names come from [here](https://github.com/bazelbuild/bazel/blob/c7e58e6ce0a78fdaff2d716b4864a5ace8917626/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java#L76-L90): + # NOTE: Ensure these are listed in toolchain_tools in toolchain/internal/common.bzl. + tool_paths = [ + tool_path( + name = "ar", + path = tools_path_prefix + "ar", + ), + tool_path( + name = "cpp", + path = tools_path_prefix + "g++", + ), + tool_path( + name = "gcc", + path = tools_path_prefix + "gcc", + ), + tool_path( + name = "gcov", + path = tools_path_prefix + "gcov", + ), + tool_path( + name = "ld", + path = tools_path_prefix + ctx.attr.ld, + ), + tool_path( + name = "nm", + path = tools_path_prefix + "nm", + ), + tool_path( + name = "objcopy", + path = tools_path_prefix + "objcopy", + ), + tool_path( + name = "objdump", + path = tools_path_prefix + "objdump", + ), + tool_path( + name = "strip", + path = tools_path_prefix + "strip", + ), + ] + + cxx_flags = [] + + # Replace flags with any user-provided overrides. + if "compile_flags" in compiler_configuration: + compile_flags = _fmt_flags(compiler_configuration["compile_flags"], toolchain_path_prefix) + if "cxx_flags" in compiler_configuration: + cxx_flags = _fmt_flags(compiler_configuration["cxx_flags"], toolchain_path_prefix) + if "link_flags" in compiler_configuration: + link_flags = _fmt_flags(compiler_configuration["link_flags"], toolchain_path_prefix) + if "link_libs" in compiler_configuration: + link_libs = _fmt_flags(compiler_configuration["link_libs"], toolchain_path_prefix) + if "opt_compile_flags" in compiler_configuration: + opt_compile_flags = _fmt_flags(compiler_configuration["opt_compile_flags"], toolchain_path_prefix) + if "opt_link_flags" in compiler_configuration: + opt_link_flags = _fmt_flags(compiler_configuration["opt_link_flags"], toolchain_path_prefix) + if "dbg_compile_flags" in compiler_configuration: + dbg_compile_flags = _fmt_flags(compiler_configuration["dbg_compile_flags"], toolchain_path_prefix) + if "coverage_compile_flags" in compiler_configuration: + coverage_compile_flags = _fmt_flags(compiler_configuration["coverage_compile_flags"], toolchain_path_prefix) + if "coverage_link_flags" in compiler_configuration: + coverage_link_flags = _fmt_flags(compiler_configuration["coverage_link_flags"], toolchain_path_prefix) + if "unfiltered_compile_flags" in compiler_configuration: + unfiltered_compile_flags = _fmt_flags(compiler_configuration["unfiltered_compile_flags"], toolchain_path_prefix) + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + compiler = "gcc", + toolchain_identifier = target_cpu + "-linux-gnu", + host_system_name = "local", + target_cpu = target_cpu, + target_system_name = target_cpu + "-linux-gnu", + target_libc = ctx.attr.target_libc, + # abi_version = "unknown", + # abi_libc_version = "unknown", + cxx_builtin_include_directories = cxx_builtin_include_directories, + tool_paths = tool_paths, + ) + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "target_cpu": attr.string(), + "toolchain_path_prefix": attr.string(), + "tools_path_prefix": attr.string(), + "compiler_configuration": attr.string_list_dict(allow_empty = True, default = {}), + "target_libc": attr.string(default = "gnu"), + "ld": attr.string(default = "gcc"), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/wrapper b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/wrapper new file mode 100755 index 000000000000..12c8a0830ad6 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/wrapper @@ -0,0 +1,14 @@ +#!/bin/bash + +PREFIX= +if [[ ! -z ${EXT_BUILD_ROOT} ]]; then + PREFIX=${EXT_BUILD_ROOT}/ +elif [[ ! -e external/gcc-11-x86_64-linux-musl-cross/bin ]]; then + echo "EXT_BUILD_ROOT is not set and wrapper can't find the toolchain, is this script running with the correct environment (foreign_cc rules, cc_* rules)?" + exit 1 +fi + +NAME=$(/usr/bin/basename "$0") +TOOLCHAIN_BINDIR=${PREFIX}external/gcc-11-x86_64-linux-musl-cross/bin + +exec "${TOOLCHAIN_BINDIR}"/"${NAME}" "$@" \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-addr2line b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-addr2line new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-addr2line @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ar b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ar new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ar @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-as b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-as new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-as @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++ new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++ @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++filt b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++filt new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-c++filt @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cc@ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cc@ new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cc@ @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cpp b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cpp new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-cpp @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-dwp b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-dwp new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-dwp @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-elfedit b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-elfedit new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-elfedit @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-g++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-g++ new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-g++ @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-11.2.1 b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-11.2.1 new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-11.2.1 @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ar b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ar new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ar @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-nm b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-nm new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-nm @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ranlib b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ranlib new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcc-ranlib @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-dump b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-dump new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-dump @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-tool b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-tool new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gcov-tool @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gfortran b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gfortran new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gfortran @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gprof b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gprof new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-gprof @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.bfd b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.bfd new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.bfd @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.gold b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.gold new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ld.gold @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-lto-dump b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-lto-dump new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-lto-dump @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-nm b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-nm new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-nm @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objcopy b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objcopy new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objcopy @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objdump b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objdump new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-objdump @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ranlib b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ranlib new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-ranlib @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-readelf b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-readelf new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-readelf @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-size b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-size new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-size @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strings b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strings new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strings @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strip b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strip new file mode 120000 index 000000000000..22cb46f8d789 --- /dev/null +++ b/build/toolchain/gcc-11-x86_64-linux-musl-wrappers/x86_64-linux-musl-strip @@ -0,0 +1 @@ +wrapper \ No newline at end of file diff --git a/build/toolchain/generate_wrappers.sh b/build/toolchain/generate_wrappers.sh new file mode 100755 index 000000000000..93b2b83d4a74 --- /dev/null +++ b/build/toolchain/generate_wrappers.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e + +name=$1 +wrapper=$2 +prefix=$3 +dummy_file=$4 + +if [[ -z $name || -z $wrapper || -z $prefix ]]; then + echo "Usage: $0 " + exit 1 +fi + +cwd=$(realpath $(dirname $(readlink -f ${BASH_SOURCE[0]}))) +dir=wrappers-$name +mkdir -p $cwd/$dir +cp $wrapper $cwd/$dir/ +chmod 755 $cwd/$dir/wrapper + +pushd $cwd/$dir >/dev/null + +tools="addr2line ar as c++ cc@ c++filt cpp dwp elfedit g++ gcc gcc-ar gcc-nm gcc-ranlib gcov gcov-dump gcov-tool gfortran gprof ld ld.bfd ld.gold lto-dump nm objcopy objdump ranlib readelf size strings strip" +for tool in $tools; do + ln -sf wrapper $prefix$tool +done + +popd >/dev/null + +if [[ -n $dummy_file ]]; then + touch $dummy_file +fi + diff --git a/build/toolchain/managed_toolchain.bzl b/build/toolchain/managed_toolchain.bzl new file mode 100644 index 000000000000..793b335924e9 --- /dev/null +++ b/build/toolchain/managed_toolchain.bzl @@ -0,0 +1,152 @@ +load(":cc_toolchain_config.bzl", "cc_toolchain_config") + +aarch64_glibc_distros = { + "rhel9": "11", + "rhel8": "8", + "aws2023": "11", + "aws2": "7", +} + +def _generate_wrappers_impl(ctx): + wrapper_file = ctx.actions.declare_file("wrapper") + ctx.actions.expand_template( + template = ctx.file._wrapper_template, + output = wrapper_file, + substitutions = { + "{{TOOLCHAIN_NAME}}": ctx.attr.toolchain_name, + }, + is_executable = True, + ) + + dummy_output = ctx.actions.declare_file(ctx.attr.name + ".wrapper-marker") + + ctx.actions.run_shell( + command = "build/toolchain/generate_wrappers.sh %s %s %s %s" % ( + ctx.attr.toolchain_name, + wrapper_file.path, + ctx.attr.tools_prefix, + dummy_output.path, + ), + progress_message = "Create wrappers for " + ctx.attr.toolchain_name, + inputs = [wrapper_file], + outputs = [dummy_output], + ) + + return [DefaultInfo(files = depset([dummy_output, wrapper_file]))] + +generate_wrappers = rule( + implementation = _generate_wrappers_impl, + attrs = { + "toolchain_name": attr.string(mandatory = True), + "tools_prefix": attr.string(mandatory = True), + "_wrapper_template": attr.label( + default = "//build/toolchain:templates/wrapper", + allow_single_file = True, + ), + }, +) + +def define_managed_toolchain( + name = None, + arch = "x86_64", + vendor = "unknown", + libc = "gnu", + gcc_version = "11", + ld = "gcc", + target_compatible_with = []): + identifier = "{arch}-{vendor}-linux-{libc}-gcc-{gcc_version}".format( + arch = arch, + vendor = vendor, + libc = libc, + gcc_version = gcc_version, + ) + + tools_prefix = "{arch}-{vendor}-linux-{libc}-".format( + arch = arch, + vendor = vendor, + libc = libc, + ) + + native.toolchain( + name = "%s_toolchain" % identifier, + exec_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:%s" % arch, + ] + target_compatible_with, + toolchain = ":%s_cc_toolchain" % identifier, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) + + cc_toolchain_config( + name = "%s_cc_toolchain_config" % identifier, + ld = ld, + target_cpu = arch, + target_libc = libc, + toolchain_path_prefix = "wrappers-%s/" % identifier, # is this required? + tools_path_prefix = "wrappers-%s/%s" % (identifier, tools_prefix), + ) + + generate_wrappers( + name = "%s_wrappers" % identifier, + toolchain_name = identifier, + tools_prefix = tools_prefix, + ) + + native.filegroup( + name = "%s_files" % identifier, + srcs = [ + ":%s_wrappers" % identifier, + "@%s//:toolchain" % identifier, + ], + ) + + native.cc_toolchain( + name = "%s_cc_toolchain" % identifier, + all_files = ":%s_files" % identifier, + compiler_files = ":%s_files" % identifier, + dwp_files = ":empty", + linker_files = "%s_files" % identifier, + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 0, + toolchain_config = ":%s_cc_toolchain_config" % identifier, + toolchain_identifier = "%s_cc_toolchain" % identifier, + ) + +def register_managed_toolchain(name = None, arch = "x86_64", vendor = "unknown", libc = "gnu", gcc_version = "11"): + identifier = "{arch}-{vendor}-linux-{libc}-gcc-{gcc_version}".format( + arch = arch, + vendor = vendor, + libc = libc, + gcc_version = gcc_version, + ) + native.register_toolchains("//build/toolchain:%s_toolchain" % identifier) + +def register_all_toolchains(name = None): + native.register_toolchains("//build/toolchain:local_aarch64-linux-gnu_toolchain") + + register_managed_toolchain( + arch = "x86_64", + gcc_version = "11", + libc = "musl", + vendor = "alpine", + ) + + register_managed_toolchain( + arch = "aarch64", + gcc_version = "11", + libc = "musl", + vendor = "alpine", + ) + + for vendor in aarch64_glibc_distros: + register_managed_toolchain( + arch = "aarch64", + gcc_version = aarch64_glibc_distros[vendor], + libc = "gnu", + vendor = vendor, + ) diff --git a/build/toolchain/repositories.bzl b/build/toolchain/repositories.bzl new file mode 100644 index 000000000000..d6bb18d70f0e --- /dev/null +++ b/build/toolchain/repositories.bzl @@ -0,0 +1,28 @@ +"""A module defining the third party dependency OpenResty""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def toolchain_repositories(): + http_archive( + name = "gcc-11-x86_64-linux-musl-cross", + url = "https://more.musl.cc/11/x86_64-linux-musl/x86_64-linux-musl-cross.tgz", + sha256 = "c6226824d6b7214ce974344b186179c9fa89be3c33dd7431c4b6585649ce840b", + strip_prefix = "x86_64-linux-musl-cross", + build_file_content = """ +filegroup( + name = "toolchain", + srcs = glob( + include = [ + "bin/**", + "include/**", + "lib/**", + "libexec/**", + "share/**", + "x86_64-linux-musl/**", + ], + exclude = ["usr"], + ), + visibility = ["//visibility:public"], +) + """, + ) diff --git a/build/toolchain/templates/wrapper b/build/toolchain/templates/wrapper new file mode 100644 index 000000000000..cb52306cfca1 --- /dev/null +++ b/build/toolchain/templates/wrapper @@ -0,0 +1,14 @@ +#!/bin/bash + +PREFIX= +if [[ ! -z ${EXT_BUILD_ROOT} ]]; then + PREFIX=${EXT_BUILD_ROOT}/ +elif [[ ! -e external/{{TOOLCHAIN_NAME}}/bin ]]; then + echo "EXT_BUILD_ROOT is not set and wrapper can't find the toolchain, is this script running with the correct environment (foreign_cc rules, cc_* rules)?" + exit 1 +fi + +NAME=$(/usr/bin/basename "$0") +TOOLCHAIN_BINDIR=${PREFIX}external/{{TOOLCHAIN_NAME}}/bin + +exec "${TOOLCHAIN_BINDIR}"/"${NAME}" "$@" \ No newline at end of file diff --git a/scripts/backoff.sh b/scripts/backoff.sh new file mode 100644 index 000000000000..167b73bbbfe5 --- /dev/null +++ b/scripts/backoff.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# This script is from the Kong/kong-build-tools repo. +# Retries a command a configurable number of times with backoff. +# +# The retry count is given by ATTEMPTS (default 5), the initial backoff +# timeout is given by TIMEOUT in seconds (default 1.) +# +# Successive backoffs double the timeout. +function with_backoff { + local max_attempts=${ATTEMPTS-5} + local timeout=${TIMEOUT-5} + local attempt=1 + local exitCode=0 + + while (( $attempt < $max_attempts )) + do + if "$@" + then + return 0 + else + exitCode=$? + fi + + echo "Failure! Retrying in $timeout.." 1>&2 + sleep $timeout + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done + + if [[ $exitCode != 0 ]] + then + echo "You've failed me for the last time! ($*)" 1>&2 + fi + + return $exitCode +} diff --git a/scripts/build-kong.sh b/scripts/build-kong.sh new file mode 100644 index 000000000000..a672930778e9 --- /dev/null +++ b/scripts/build-kong.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -e + +# This script is from the Kong/kong-build-tools repo, and is used to build Kong. + +source .requirements +source scripts/backoff.sh + +ROCKS_CONFIG=$(mktemp) +echo " +rocks_trees = { + { name = [[system]], root = [[/tmp/build/usr/local]] } +} +" > $ROCKS_CONFIG + +if [ -e "/.dockerenv" ]; then + cp -r /tmp/build/usr/local/* /usr/local/ +else + # TODO: skip on macOS + # roolesskit create mount_namespaces(7), thus this mount doesn't + # affect host and will be cleanup upon exit + mount -o bind,ro /tmp/build/usr/local/ /usr/local +fi + +export LUAROCKS_CONFIG=$ROCKS_CONFIG +export LUA_PATH="/usr/local/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;;" +export PATH=$PATH:/usr/local/openresty/luajit/bin + +/usr/local/bin/luarocks --version +/usr/local/kong/bin/openssl version || true +ldd /usr/local/openresty/nginx/sbin/nginx || true +strings /usr/local/openresty/nginx/sbin/nginx | grep rpath || true +strings /usr/local/openresty/bin/openresty | grep rpath || true +find /usr/local/kong/lib/ || true +/usr/local/openresty/bin/openresty -V || true + +ROCKSPEC_VERSION=$(basename kong-*.rockspec) \ +&& ROCKSPEC_VERSION=${ROCKSPEC_VERSION%.*} \ +&& ROCKSPEC_VERSION=${ROCKSPEC_VERSION#"kong-"} + +mkdir -p /tmp/plugin + +if [ "$SSL_PROVIDER" = "boringssl" ]; then + sed -i 's/fips = off/fips = on/g' kong/templates/kong_defaults.lua +fi + +# EE +LUAROCKS_ARGS=() +if [ -e "scripts/build-kong-ee.sh" ]; then + LUAROCKS_ARGS+=("YAML_LIBDIR=/tmp/build/usr/local/kong/lib") + LUAROCKS_ARGS+=("YAML_INCDIR=/tmp/yaml") +fi + +with_backoff /usr/local/bin/luarocks make kong-${ROCKSPEC_VERSION}.rockspec \ +CRYPTO_DIR=/usr/local/kong \ +OPENSSL_DIR=/usr/local/kong \ +EXPAT_DIR=/usr/local/kong \ +LIBXML2_DIR=/usr/local/kong \ +CFLAGS="-L/tmp/build/usr/local/kong/lib -Wl,-rpath,/usr/local/kong/lib -O2 -std=gnu99 -fPIC" \ +${LUAROCKS_ARGS[@]} + +mkdir -p /tmp/build/etc/kong +cp -Lf kong.conf.default /tmp/build/usr/local/lib/luarocks/rock*/kong/$ROCKSPEC_VERSION/ +cp -Lf kong.conf.default /tmp/build/etc/kong/kong.conf.default + +# /usr/local/kong/include is usually created by other C libraries, like openssl +# call mkdir here to make sure it's created +if [ -e "kong/include" ]; then + mkdir -p /tmp/build/usr/local/kong/include + cp -Lrf kong/include/* /tmp/build/usr/local/kong/include/ +fi + +# circular dependency of CI: remove after https://github.com/Kong/kong-distributions/pull/791 is merged +if [ -e "kong/pluginsocket.proto" ]; then + cp -Lf kong/pluginsocket.proto /tmp/build/usr/local/kong/include/kong +fi + +with_backoff curl -fsSLo /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip +unzip -o /tmp/protoc.zip -d /tmp/protoc 'include/*' +cp -rf /tmp/protoc/include/google /tmp/build/usr/local/kong/include/ + +cp COPYRIGHT /tmp/build/usr/local/kong/ +cp bin/kong /tmp/build/usr/local/bin/kong +sed -i 's/resty/\/usr\/local\/openresty\/bin\/resty/' /tmp/build/usr/local/bin/kong +sed -i 's/\/tmp\/build//g' /tmp/build/usr/local/bin/openapi2kong || true +grep -l -I -r '\/tmp\/build' /tmp/build/ || true +sed -i 's/\/tmp\/build//' $(grep -l -I -r '\/tmp\/build' /tmp/build/) || true + +# EE +if [ -e "scripts/build-kong-ee.sh" ]; then + ./scripts/build-kong-ee.sh +fi diff --git a/scripts/check_spec_files_spelling.sh b/scripts/check_spec_files_spelling.sh new file mode 100755 index 000000000000..556589c7c93e --- /dev/null +++ b/scripts/check_spec_files_spelling.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e + +function red() { + echo -e "\033[1;31m$*\033[0m" +} + +readarray -t FOUND < \ +<( + git ls-files 'spec/[0-9]**.lua' \ + | grep -vE \ + -e '_spec.lua$' \ + -f spec/on_demand_specs +) + +if (( ${#FOUND[@]} > 0 )); then + echo + red "----------------------------------------------------------------" + echo "Found some files in spec directory that do not have the _spec suffix, please check if you're misspelling them. If there is an exception, please add the coressponding files(or their path regexes) into the whitelist spec/on_demand_specs." + echo + echo "Possible misspelling file list:" + echo + printf "%s\n" "${FOUND[@]}" + red "----------------------------------------------------------------" + exit 1 +fi diff --git a/scripts/explain_manifest/.gitignore b/scripts/explain_manifest/.gitignore new file mode 100644 index 000000000000..bee8a64b79a9 --- /dev/null +++ b/scripts/explain_manifest/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/scripts/explain_manifest/config.py b/scripts/explain_manifest/config.py new file mode 100644 index 000000000000..1d64a5fa98b0 --- /dev/null +++ b/scripts/explain_manifest/config.py @@ -0,0 +1,18 @@ + +from globmatch import glob_match + +from main import FileInfo + +def transform(f: FileInfo): + # XXX: libxslt uses libtool and it injects some extra rpaths + # we only care about the kong library rpath so removing it here + # until we find a way to remove the extra rpaths from it + # It should have no side effect as the extra rpaths are long random + # paths created by bazel. + + if glob_match(f.path, ["**/kong/lib/libxslt.so*", "**/kong/lib/libexslt.so*"]): + if f.rpath and "/usr/local/kong/lib" in f.rpath: + f.rpath = "/usr/local/kong/lib" + elif f.runpath and "/usr/local/kong/lib" in f.runpath: + f.runpath = "/usr/local/kong/lib" + # otherwise remain unmodified diff --git a/scripts/explain_manifest/filelist.txt b/scripts/explain_manifest/filelist.txt new file mode 100644 index 000000000000..29667eae7d4a --- /dev/null +++ b/scripts/explain_manifest/filelist.txt @@ -0,0 +1,8 @@ +**/*.so +**/kong/lib/**.so* +**/kong/gui +**/kong/portal +**/kong/include/kong +**/kong/include/google +**/openresty/nginx/sbin/nginx +**/share/xml/xsd diff --git a/scripts/explain_manifest/fixtures/alpine-amd64.txt b/scripts/explain_manifest/fixtures/alpine-amd64.txt new file mode 100644 index 000000000000..d18d9c9af44c --- /dev/null +++ b/scripts/explain_manifest/fixtures/alpine-amd64.txt @@ -0,0 +1,224 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libc.so + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libc.so + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libnettle.so.8 + - libgmp.so.10 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libonig.so.5 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libopentracing.so.1 + Needed : + - libc.so + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so + +- Path : /usr/local/openresty/lualib/librestysignal.so + Needed : + - libc.so + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libluajit-5.1.so.2 + - libstdc++.so.6 + - libopentracing.so.1 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libgcc_s.so.1 + - libc.so + Rpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + - nginx-opentracing/opentracing + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt b/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt new file mode 100644 index 000000000000..de5349d37966 --- /dev/null +++ b/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt @@ -0,0 +1,278 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libdd_opentracing_plugin.so + Needed : + - libpthread.so.0 + - libstdc++.so.6 + - libgcc_s.so.1 + - libc.so.6 + - ld-linux-x86-64.so.2 + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libgcc_s.so.1 + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libopentracing.so.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libm.so.6 + - libstdc++.so.6 + - libopentracing.so.1 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libgcc_s.so.1 + - libc.so.6 + Rpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + - nginx-opentracing/opentracing + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/debian-10-amd64.txt b/scripts/explain_manifest/fixtures/debian-10-amd64.txt new file mode 100644 index 000000000000..d9dceca2bd89 --- /dev/null +++ b/scripts/explain_manifest/fixtures/debian-10-amd64.txt @@ -0,0 +1,259 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libgcc_s.so.1 + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libm.so.6 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Runpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/debian-11-amd64.txt b/scripts/explain_manifest/fixtures/debian-11-amd64.txt new file mode 100644 index 000000000000..4338b77fccf7 --- /dev/null +++ b/scripts/explain_manifest/fixtures/debian-11-amd64.txt @@ -0,0 +1,241 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + +- Path : /usr/local/openresty/lualib/rds/parser.so + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Runpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory diff --git a/scripts/explain_manifest/fixtures/el7-amd64.txt b/scripts/explain_manifest/fixtures/el7-amd64.txt new file mode 100644 index 000000000000..aa97d0277db7 --- /dev/null +++ b/scripts/explain_manifest/fixtures/el7-amd64.txt @@ -0,0 +1,258 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libm.so.6 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Rpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/el8-amd64.txt b/scripts/explain_manifest/fixtures/el8-amd64.txt new file mode 100644 index 000000000000..5d976ca85119 --- /dev/null +++ b/scripts/explain_manifest/fixtures/el8-amd64.txt @@ -0,0 +1,259 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libgcc_s.so.1 + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Rpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libm.so.6 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Rpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/ubuntu-18.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-18.04-amd64.txt new file mode 100644 index 000000000000..8bf9b520fa90 --- /dev/null +++ b/scripts/explain_manifest/fixtures/ubuntu-18.04-amd64.txt @@ -0,0 +1,211 @@ +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/include/kong + Type : directory + +- Path : /usr/local/kong/lib/engines-1.1/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.1.1 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + - libcrypto.so.1.1 (OPENSSL_1_1_0) + +- Path : /usr/local/kong/lib/engines-1.1/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.1.1 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5) + +- Path : /usr/local/kong/lib/engines-1.1/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.1.1 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5) + - libcrypto.so.1.1 (OPENSSL_1_1_0) + +- Path : /usr/local/kong/lib/libcrypto.so.1.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.15, GLIBC_2.16, GLIBC_2.17, GLIBC_2.2.5, GLIBC_2.25, GLIBC_2.3, GLIBC_2.3.4, GLIBC_2.4, GLIBC_2.7) + - libdl.so.2 (GLIBC_2.2.5) + +- Path : /usr/local/kong/lib/libssl.so.1.1 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.1.1 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + - libcrypto.so.1.1 (OPENSSL_1_1_0, OPENSSL_1_1_0d, OPENSSL_1_1_0f, OPENSSL_1_1_0i, OPENSSL_1_1_1) + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.3, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.15, GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.1.1 + - libcrypto.so.1.1 + - libc.so.6 + Runpath : /usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + - libcrypto.so.1.1 (OPENSSL_1_1_0) + - libssl.so.1.1 (OPENSSL_1_1_0, OPENSSL_1_1_1) + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.2.5, GLIBC_2.3.4, GLIBC_2.4) + +- Path : /usr/local/openresty/lualib/libatc_router.so + Needed : + - libgcc_s.so.1 + - librt.so.1 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + - ld-linux-x86-64.so.2 + - libstdc++.so.6 + Version Requirement: + - ld-linux-x86-64.so.2 (GLIBC_2.3) + - libc.so.6 (GLIBC_2.10, GLIBC_2.14, GLIBC_2.15, GLIBC_2.18, GLIBC_2.2.5, GLIBC_2.25, GLIBC_2.27, GLIBC_2.3, GLIBC_2.3.4, GLIBC_2.4, GLIBC_2.5, GLIBC_2.6, GLIBC_2.9) + - libdl.so.2 (GLIBC_2.2.5) + - libgcc_s.so.1 (GCC_3.0, GCC_3.3, GCC_4.2.0) + - libpthread.so.0 (GLIBC_2.12, GLIBC_2.2.5) + - librt.so.1 (GLIBC_2.2.5) + +- Path : /usr/local/openresty/lualib/librestysignal.so + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + Version Requirement: + - libc.so.6 (GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + Version Requirement: + - libc.so.6 (GLIBC_2.14, GLIBC_2.2.5, GLIBC_2.4) + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libssl.so.1.1 + - libcrypto.so.1.1 + - libz.so.1 + - libc.so.6 + Runpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Version Requirement: + - libc.so.6 (GLIBC_2.10, GLIBC_2.11, GLIBC_2.14, GLIBC_2.17, GLIBC_2.2.5, GLIBC_2.27, GLIBC_2.3, GLIBC_2.3.2, GLIBC_2.3.4, GLIBC_2.4, GLIBC_2.7) + - libcrypt.so.1 (GLIBC_2.2.5) + - libcrypto.so.1.1 (OPENSSL_1_1_0) + - libdl.so.2 (GLIBC_2.2.5) + - libpthread.so.0 (GLIBC_2.12, GLIBC_2.2.5, GLIBC_2.3.2) + - libssl.so.1.1 (OPENSSL_1_1_0, OPENSSL_1_1_1) + Modules : + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-events + - lua-resty-lmdb + OpenSSL : OpenSSL 1.1.1t 7 Feb 2023 + diff --git a/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt new file mode 100644 index 000000000000..1804797539d0 --- /dev/null +++ b/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt @@ -0,0 +1,247 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libgcc_s.so.1 + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libm.so.6 + - libonig.so.5 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libpthread.so.0 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libpthread.so.0 + - libm.so.6 + - libdl.so.2 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libdl.so.2 + - libpthread.so.0 + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Runpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt new file mode 100644 index 000000000000..bad7252ad6f6 --- /dev/null +++ b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt @@ -0,0 +1,233 @@ +- Path : /etc/kong/kong.logrotate + +- Path : /etc/logrotate.d/kong-enterprise-edition + Link : /etc/kong/kong.logrotate + Type : link + +- Path : /lib/systemd/system/kong-enterprise-edition.service + +- Path : /usr/local/kong/gui + Type : directory + +- Path : /usr/local/kong/include/google + Type : directory + +- Path : /usr/local/kong/lib/engines-3/afalg.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/capi.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/loader_attic.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/engines-3/padlock.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libcrypto.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libexpat.so.1.8.10 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libgmp.so.10.4.1 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libhogweed.so.6.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libnettle.so.8 + - libgmp.so.10 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libjq.so.1.0.4 + Needed : + - libm.so.6 + - libonig.so.5 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/liblicense_utils.so + Needed : + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libnettle.so.8.4 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libonig.so.5.3.0 + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/libpasswdqc.so.0 + Needed : + - libc.so.6 + +- Path : /usr/local/kong/lib/libssl.so.3 + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/lib/ossl-modules/legacy.so + Needed : + - libstdc++.so.6 + - libm.so.6 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/kong/portal + Type : directory + +- Path : /usr/local/lib/lua/5.1/_openssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libm.so.6 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/bcrypt.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lfs.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lpeg.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lsyslog.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_pack.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lua_system_constants.so + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/lxp.so + Needed : + - libexpat.so.1 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/mime/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/pb.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/core.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/serial.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/socket/unix.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/ssl.so + Needed : + - libssl.so.3 + - libcrypto.so.3 + - libc.so.6 + Runpath : /usr/local/kong/lib + +- Path : /usr/local/lib/lua/5.1/yaml.so + Needed : + - libyaml-0.so.2 + - libc.so.6 + +- Path : /usr/local/openresty/lualib/cjson.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/librestysignal.so + +- Path : /usr/local/openresty/lualib/rds/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/lualib/redis/parser.so + Needed : + - libc.so.6 + +- Path : /usr/local/openresty/nginx/sbin/nginx + Needed : + - libcrypt.so.1 + - libluajit-5.1.so.2 + - libssl.so.3 + - libcrypto.so.3 + - libz.so.1 + - libc.so.6 + Runpath : /usr/local/openresty/luajit/lib:/usr/local/kong/lib + Modules : + - kong-licensing/ngx_module + - lua-kong-nginx-module + - lua-kong-nginx-module/stream + - lua-resty-openssl-aux-module + - lua-resty-openssl-aux-module/stream + OpenSSL : OpenSSL 3.1.4 24 Oct 2023 + DWARF : True + DWARF - ngx_http_request_t related DWARF DIEs: True + +- Path : /usr/local/share/lua/5.1/kong/portal + Type : directory + diff --git a/scripts/explain_manifest/main.py b/scripts/explain_manifest/main.py new file mode 100755 index 000000000000..053c87fb55d4 --- /dev/null +++ b/scripts/explain_manifest/main.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import glob +import atexit +import argparse +import tempfile +from pathlib import Path + +import lief +from globmatch import glob_match + +import config + + +class ExplainOpts(): + # General + owners = True + mode = True + size = False + # ELF + merge_rpaths_runpaths = False + imported_symbols = False + exported_symbols = False + version_requirement = True + + @classmethod + def from_args(this, args): + this.owners = args.owners + this.mode = args.mode + this.size = args.size + this.merge_rpaths_runpaths = args.merge_rpaths_runpaths + this.imported_symbols = args.imported_symbols + this.exported_symbols = args.exported_symbols + this.version_requirement = args.version_requirement + + return this + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--path", "-p", help="Path to the directory to compare", required=True) + parser.add_argument( + "--output", "-o", help="Path to output manifest, use - to write to stdout", default="-") + parser.add_argument( + "--file_list", "-f", help="Path to the files list to explain for manifest; " + \ + "each line in the file should be a glob pattern of full path") + parser.add_argument( + "--owners", help="Export and compare owner and group", action="store_true") + parser.add_argument( + "--mode", help="Export and compare mode", action="store_true") + parser.add_argument( + "--size", help="Export and compare size", action="store_true") + parser.add_argument("--merge_rpaths_runpaths", + help="Treate RPATH and RUNPATH as same", action="store_true") + parser.add_argument( + "--imported_symbols", help="Export and compare imported symbols", action="store_true") + parser.add_argument( + "--exported_symbols", help="Export and compare exported symbols", action="store_true") + parser.add_argument("--version_requirement", + help="Export and compare exported symbols (default to True)", + action="store_true", default=True) + + return parser.parse_args() + + +def read_glob(path): + if not path: + return ["**"] + + with open(path, "r") as f: + return f.read().splitlines() + + +def gather_files(path): + ext = os.path.splitext(path)[1] + if ext in (".deb", ".rpm") or path.endswith(".apk.tar.gz"): + t = tempfile.TemporaryDirectory() + atexit.register(t.cleanup) + + if ext == ".deb": + code = os.system( + "ar p %s data.tar.gz | tar -C %s -xz" % (path, t.name)) + elif ext == ".rpm": + # GNU gpio and rpm2cpio is needed + code = os.system( + "rpm2cpio %s | cpio --no-preserve-owner --no-absolute-filenames -idm -D %s" % (path, t.name)) + elif ext == ".gz": + code = os.system("tar -C %s -xf %s" % (t.name, path)) + + if code != 0: + raise Exception("Failed to extract %s" % path) + + return t.name + elif not Path(path).is_dir(): + raise Exception("Don't know how to process \"%s\"" % path) + + return path + + +class FileInfo(): + def __init__(self, path, relpath): + self.path = path + self.relpath = relpath + self.mode = os.stat(path).st_mode + self.uid = os.stat(path).st_uid + self.gid = os.stat(path).st_gid + self.size = os.stat(path).st_size + + if Path(path).is_symlink(): + self.link = os.readlink(path) + elif Path(path).is_dir(): + self.directory = True + + def explain(self, opts): + lines = [("Path", self.relpath)] + if hasattr(self, "link"): + lines.append(("Link", self.link)) + lines.append(("Type", "link")) + elif hasattr(self, "directory"): + lines.append(("Type", "directory")) + + if opts.owners: + lines.append(("Uid,Gid", "%s, %s" % (self.uid, self.gid))) + if opts.mode: + lines.append(("Mode", oct(self.mode))) + if opts.size: + lines.append(("Size", self.size)) + + return lines + + +class ElfFileInfo(FileInfo): + def __init__(self, path, relpath): + super().__init__(path, relpath) + + self.needed = [] + self.rpath = None + self.runpath = None + self.get_exported_symbols = None + self.get_imported_symbols = None + self.version_requirement = [] + + binary = lief.parse(path) + if not binary: # not an ELF file, malformed, etc + return + + for d in binary.dynamic_entries: + if d.tag == lief.ELF.DYNAMIC_TAGS.NEEDED: + self.needed.append(d.name) + elif d.tag == lief.ELF.DYNAMIC_TAGS.RPATH: + self.rpath = d.name + elif d.tag == lief.ELF.DYNAMIC_TAGS.RUNPATH: + self.runpath = d.name + + # create closures and lazily evaluated + self.get_exported_symbols = lambda: sorted( + [d.name for d in binary.exported_symbols]) + self.get_imported_symbols = lambda: sorted( + [d.name for d in binary.imported_symbols]) + + for f in binary.symbols_version_requirement: + self.version_requirement.append("%s (%s)" % ( + f.name, ", ".join(sorted([a.name for a in f.get_auxiliary_symbols()])))) + self.version_requirement = sorted(self.version_requirement) + + def explain(self, opts): + pline = super().explain(opts) + + lines = [] + + if self.needed: + lines.append(("Needed", self.needed)) + if self.rpath: + lines.append(("Rpath", self.rpath)) + if self.runpath: + lines.append(("Runpath", self.runpath)) + if opts.exported_symbols and self.get_exported_symbols: + lines.append(("Exported", self.get_exported_symbols())) + if opts.imported_symbols and self.get_imported_symbols: + lines.append(("Imported", self.get_imported_symbols())) + if opts.version_requirement and self.version_requirement: + lines.append(("Version Requirement", self.version_requirement)) + + return pline + lines + + +class NginxInfo(ElfFileInfo): + def __init__(self, path, relpath): + super().__init__(path, relpath) + + self.modules = [] + self.linked_openssl = None + + binary = lief.parse(path) + + for s in binary.strings: + if re.match("\s*--prefix=/", s): + for m in re.findall("add(?:-dynamic)?-module=(.*?) ", s): + if m.startswith("../"): # skip bundled modules + continue + pdir = os.path.basename(os.path.dirname(m)) + mname = os.path.basename(m) + if pdir in ("external", "distribution"): + self.modules.append(mname) + else: + self.modules.append(os.path.join(pdir, mname)) + self.modules = sorted(self.modules) + elif m := re.match("^built with (.+) \(running with", s): + self.linked_openssl = m.group(1).strip() + + def explain(self, opts): + pline = super().explain(opts) + + lines = [] + lines.append(("Modules", self.modules)) + lines.append(("OpenSSL", self.linked_openssl)) + + return pline + lines + + +def walk_files(path, globs): + results = [] + for file in sorted(glob.glob("**", root_dir=path, recursive=True)): + if not glob_match(file, globs): + continue + + full_path = os.path.join(path, file) + + if not file.startswith("/") and not file.startswith("./"): + file = '/' + file # prettifier + + if os.path.basename(file) == "nginx": + f = NginxInfo(full_path, file) + elif os.path.splitext(file)[1] == ".so" or os.path.basename(os.path.dirname(file)) in ("bin", "lib", "lib64", "sbin"): + p = Path(full_path) + if p.is_symlink(): + continue + f = ElfFileInfo(full_path, file) + else: + f = FileInfo(full_path, file) + + config.transform(f) + results.append(f) + + return results + + +def write_manifest(title, results, opts: ExplainOpts, output): + if output == "-": + f = sys.stdout + else: + f = open(output, "w") + + print("# Manifest for %s\n\n" % title) + + for result in results: + entries = result.explain(opts) + ident = 2 + first = True + for k, v in entries: + if isinstance(v, list): + v = ("\n" + " " * ident + "- ").join([""] + v) + else: + v = " %s" % v + if first: + f.write("-" + (" " * (ident-1))) + first = False + else: + f.write(" " * ident) + f.write("%-10s:%s\n" % (k, v)) + f.write("\n") + + f.flush() + + if f != sys.stdout: + f.close() + + +if __name__ == "__main__": + args = parse_args() + + globs = read_glob(args.file_list) + + directory = gather_files(args.path) + + infos = walk_files(directory, globs) + + if Path(args.path).is_file(): + title = "contents in archive %s" % args.path + else: + title = "contents in directory %s" % args.path + + write_manifest(title, infos, ExplainOpts.from_args(args), args.output) diff --git a/scripts/explain_manifest/requirements.txt b/scripts/explain_manifest/requirements.txt new file mode 100644 index 000000000000..64127b9f7d84 --- /dev/null +++ b/scripts/explain_manifest/requirements.txt @@ -0,0 +1,2 @@ +lief==0.12.* +globmatch==2.0.* \ No newline at end of file diff --git a/scripts/grep-kong-version.sh b/scripts/grep-kong-version.sh new file mode 100755 index 000000000000..ecc7d1c683fd --- /dev/null +++ b/scripts/grep-kong-version.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# unofficial strict mode +set -euo pipefail + +kong_version=$(echo kong-*.rockspec | sed 's,.*/,,' | cut -d- -f2) + +if test -f "kong/enterprise_edition/meta.lua"; then + ee_patch=$(grep -o -E 'ee_patch[ \t]+=[ \t]+[0-9]+' kong/enterprise_edition/meta.lua | awk '{print $3}') + kong_version="$kong_version.$ee_patch" +fi + +echo "$kong_version" diff --git a/scripts/release-kong.sh b/scripts/release-kong.sh new file mode 100755 index 000000000000..4ca79d731849 --- /dev/null +++ b/scripts/release-kong.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# This script is from the Kong/kong-build-tools repo, and is used to release Kong to Pulp. +set -eo pipefail + +source .requirements +source scripts/backoff.sh + +KONG_VERSION=$(bash scripts/grep-kong-version.sh) +KONG_RELEASE_LABEL=${KONG_RELEASE_LABEL:-$KONG_VERSION} + +PULP_HOST=${PULP_HOST:-"https://api.download-dev.konghq.com"} +PULP_USERNAME=${PULP_USERNAME:-"admin"} +PULP_PASSWORD=${PULP_PASSWORD:-} + +PULP_DOCKER_IMAGE="kong/release-script" + +# Variables used by the release script +ARCHITECTURE=${ARCHITECTURE:-amd64} +PACKAGE_TYPE=${PACKAGE_TYPE:-deb} +ARTIFACT_TYPE=${ARTIFACT_TYPE:-debian} + +ARTIFACT_PREFIX=${ARTIFACT_PREFIX:-"bazel-bin/pkg"} +ARTIFACT=${ARTIFACT:-"kong.deb"} +ARTIFACT_VERSION=${ARTIFACT_VERSION:-} + +KONG_ARTIFACT=$ARTIFACT_PREFIX/$ARTIFACT + +# TODO: remove this once we have a better way to determine if we are releasing +case "$ARTIFACT_TYPE" in + debian|ubuntu) + OUTPUT_FILE_SUFFIX=".$ARTIFACT_VERSION.$ARCHITECTURE.deb" + ;; + centos) + OUTPUT_FILE_SUFFIX=".el$ARTIFACT_VERSION.$ARCHITECTURE.rpm" + ;; + rhel) + OUTPUT_FILE_SUFFIX=".rhel$ARTIFACT_VERSION.$ARCHITECTURE.rpm" + ;; + alpine) + OUTPUT_FILE_SUFFIX=".$ARCHITECTURE.apk.tar.gz" + ;; + amazonlinux) + OUTPUT_FILE_SUFFIX=".aws.$ARCHITECTURE.rpm" + ;; + src) + OUTPUT_FILE_SUFFIX=".tar.gz" + ;; +esac + + +DIST_FILE="$KONG_PACKAGE_NAME-$KONG_RELEASE_LABEL$OUTPUT_FILE_SUFFIX" + +function push_package () { + + local dist_version="--dist-version $ARTIFACT_VERSION" + + # TODO: CE gateway-src + + if [ "$ARTIFACT_TYPE" == "alpine" ]; then + dist_version= + fi + + if [ "$ARTIFACT_VERSION" == "18.04" ]; then + dist_version="--dist-version bionic" + fi + if [ "$ARTIFACT_VERSION" == "20.04" ]; then + dist_version="--dist-version focal" + fi + if [ "$ARTIFACT_VERSION" == "22.04" ]; then + dist_version="--dist-version jammy" + fi + + set -x + + local release_args="--package-type gateway" + if [[ "$EDITION" == "enterprise" ]]; then + release_args="$release_args --enterprise" + fi + + # pre-releases go to `/internal/` + if [[ "$OFFICIAL_RELEASE" == "true" ]]; then + release_args="$release_args --publish" + else + release_args="$release_args --internal" + fi + + docker run \ + -e PULP_HOST="$PULP_HOST" \ + -e PULP_USERNAME="$PULP_USERNAME" \ + -e PULP_PASSWORD="$PULP_PASSWORD" \ + -v "$(pwd)/$KONG_ARTIFACT:/files/$DIST_FILE" \ + -i $PULP_DOCKER_IMAGE \ + --file "/files/$DIST_FILE" \ + --dist-name "$ARTIFACT_TYPE" $dist_version \ + --major-version "${KONG_VERSION%%.*}.x" \ + $release_args + + if [[ $? -ne 0 ]]; then + exit 1 + fi +} + +with_backoff push_package + +echo -e "\nReleasing Kong '$KONG_RELEASE_LABEL' of '$ARTIFACT_TYPE $ARTIFACT_VERSION' done" + +exit 0 diff --git a/scripts/test-upgrade-path.sh b/scripts/test-upgrade-path.sh new file mode 100755 index 000000000000..fa1477181561 --- /dev/null +++ b/scripts/test-upgrade-path.sh @@ -0,0 +1,188 @@ +#!/bin/bash + +set -e + +trap "echo exiting because of error" 0 + +function usage() { + cat 1>&2 < [ ... ] + + and need to be git versions + + Options: + -n just run the tests, don't build containers (they need to already exist) + -i proceed even if not all migrations have tests + -d postgres|cassandra select database type + +EOF +} + +DATABASE=postgres + +args=$(getopt nd:i $*) +if [ $? -ne 0 ] +then + usage + exit 1 +fi +set -- $args + +while :; do + case "$1" in + -n) + NO_BUILD=1 + shift + ;; + -d) + DATABASE=$2 + shift + shift + ;; + -i) + IGNORE_MISSING_TESTS=1 + shift + ;; + --) + shift + break + ;; + *) + usage + exit 1 + ;; + esac +done + +if [ $# -lt 2 ] +then + echo "Missing or " + usage + exit 1 +fi + +OLD_VERSION=$1 +NEW_VERSION=$2 +shift ; shift +TESTS=$* + +NETWORK_NAME=migration-$OLD_VERSION-$NEW_VERSION + +# Between docker-compose v1 and docker-compose v2, the delimiting +# character for container names was changed from "-" to "_". +if [[ "$(docker-compose --version)" =~ v2 ]] +then + OLD_CONTAINER=$(gojira prefix -t $OLD_VERSION)-kong-1 + NEW_CONTAINER=$(gojira prefix -t $NEW_VERSION)-kong-1 +else + OLD_CONTAINER=$(gojira prefix -t $OLD_VERSION)_kong_1 + NEW_CONTAINER=$(gojira prefix -t $NEW_VERSION)_kong_1 +fi + +function build_containers() { + echo "Building containers" + + gojira up -t $OLD_VERSION --network $NETWORK_NAME --$DATABASE + gojira run -t $OLD_VERSION -- make dev + gojira up -t $NEW_VERSION --alone --network $NETWORK_NAME --$DATABASE + gojira run -t $NEW_VERSION -- make dev +} + +function initialize_test_list() { + echo "Determining tests to run" + + # Prepare list of tests to run + if [ -z "$TESTS" ] + then + all_tests_file=$(mktemp) + available_tests_file=$(mktemp) + + gojira run -t $NEW_VERSION -- kong migrations status \ + | jq -r '.new_migrations | .[] | (.namespace | gsub("[.]"; "/")) as $namespace | .migrations[] | "\($namespace)/\(.)_spec.lua" | gsub("^kong"; "spec/05-migration")' \ + | sort > $all_tests_file + gojira run -t $NEW_VERSION -- ls 2>/dev/null $(cat $all_tests_file) \ + | sort > $available_tests_file + + if [ "$IGNORE_MISSING_TESTS" = "1" ] + then + TESTS=$(cat $available_tests_file) + else + if ! cmp -s $available_tests_file $all_tests_file + then + echo "Not all migrations have corresponding tests, cannot continue. Missing test(s):" + echo + comm -13 $available_tests_file $all_tests_file \ + | perl -pe 's/^/ /g' + echo + rm $available_tests_file $all_tests_file + exit 1 + fi + TESTS=$(cat $all_tests_file) + fi + rm $available_tests_file $all_tests_file + fi + + echo "Going to run:" + echo $TESTS | perl -pe 's/(^| )/\n /g' + + # Make tests available in OLD container + TESTS_TAR=/tmp/upgrade-tests-$$.tar + docker exec ${NEW_CONTAINER} tar cf ${TESTS_TAR} spec/upgrade_helpers.lua $TESTS + docker cp ${NEW_CONTAINER}:${TESTS_TAR} ${TESTS_TAR} + docker cp ${TESTS_TAR} ${OLD_CONTAINER}:${TESTS_TAR} + docker exec ${OLD_CONTAINER} tar xf ${TESTS_TAR} + rm ${TESTS_TAR} +} + +function run_tests() { + # Run the tests + BUSTED="env KONG_DNS_RESOLVER= KONG_TEST_CASSANDRA_KEYSPACE=kong KONG_TEST_PG_DATABASE=kong bin/busted" + + while true + do + # Initialize database + gojira run -t $OLD_VERSION -- kong migrations reset --yes || true + gojira run -t $OLD_VERSION -- kong migrations bootstrap + + if [ -z "$TEST_LIST_INITIALIZED" ] + then + initialize_test_list + TEST_LIST_INITIALIZED=1 + set $TESTS + fi + + # Run test + TEST=$1 + shift + + echo + echo -------------------------------------------------------------------------------- + echo Running $TEST + + echo ">> Setting up tests" + gojira run -t $OLD_VERSION -- $BUSTED -t setup $TEST + echo ">> Running migrations" + gojira run -t $NEW_VERSION -- kong migrations up + echo ">> Testing old_after_up,all_phases" + gojira run -t $OLD_VERSION -- $BUSTED -t old_after_up,all_phases $TEST + echo ">> Testing new_after_up,all_phases" + gojira run -t $NEW_VERSION -- $BUSTED -t new_after_up,all_phases $TEST + echo ">> Finishing migrations" + gojira run -t $NEW_VERSION -- kong migrations finish + echo ">> Testing new_after_finish,all_phases" + gojira run -t $NEW_VERSION -- $BUSTED -t new_after_finish,all_phases $TEST + + if [ -z "$1" ] + then + break + fi + done +} + +if [ -z "$NO_BUILD" ] +then + build_containers +fi +run_tests + +trap "" 0 diff --git a/scripts/validate-rockspec b/scripts/validate-rockspec new file mode 100755 index 000000000000..71990c36f1f4 --- /dev/null +++ b/scripts/validate-rockspec @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -euo pipefail + +shopt -s nullglob + +fail() { + echo "Failure: $@" + exit 1 +} + +lint() { + local spec=$1 + + echo "Linting (luarocks)..." + + if ! luarocks lint "$spec"; then + fail "luarocks lint returned error" + fi + + echo "Linting (luacheck)..." + + # luacheck helps to point out some semantic issues (like duplicate + # table keys) + if ! luacheck \ + --quiet \ + --no-global \ + -- - \ + < "$spec"; + then + fail "luacheck returned error" + fi +} + +read_modules() { + local spec="$1" + resty -e ' + local fn = loadstring(io.stdin:read("*a")) + local rock = {} + setfenv(fn, rock) + fn() + + for mod, fname in pairs(rock.build.modules) do + print(fname .. "|" .. mod) + end + ' < "$spec" +} + +check_modules() { + local spec=$1 + local -A files=() + + echo "Checking modules..." + + local failed=0 + + for line in $(read_modules "$spec"); do + fname=${line%|*} + module=${line#*|} + + files[$fname]="$module" + + if [[ ! -f $fname ]]; then + : $(( failed++ )) + echo "Module ($module) file ($fname) is missing" + fi + done + + for fname in $(git ls-files 'kong/*.lua'); do + if [[ -z ${files[$fname]:-} ]]; then + : $(( failed++ )) + echo "File ($fname) not found in rockspec ($spec)" + fi + done + + if (( failed > 0 )); then + fail "rockspec build.modules is invalid" + fi +} + + +main() { + local files=(kong-*.rockspec) + local spec + + if (( ${#files[@]} == 0 )); then + fail "no rockspec file found" + + elif (( ${#files[@]} > 1 )); then + fail "multiple rockspec files found: ${files[*]}" + + else + spec=${files[0]} + fi + + echo "Found rockspec file to validate: $spec" + + lint "$spec" + + check_modules "$spec" + + echo "OK!" +} + + +main "$@"