diff --git a/.github/matrix-commitly.yml b/.github/matrix-commitly.yml index 7685340597c3..5e52cbc80f73 100644 --- a/.github/matrix-commitly.yml +++ b/.github/matrix-commitly.yml @@ -1,7 +1,7 @@ # please see matrix-full.yml for meaning of each field build-packages: - label: ubuntu-22.04 - os: ubuntu-22.04 + image: ubuntu:22.04 package: deb check-manifest-suite: ubuntu-22.04-amd64 diff --git a/.github/matrix-full.yml b/.github/matrix-full.yml index b011607f4c89..376fcac72ef7 100644 --- a/.github/matrix-full.yml +++ b/.github/matrix-full.yml @@ -12,9 +12,11 @@ build-packages: package: deb check-manifest-suite: ubuntu-20.04-amd64 - label: ubuntu-22.04 + image: ubuntu:22.04 package: deb check-manifest-suite: ubuntu-22.04-amd64 - label: ubuntu-22.04-arm64 + image: ubuntu:22.04 package: deb bazel-args: --platforms=//:generic-crossbuild-aarch64 check-manifest-suite: ubuntu-22.04-arm64 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7b8170b387ea..bc58a03990d5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -259,6 +259,7 @@ jobs: run: | echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV + echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV - name: Build & install dependencies run: | @@ -290,6 +291,7 @@ jobs: DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_TRACE_GIT_METADATA_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }} uses: Kong/gateway-test-scheduler/runner@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3 with: tests-to-run-file: test-chunk.${{ matrix.runner }}.json @@ -297,6 +299,14 @@ jobs: test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }} setup-venv-path: ${{ env.BUILD_ROOT }} + - name: Upload error logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: busted-test-errlogs-${{ matrix.runner }} + path: ${{ env.SPEC_ERRLOG_CACHE_DIR }} + retention-days: 1 + - name: Upload test rerun information if: always() uses: actions/upload-artifact@v3 @@ -361,6 +371,10 @@ jobs: $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) $CPAN_DOWNLOAD/cpanm --notest Test::Nginx + - name: Generate environment variables + run: | + echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV + - name: Tests env: TEST_SUITE: pdk @@ -372,6 +386,14 @@ jobs: eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) prove -I. -r t + - name: Upload error logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: PDK-test-errlogs + path: ${{ env.SPEC_ERRLOG_CACHE_DIR }} + retention-days: 1 + - name: Archive coverage stats file uses: actions/upload-artifact@v3 if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} diff --git a/.github/workflows/deck-integration.yml b/.github/workflows/deck-integration.yml new file mode 100644 index 000000000000..46118badd8dd --- /dev/null +++ b/.github/workflows/deck-integration.yml @@ -0,0 +1,114 @@ +name: Gateway decK Integration Tests + +on: + pull_request: + paths: + - 'kong/db/schema/**/*.lua' + - 'kong/**/schema.lua' + - 'kong/plugins/**/daos.lua' + - 'kong/db/dao/*.lua' + - 'kong/api/**/*.lua' + +permissions: + pull-requests: write + +env: + LIBRARY_PREFIX: /usr/local/kong + TEST_RESULTS_XML_OUTPUT: test-results + BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build + +# cancel previous runs if new commits are pushed to the PR +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: ./.github/workflows/build.yml + with: + relative-build-root: bazel-bin/build + + deck-integration: + name: Gateway decK integration tests + runs-on: ubuntu-22.04 + needs: build + timeout-minutes: 5 + + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: kong + POSTGRES_DB: kong + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8 + + steps: + - name: Install packages + run: sudo apt update && sudo apt install -y libyaml-dev valgrind libprotobuf-dev libpam-dev postgresql-client jq + + - name: Checkout Kong source code + uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + + - name: Lookup build cache + id: cache-deps + uses: actions/cache@v4 + with: + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build.outputs.cache-key }} + + - name: Install Kong dev + run: make dev + + - name: Tests + id: deck_tests + continue-on-error: true + env: + KONG_TEST_PG_DATABASE: kong + KONG_TEST_PG_USER: kong + KONG_TEST_DATABASE: postgres + run: | + mkdir $TEST_RESULTS_XML_OUTPUT + source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh + bin/busted spec/06-third-party/01-deck -o hjtest -Xoutput $(realpath $TEST_RESULTS_XML_OUTPUT)/report.xml -v + + - name: Find review if exists + id: find-review + uses: actions/github-script@v7 + with: + result-encoding: json + retries: 3 + script: | + const reviews = await github.paginate(github.rest.pulls.listReviews, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + + const botReview = reviews.reverse().find(review => { + return review.user.login === "github-actions[bot]" && review.body.includes("decK integration tests"); + }); + + if (botReview && botReview.state === "CHANGES_REQUESTED") { + return { "review_id": botReview.id }; + } else { + return { "review_id": "" }; + } + + - name: Request changes if failures are detected + if: ${{ fromJson(steps.find-review.outputs.result).review_id == '' && steps.deck_tests.outcome != 'success' }} + uses: actions/github-script@v7 + with: + script: | + github.rest.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + event: 'REQUEST_CHANGES', + body: `## decK integration tests\n\n:warning: failure detected. Please check [the workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details.` + }) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087a2c58bea3..70c0858bdb63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Table of Contents +- [3.7.0](#370) +- [3.6.1](#361) +- [3.5.0](#350) +- [3.4.2](#342) +- [3.4.1](#341) - [3.4.0](#340) - [3.3.0](#330) - [3.2.0](#320) @@ -10,19 +15,1084 @@ ## Unreleased -### Additions +Individual unreleased changelog entries can be located at [changelog/unreleased](changelog/unreleased). They will be assembled into [CHANGELOG.md](CHANGELOG.md) once released. -#### Core +## 3.7.0 +### Kong -#### Plugins -### Fixes +#### Performance +##### Performance -#### Core +- Improved proxy performance by refactoring internal hooking mechanism. + [#12784](https://github.com/Kong/kong/issues/12784) -#### Plugins +- Sped up the router matching when the `router_flavor` is `traditional_compatible` or `expressions`. + [#12467](https://github.com/Kong/kong/issues/12467) +##### Plugin -### Dependencies +- **Opentelemetry**: Increased queue max batch size to 200. + [#12488](https://github.com/Kong/kong/issues/12488) + +#### Breaking Changes +##### Plugin + +- **AI Proxy**: To support the new messages API of `Anthropic`, the upstream path of the `Anthropic` for `llm/v1/chat` route type has changed from `/v1/complete` to `/v1/messages`. + [#12699](https://github.com/Kong/kong/issues/12699) + + +#### Dependencies +##### Core + +- Bumped atc-router from v1.6.0 to v1.6.2 + [#12231](https://github.com/Kong/kong/issues/12231) + +- Bumped libexpat to 2.6.2 + [#12910](https://github.com/Kong/kong/issues/12910) + +- Bumped lua-kong-nginx-module from 0.8.0 to 0.11.0 + [#12752](https://github.com/Kong/kong/issues/12752) + +- Bumped lua-protobuf to 0.5.1 + [#12834](https://github.com/Kong/kong/issues/12834) + + +- Bumped lua-resty-acme to 0.13.0 + [#12909](https://github.com/Kong/kong/issues/12909) + +- Bumped lua-resty-aws from 1.3.6 to 1.4.1 + [#12846](https://github.com/Kong/kong/issues/12846) + +- Bumped lua-resty-lmdb from 1.4.1 to 1.4.2 + [#12786](https://github.com/Kong/kong/issues/12786) + + +- Bumped lua-resty-openssl from 1.2.0 to 1.3.1 + [#12665](https://github.com/Kong/kong/issues/12665) + + +- Bumped lua-resty-timer-ng to 0.2.7 + [#12756](https://github.com/Kong/kong/issues/12756) + +- Bumped PCRE from the legacy libpcre 8.45 to libpcre2 10.43 + [#12366](https://github.com/Kong/kong/issues/12366) + +- Bumped penlight to 1.14.0 + [#12862](https://github.com/Kong/kong/issues/12862) + +##### Default + +- Added package `tzdata` to DEB Docker image for convenient timezone setting. + [#12609](https://github.com/Kong/kong/issues/12609) + +- Bumped lua-resty-http to 0.17.2. + [#12908](https://github.com/Kong/kong/issues/12908) + + +- Bumped LuaRocks from 3.9.2 to 3.11.0 + [#12662](https://github.com/Kong/kong/issues/12662) + +- Bumped `ngx_wasm_module` to `91d447ffd0e9bb08f11cc69d1aa9128ec36b4526` + [#12011](https://github.com/Kong/kong/issues/12011) + + +- Bumped `V8` version to `12.0.267.17` + [#12704](https://github.com/Kong/kong/issues/12704) + + +- Bumped `Wasmtime` version to `19.0.0` + [#12011](https://github.com/Kong/kong/issues/12011) + + +- Improved the robustness of lua-cjson when handling unexpected input. + [#12904](https://github.com/Kong/kong/issues/12904) + +#### Features +##### Configuration + +- TLSv1.1 and lower versions are disabled by default in OpenSSL 3.x. + [#12420](https://github.com/Kong/kong/issues/12420) + +- Introduced `nginx_wasm_main_shm_kv` configuration parameter, which enables +Wasm filters to use the Proxy-Wasm operations `get_shared_data` and +`set_shared_data` without namespaced keys. + [#12663](https://github.com/Kong/kong/issues/12663) + + +- **Schema**: Added a deprecation field attribute to identify deprecated fields + [#12686](https://github.com/Kong/kong/issues/12686) + +- Added the `wasm_filters` configuration parameter for enabling individual filters + [#12843](https://github.com/Kong/kong/issues/12843) +##### Core + +- Added `events:ai:response_tokens`, `events:ai:prompt_tokens` and `events:ai:requests` to the anonymous report to start counting AI usage + [#12924](https://github.com/Kong/kong/issues/12924) + + +- Improved config handling when the CP runs with the router set to the `expressions` flavor: + - If mixed config is detected and a lower DP is attached to the CP, no config will be sent at all + - If the expression is invalid on the CP, no config will be sent at all + - If the expression is invalid on a lower DP, it will be sent to the DP and DP validation will catch this and communicate back to the CP (this could result in partial config application) + [#12967](https://github.com/Kong/kong/issues/12967) + +- The route entity now supports the following fields when the +`router_flavor` is `expressions`: `methods`, `hosts`, `paths`, `headers`, +`snis`, `sources`, `destinations`, and `regex_priority`. +The meaning of these fields are consistent with the traditional route entity. + [#12667](https://github.com/Kong/kong/issues/12667) +##### PDK + +- Added the `latencies.receive` property to the log serializer + [#12730](https://github.com/Kong/kong/issues/12730) +##### Plugin + +- AI Proxy now reads most prompt tuning parameters from the client, +while the plugin config parameters under `model_options` are now just defaults. +This fixes support for using the respective provider's native SDK. + [#12903](https://github.com/Kong/kong/issues/12903) + +- AI Proxy now has a `preserve` option for `route_type`, where the requests and responses +are passed directly to the upstream LLM. This is to enable compatibility with any +and all models and SDKs that may be used when calling the AI services. + [#12903](https://github.com/Kong/kong/issues/12903) + +- **Prometheus**: Added workspace label to Prometheus plugin metrics. + [#12836](https://github.com/Kong/kong/issues/12836) + +- **AI Proxy**: Added support for streaming event-by-event responses back to the client on supported providers. + [#12792](https://github.com/Kong/kong/issues/12792) + +- **AI Prompt Guard**: Increased the maximum length of regex expressions to 500 for the allow and deny parameters. + [#12731](https://github.com/Kong/kong/issues/12731) + +- Addded support for EdDSA algorithms in JWT plugin + [#12726](https://github.com/Kong/kong/issues/12726) + + +- Added support for ES512, PS256, PS384, PS512 algorithms in JWT plugin + [#12638](https://github.com/Kong/kong/issues/12638) + +- **OpenTelemetry, Zipkin**: The propagation module has been reworked. The new +options allow better control over the configuration of tracing headers propagation. + [#12670](https://github.com/Kong/kong/issues/12670) +##### Default + +- Added support for debugging with EmmyLuaDebugger. This feature is a +tech preview and not officially supported by Kong Inc. for now. + [#12899](https://github.com/Kong/kong/issues/12899) + +#### Fixes +##### CLI Command + +- Fixed an issue where the `pg_timeout` was overridden to `60s` even if `--db-timeout` +was not explicitly passed in CLI arguments. + [#12981](https://github.com/Kong/kong/issues/12981) +##### Configuration + +- Fixed the default value in kong.conf.default documentation from 1000 to 10000 +for the `upstream_keepalive_max_requests` option. + [#12643](https://github.com/Kong/kong/issues/12643) + +- Fixed an issue where an external plugin (Go, Javascript, or Python) would fail to +apply a change to the plugin config via the Admin API. + [#12718](https://github.com/Kong/kong/issues/12718) + +- Disabled usage of the Lua DNS resolver from proxy-wasm by default. + [#12825](https://github.com/Kong/kong/issues/12825) + +- Set security level of gRPC's TLS to 0 when `ssl_cipher_suite` is set to `old`. + [#12613](https://github.com/Kong/kong/issues/12613) +##### Core + +- Fixed an issue where `POST /config?flatten_errors=1` could not return a proper response if the input included duplicate upstream targets. + [#12797](https://github.com/Kong/kong/issues/12797) + +- **DNS Client**: Ignore a non-positive values on resolv.conf for options timeout, and use a default value of 2 seconds instead. + [#12640](https://github.com/Kong/kong/issues/12640) + +- Updated the file permission of `kong.logrotate` to 644. + [#12629](https://github.com/Kong/kong/issues/12629) + +- Fixed a problem on hybrid mode DPs, where a certificate entity configured with a vault reference may not get refreshed on time. + [#12868](https://github.com/Kong/kong/issues/12868) + +- Fixed the missing router section for the output of the request-debugging. + [#12234](https://github.com/Kong/kong/issues/12234) + +- Fixed an issue in the internal caching logic where mutexes could get never unlocked. + [#12743](https://github.com/Kong/kong/issues/12743) + + +- Fixed an issue where the router didn't work correctly +when the route's configuration changed. + [#12654](https://github.com/Kong/kong/issues/12654) + +- Fixed an issue where SNI-based routing didn't work +using `tls_passthrough` and the `traditional_compatible` router flavor. + [#12681](https://github.com/Kong/kong/issues/12681) + +- Fixed a bug that `X-Kong-Upstream-Status` didn't appear in the response headers even if it was set in the `headers` parameter in the `kong.conf` file when the response was hit and returned by the Proxy Cache plugin. + [#12744](https://github.com/Kong/kong/issues/12744) + +- Fixed vault initialization by postponing vault reference resolving on init_worker + [#12554](https://github.com/Kong/kong/issues/12554) + +- Fixed a bug that allowed vault secrets to refresh even when they had no TTL set. + [#12877](https://github.com/Kong/kong/issues/12877) + +- **Vault**: do not use incorrect (default) workspace identifier when retrieving vault entity by prefix + [#12572](https://github.com/Kong/kong/issues/12572) + +- **Core**: Fixed unexpected table nil panic in the balancer's stop_healthchecks function + [#12865](https://github.com/Kong/kong/issues/12865) + + +- Use `-1` as the worker ID of privileged agent to avoid access issues. + [#12385](https://github.com/Kong/kong/issues/12385) + +- **Plugin Server**: Fixed an issue where Kong failed to properly restart MessagePack-based pluginservers (used in Python and Javascript plugins, for example). + [#12582](https://github.com/Kong/kong/issues/12582) + +- Reverted the hard-coded limitation of the `ngx.read_body()` API in OpenResty upstreams' new versions when downstream connections are in HTTP/2 or HTTP/3 stream modes. + [#12658](https://github.com/Kong/kong/issues/12658) + +- Each Kong cache instance now utilizes its own cluster event channel. This approach isolates cache invalidation events and reducing the generation of unnecessary worker events. + [#12321](https://github.com/Kong/kong/issues/12321) + +- Updated telemetry collection for AI Plugins to allow multiple plugins data to be set for the same request. + [#12583](https://github.com/Kong/kong/issues/12583) +##### PDK + +- **PDK:** Fixed `kong.request.get_forwarded_port` to always return a number, +which was caused by an incorrectly stored string value in `ngx.ctx.host_port`. + [#12806](https://github.com/Kong/kong/issues/12806) + +- The value of `latencies.kong` in the log serializer payload no longer includes +the response receive time, so it now has the same value as the +`X-Kong-Proxy-Latency` response header. Response receive time is recorded in +the new `latencies.receive` metric, so if desired, the old value can be +calculated as `latencies.kong + latencies.receive`. **Note:** this also +affects payloads from all logging plugins that use the log serializer: +`file-log`, `tcp-log`, `udp-log`,`http-log`, `syslog`, and `loggly`, e.g. +[descriptions of JSON objects for the HTTP Log Plugin's log format](https://docs.konghq.com/hub/kong-inc/http-log/log-format/#json-object-descriptions). + [#12795](https://github.com/Kong/kong/issues/12795) + +- **Tracing**: enhanced robustness of trace ID parsing + [#12848](https://github.com/Kong/kong/issues/12848) +##### Plugin + +- **AI-proxy-plugin**: Fixed the bug that the `route_type` `/llm/v1/chat` didn't include the analytics in the responses. + [#12781](https://github.com/Kong/kong/issues/12781) + +- **ACME**: Fixed an issue where the certificate was not successfully renewed during ACME renewal. + [#12773](https://github.com/Kong/kong/issues/12773) + +- **AWS-Lambda**: Fixed an issue where the latency attributed to AWS Lambda API requests was counted as part of the latency in Kong. + [#12835](https://github.com/Kong/kong/issues/12835) + +- **Jwt**: Fixed an issue where the plugin would fail when using invalid public keys for ES384 and ES512 algorithms. + [#12724](https://github.com/Kong/kong/issues/12724) + + +- Added WWW-Authenticate headers to all 401 responses in the Key Auth plugin. + [#11794](https://github.com/Kong/kong/issues/11794) + +- **Opentelemetry**: Fixed an OTEL sampling mode Lua panic bug, which happened when the `http_response_header_for_traceid` option was enabled. + [#12544](https://github.com/Kong/kong/issues/12544) + +- Improve error handling in AI plugins. + [#12991](https://github.com/Kong/kong/issues/12991) + +- **ACME**: Fixed migration of redis configuration. + [#12989](https://github.com/Kong/kong/issues/12989) + +- **Response-RateLimiting**: Fixed migration of redis configuration. + [#12989](https://github.com/Kong/kong/issues/12989) + +- **Rate-Limiting**: Fixed migration of redis configuration. + [#12989](https://github.com/Kong/kong/issues/12989) +##### Admin API + +- **Admin API**: fixed an issue where calling the endpoint `POST /schemas/vaults/validate` was conflicting with the endpoint `/schemas/vaults/:name` which only has GET implemented, hence resulting in a 405. + [#12607](https://github.com/Kong/kong/issues/12607) +##### Default + +- Fixed a bug where, if the the ulimit setting (open files) was low, Kong would fail to start as the `lua-resty-timer-ng` exhausted the available `worker_connections`. Decreased the concurrency range of the `lua-resty-timer-ng` library from `[512, 2048]` to `[256, 1024]` to fix this bug. + [#12606](https://github.com/Kong/kong/issues/12606) + +- Fix an issue where external plugins using the protobuf-based protocol would fail to call the `kong.Service.SetUpstream` method with an error `bad argument #2 to 'encode' (table expected, got boolean)`. + [#12727](https://github.com/Kong/kong/issues/12727) + +### Kong-Manager + + + + + + +#### Features +##### Default + +- Kong Manager now supports creating and editing Expressions routes with an interactive in-browser editor with syntax highlighting and autocompletion features for Kong's Expressions language. + [#217](https://github.com/Kong/kong-manager/issues/217) + + +- Kong Manager now groups the parameters to provide a better user experience while configuring plugins. Meanwhile, several issues with the plugin form page were fixed. + [#195](https://github.com/Kong/kong-manager/issues/195) [#199](https://github.com/Kong/kong-manager/issues/199) [#201](https://github.com/Kong/kong-manager/issues/201) [#202](https://github.com/Kong/kong-manager/issues/202) [#207](https://github.com/Kong/kong-manager/issues/207) [#208](https://github.com/Kong/kong-manager/issues/208) [#209](https://github.com/Kong/kong-manager/issues/209) [#213](https://github.com/Kong/kong-manager/issues/213) [#216](https://github.com/Kong/kong-manager/issues/216) + + +#### Fixes +##### Default + +- Improved the user experience in Kong Manager by fixing various UI-related issues. + [#185](https://github.com/Kong/kong-manager/issues/185) [#188](https://github.com/Kong/kong-manager/issues/188) [#190](https://github.com/Kong/kong-manager/issues/190) [#195](https://github.com/Kong/kong-manager/issues/195) [#199](https://github.com/Kong/kong-manager/issues/199) [#201](https://github.com/Kong/kong-manager/issues/201) [#202](https://github.com/Kong/kong-manager/issues/202) [#207](https://github.com/Kong/kong-manager/issues/207) [#208](https://github.com/Kong/kong-manager/issues/208) [#209](https://github.com/Kong/kong-manager/issues/209) [#213](https://github.com/Kong/kong-manager/issues/213) [#216](https://github.com/Kong/kong-manager/issues/216) + +## 3.6.1 + +### Kong + + +#### Performance +##### Plugin + +- **Opentelemetry**: increase queue max batch size to 200 + [#12542](https://github.com/Kong/kong/issues/12542) + + + +#### Dependencies +##### Core + +- Bumped lua-resty-openssl to 1.2.1 + [#12669](https://github.com/Kong/kong/issues/12669) + + +#### Features +##### Configuration + +- now TLSv1.1 and lower is by default disabled in OpenSSL 3.x + [#12556](https://github.com/Kong/kong/issues/12556) + +#### Fixes +##### Configuration + +- Fixed default value in kong.conf.default documentation from 1000 to 10000 +for upstream_keepalive_max_requests option. + [#12648](https://github.com/Kong/kong/issues/12648) + +- Set security level of gRPC's TLS to 0 when ssl_cipher_suite is set to old + [#12616](https://github.com/Kong/kong/issues/12616) + +##### Core + +- Fix the missing router section for the output of the request-debugging + [#12649](https://github.com/Kong/kong/issues/12649) + +- revert the hard-coded limitation of the ngx.read_body() API in OpenResty upstreams' new versions when downstream connections are in HTTP/2 or HTTP/3 stream modes. + [#12666](https://github.com/Kong/kong/issues/12666) +##### Default + +- Fix a bug where the ulimit setting (open files) is low Kong will fail to start as the lua-resty-timer-ng exhausts the available worker_connections. Decrease the concurrency range of the lua-resty-timer-ng library from [512, 2048] to [256, 1024] to fix this bug. + [#12608](https://github.com/Kong/kong/issues/12608) +### Kong-Manager + +## 3.6.0 + +### Kong + + +#### Performance +##### Performance + +- Bumped the concurrency range of the lua-resty-timer-ng library from [32, 256] to [512, 2048]. + [#12275](https://github.com/Kong/kong/issues/12275) + +- Cooperatively yield when building statistics of routes to reduce the impact to proxy path latency. + [#12013](https://github.com/Kong/kong/issues/12013) + +##### Configuration + +- Bump `dns_stale_ttl` default to 1 hour so stale DNS record can be used for longer time in case of resolver downtime. + [#12087](https://github.com/Kong/kong/issues/12087) + +- Bumped default values of `nginx_http_keepalive_requests` and `upstream_keepalive_max_requests` to `10000`. These changes are optimized to work better in systems with high throughput. In a low-throughput setting, these new settings may have visible effects in loadbalancing - it can take more requests to start using all the upstreams than before. + [#12223](https://github.com/Kong/kong/issues/12223) +##### Core + +- Reuse match context between requests to avoid frequent memory allocation/deallocation + [#12258](https://github.com/Kong/kong/issues/12258) +##### PDK + +- Performance optimization to avoid unnecessary creations and garbage-collections of spans + [#12080](https://github.com/Kong/kong/issues/12080) + +#### Breaking Changes +##### Core + +- **BREAKING:** To avoid ambiguity with other Wasm-related nginx.conf directives, the prefix for Wasm `shm_kv` nginx.conf directives was changed from `nginx_wasm_shm_` to `nginx_wasm_shm_kv_` + [#11919](https://github.com/Kong/kong/issues/11919) + +- In OpenSSL 3.2, the default SSL/TLS security level has been changed from 1 to 2. + Which means security level set to 112 bits of security. As a result + RSA, DSA and DH keys shorter than 2048 bits and ECC keys shorter than + 224 bits are prohibited. In addition to the level 1 exclusions any cipher + suite using RC4 is also prohibited. SSL version 3 is also not allowed. + Compression is disabled. + [#7714](https://github.com/Kong/kong/issues/7714) + +##### Plugin + +- **azure-functions**: azure-functions plugin now eliminates upstream/request URI and only use `routeprefix` configuration field to construct request path when requesting Azure API + [#11850](https://github.com/Kong/kong/issues/11850) + +#### Deprecations +##### Plugin + +- **ACME**: Standardize redis configuration across plugins. The redis configuration right now follows common schema that is shared across other plugins. + [#12300](https://github.com/Kong/kong/issues/12300) + +- **Rate Limiting**: Standardize redis configuration across plugins. The redis configuration right now follows common schema that is shared across other plugins. + [#12301](https://github.com/Kong/kong/issues/12301) + +- **Response-RateLimiting**: Standardize redis configuration across plugins. The redis configuration right now follows common schema that is shared across other plugins. + [#12301](https://github.com/Kong/kong/issues/12301) + +#### Dependencies +##### Core + +- Bumped atc-router from 1.2.0 to 1.6.0 + [#12231](https://github.com/Kong/kong/issues/12231) + +- Bumped kong-lapis from 1.14.0.3 to 1.16.0.1 + [#12064](https://github.com/Kong/kong/issues/12064) + + +- Bumped LPEG from 1.0.2 to 1.1.0 + [#11955](https://github.com/Kong/kong/issues/11955) + [UTF-8](https://konghq.atlassian.net/browse/UTF-8) + +- Bumped lua-messagepack from 0.5.2 to 0.5.3 + [#11956](https://github.com/Kong/kong/issues/11956) + + +- Bumped lua-messagepack from 0.5.3 to 0.5.4 + [#12076](https://github.com/Kong/kong/issues/12076) + + +- Bumped lua-resty-aws from 1.3.5 to 1.3.6 + [#12439](https://github.com/Kong/kong/issues/12439) + + +- Bumped lua-resty-healthcheck from 3.0.0 to 3.0.1 + [#12237](https://github.com/Kong/kong/issues/12237) + +- Bumped lua-resty-lmdb from 1.3.0 to 1.4.1 + [#12026](https://github.com/Kong/kong/issues/12026) + +- Bumped lua-resty-timer-ng from 0.2.5 to 0.2.6 + [#12275](https://github.com/Kong/kong/issues/12275) + +- Bumped OpenResty from 1.21.4.2 to 1.25.3.1 + [#12327](https://github.com/Kong/kong/issues/12327) + +- Bumped OpenSSL from 3.1.4 to 3.2.1 + [#12264](https://github.com/Kong/kong/issues/12264) + +- Bump resty-openssl from 0.8.25 to 1.2.0 + [#12265](https://github.com/Kong/kong/issues/12265) + + +- Bumped ngx_brotli to master branch, and disabled it on rhel7 rhel9-arm64 and amazonlinux-2023-arm64 due to toolchain issues + [#12444](https://github.com/Kong/kong/issues/12444) + +- Bumped lua-resty-healthcheck from 1.6.3 to 3.0.0 + [#11834](https://github.com/Kong/kong/issues/11834) +##### Default + +- Bump `ngx_wasm_module` to `a7087a37f0d423707366a694630f1e09f4c21728` + [#12011](https://github.com/Kong/kong/issues/12011) + + +- Bump `Wasmtime` version to `14.0.3` + [#12011](https://github.com/Kong/kong/issues/12011) + + +#### Features +##### Configuration + +- display a warning message when Kong Manager is enabled but the Admin API is not enabled + [#12071](https://github.com/Kong/kong/issues/12071) + +- add DHE-RSA-CHACHA20-POLY1305 cipher to the intermediate configuration + [#12133](https://github.com/Kong/kong/issues/12133) + +- The default value of `dns_no_sync` option has been changed to `off` + [#11869](https://github.com/Kong/kong/issues/11869) + +- Allow to inject Nginx directives into Kong's proxy location block + [#11623](https://github.com/Kong/kong/issues/11623) + + +- Validate LMDB cache by Kong's version (major + minor), +wiping the content if tag mismatch to avoid compatibility issues +during minor version upgrade. + [#12026](https://github.com/Kong/kong/issues/12026) +##### Core + +- Adds telemetry collection for AI Proxy, AI Request Transformer, and AI Response Transformer, pertaining to model and provider usage. + [#12495](https://github.com/Kong/kong/issues/12495) + + +- add ngx_brotli module to kong prebuild nginx + [#12367](https://github.com/Kong/kong/issues/12367) + +- Allow primary key passed as a full entity to DAO functions. + [#11695](https://github.com/Kong/kong/issues/11695) + + +- Build deb packages for Debian 12. The debian variant of kong docker image is built using Debian 12 now. + [#12218](https://github.com/Kong/kong/issues/12218) + +- The expressions route now supports the `!` (not) operator, which allows creating routes like +`!(http.path =^ "/a")` and `!(http.path == "/a" || http.path == "/b")` + [#12419](https://github.com/Kong/kong/issues/12419) + +- Add `source` property to log serializer, indicating the response is generated by `kong` or `upstream`. + [#12052](https://github.com/Kong/kong/issues/12052) + +- Ensure Kong-owned directories are cleaned up after an uninstall using the system's package manager. + [#12162](https://github.com/Kong/kong/issues/12162) + +- Support `http.path.segments.len` and `http.path.segments.*` fields in the expressions router +which allows matching incoming (normalized) request path by individual segment or ranges of segments, +plus checking the total number of segments. + [#12283](https://github.com/Kong/kong/issues/12283) + +- `net.src.*` and `net.dst.*` match fields are now accessible in HTTP routes defined using expressions. + [#11950](https://github.com/Kong/kong/issues/11950) + +- Extend support for getting and setting Gateway values via proxy-wasm properties in the `kong.*` namespace. + [#11856](https://github.com/Kong/kong/issues/11856) + +##### PDK + +- Increase the precision of JSON number encoding from 14 to 16 decimals + [#12019](https://github.com/Kong/kong/issues/12019) +##### Plugin + +- Introduced the new **AI Prompt Decorator** plugin that enables prepending and appending llm/v1/chat messages onto consumer LLM requests, for prompt tuning. + [#12336](https://github.com/Kong/kong/issues/12336) + + +- Introduced the new **AI Prompt Guard** which can allow and/or block LLM requests based on pattern matching. + [#12427](https://github.com/Kong/kong/issues/12427) + + +- Introduced the new **AI Prompt Template** which can offer consumers and array of LLM prompt templates, with variable substitutions. + [#12340](https://github.com/Kong/kong/issues/12340) + + +- Introduced the new **AI Proxy** plugin that enables simplified integration with various AI provider Large Language Models. + [#12323](https://github.com/Kong/kong/issues/12323) + + +- Introduced the new **AI Request Transformer** plugin that enables passing mid-flight consumer requests to an LLM for transformation or sanitization. + [#12426](https://github.com/Kong/kong/issues/12426) + + +- Introduced the new **AI Response Transformer** plugin that enables passing mid-flight upstream responses to an LLM for transformation or sanitization. + [#12426](https://github.com/Kong/kong/issues/12426) + + +- Tracing Sampling Rate can now be set via the `config.sampling_rate` property of the OpenTelemetry plugin instead of it just being a global setting for the gateway. + [#12054](https://github.com/Kong/kong/issues/12054) +##### Admin API + +- add gateway edition to the root endpoint of the admin api + [#12097](https://github.com/Kong/kong/issues/12097) + +- Enable `status_listen` on `127.0.0.1:8007` by default + [#12304](https://github.com/Kong/kong/issues/12304) +##### Clustering + +- **Clustering**: Expose data plane certificate expiry date on the control plane API. + [#11921](https://github.com/Kong/kong/issues/11921) + +#### Fixes +##### Configuration + +- fix error data loss caused by weakly typed of function in declarative_config_flattened function + [#12167](https://github.com/Kong/kong/issues/12167) + +- respect custom `proxy_access_log` + [#12073](https://github.com/Kong/kong/issues/12073) +##### Core + +- prevent ca to be deleted when it's still referenced by other entities and invalidate the related ca store caches when a ca cert is updated. + [#11789](https://github.com/Kong/kong/issues/11789) + +- Now cookie names are validated against RFC 6265, which allows more characters than the previous validation. + [#11881](https://github.com/Kong/kong/issues/11881) + + +- Remove nulls only if the schema has transformations definitions. +Improve performance as most schemas does not define transformations. + [#12284](https://github.com/Kong/kong/issues/12284) + +- Fix a bug that the error_handler can not provide the meaningful response body when the internal error code 494 is triggered. + [#12114](https://github.com/Kong/kong/issues/12114) + +- Header value matching (`http.headers.*`) in `expressions` router flavor are now case sensitive. +This change does not affect on `traditional_compatible` mode +where header value match are always performed ignoring the case. + [#11905](https://github.com/Kong/kong/issues/11905) + +- print error message correctly when plugin fails + [#11800](https://github.com/Kong/kong/issues/11800) + +- fix ldoc intermittent failure caused by LuaJIT error. + [#11983](https://github.com/Kong/kong/issues/11983) + +- use NGX_WASM_MODULE_BRANCH environment variable to set ngx_wasm_module repository branch when building Kong. + [#12241](https://github.com/Kong/kong/issues/12241) + +- Eliminate asynchronous timer in syncQuery() to prevent hang risk + [#11900](https://github.com/Kong/kong/issues/11900) + +- **tracing:** Fixed an issue where a DNS query failure would cause a tracing failure. + [#11935](https://github.com/Kong/kong/issues/11935) + +- Expressions route in `http` and `stream` subsystem now have stricter validation. +Previously they share the same validation schema which means admin can configure expressions +route using fields like `http.path` even for stream routes. This is no longer allowed. + [#11914](https://github.com/Kong/kong/issues/11914) + +- **Tracing**: dns spans are now correctly generated for upstream dns queries (in addition to cosocket ones) + [#11996](https://github.com/Kong/kong/issues/11996) + +- Validate private and public key for `keys` entity to ensure they match each other. + [#11923](https://github.com/Kong/kong/issues/11923) + +- **proxy-wasm**: Fixed "previous plan already attached" error thrown when a filter triggers re-entrancy of the access handler. + [#12452](https://github.com/Kong/kong/issues/12452) +##### PDK + +- response.set_header support header argument with table array of string + [#12164](https://github.com/Kong/kong/issues/12164) + +- Fix an issue that when using kong.response.exit, the Transfer-Encoding header set by user is not removed + [#11936](https://github.com/Kong/kong/issues/11936) + +- **Plugin Server**: fix an issue where every request causes a new plugin instance to be created + [#12020](https://github.com/Kong/kong/issues/12020) +##### Plugin + +- Add missing WWW-Authenticate headers to 401 response in basic auth plugin. + [#11795](https://github.com/Kong/kong/issues/11795) + +- Enhance error responses for authentication failures in the Admin API + [#12456](https://github.com/Kong/kong/issues/12456) + +- Expose metrics for serviceless routes + [#11781](https://github.com/Kong/kong/issues/11781) + +- **Rate Limiting**: fix to provide better accuracy in counters when sync_rate is used with the redis policy. + [#11859](https://github.com/Kong/kong/issues/11859) + +- **Rate Limiting**: fix an issuer where all counters are synced to the same DB at the same rate. + [#12003](https://github.com/Kong/kong/issues/12003) + +- **Datadog**: Fix a bug that datadog plugin is not triggered for serviceless routes. In this fix, datadog plugin is always triggered, and the value of tag `name`(service_name) is set as an empty value. + [#12068](https://github.com/Kong/kong/issues/12068) +##### Clustering + +- Fix a bug causing data-plane status updates to fail when an empty PING frame is received from a data-plane + [#11917](https://github.com/Kong/kong/issues/11917) +### Kong-Manager + + + + + + +#### Features +##### Default + +- Added a JSON/YAML format preview for all entity forms. + [#157](https://github.com/Kong/kong-manager/issues/157) + + +- Adopted resigned basic components for better UI/UX. + [#131](https://github.com/Kong/kong-manager/issues/131) [#166](https://github.com/Kong/kong-manager/issues/166) + + +- Kong Manager and Konnect now share the same UI for plugin selection page and plugin form page. + [#143](https://github.com/Kong/kong-manager/issues/143) [#147](https://github.com/Kong/kong-manager/issues/147) + + +#### Fixes +##### Default + +- Standardized notification text format. + [#140](https://github.com/Kong/kong-manager/issues/140) + +## 3.5.0 +### Kong + + +#### Performance +##### Configuration + +- Bumped the default value of `upstream_keepalive_pool_size` to `512` and `upstream_keepalive_max_requests` to `1000` + [#11515](https://github.com/Kong/kong/issues/11515) +##### Core + +- refactor workspace id and name retrieval + [#11442](https://github.com/Kong/kong/issues/11442) + +#### Breaking Changes +##### Plugin + +- **Session**: a new configuration field `read_body_for_logout` was added with a default value of `false`, that changes behavior of `logout_post_arg` in a way that it is not anymore considered if the `read_body_for_logout` is not explicitly set to `true`. This is to avoid session plugin from reading request bodies by default on e.g. `POST` request for logout detection. + [#10333](https://github.com/Kong/kong/issues/10333) + + +#### Dependencies +##### Core + +- Bumped resty.openssl from 0.8.23 to 0.8.25 + [#11518](https://github.com/Kong/kong/issues/11518) + +- Fix incorrect LuaJIT register allocation for IR_*LOAD on ARM64 + [#11638](https://github.com/Kong/kong/issues/11638) + +- Fix LDP/STP fusing for unaligned accesses on ARM64 + [#11639](https://github.com/Kong/kong/issues/11639) + + +- Bump lua-kong-nginx-module from 0.6.0 to 0.8.0 + [#11663](https://github.com/Kong/kong/issues/11663) + +- Fix incorrect LuaJIT LDP/STP fusion on ARM64 which may sometimes cause incorrect logic + [#11537](https://github.com/Kong/kong/issues/11537) + +##### Default + +- Bumped lua-resty-healthcheck from 1.6.2 to 1.6.3 + [#11360](https://github.com/Kong/kong/issues/11360) + +- Bumped OpenResty from 1.21.4.1 to 1.21.4.2 + [#11360](https://github.com/Kong/kong/issues/11360) + +- Bumped LuaSec from 1.3.1 to 1.3.2 + [#11553](https://github.com/Kong/kong/issues/11553) + + +- Bumped lua-resty-aws from 1.3.1 to 1.3.5 + [#11613](https://github.com/Kong/kong/issues/11613) + + +- bump OpenSSL from 3.1.1 to 3.1.4 + [#11844](https://github.com/Kong/kong/issues/11844) + + +- Bumped kong-lapis from 1.14.0.2 to 1.14.0.3 + [#11849](https://github.com/Kong/kong/issues/11849) + + +- Bumped ngx_wasm_module to latest rolling release version. + [#11678](https://github.com/Kong/kong/issues/11678) + +- Bump Wasmtime version to 12.0.2 + [#11738](https://github.com/Kong/kong/issues/11738) + +- Bumped lua-resty-aws from 1.3.0 to 1.3.1 + [#11419](https://github.com/Kong/kong/pull/11419) + +- Bumped lua-resty-session from 4.0.4 to 4.0.5 + [#11416](https://github.com/Kong/kong/pull/11416) + + +#### Features +##### Core + +- Add a new endpoint `/schemas/vaults/:name` to retrieve the schema of a vault. + [#11727](https://github.com/Kong/kong/issues/11727) + +- rename `privileged_agent` to `dedicated_config_processing. Enable `dedicated_config_processing` by default + [#11784](https://github.com/Kong/kong/issues/11784) + +- Support observing the time consumed by some components in the given request. + [#11627](https://github.com/Kong/kong/issues/11627) + +- Plugins can now implement `Plugin:configure(configs)` function that is called whenever there is a change in plugin entities. An array of current plugin configurations is passed to the function, or `nil` in case there is no active configurations for the plugin. + [#11703](https://github.com/Kong/kong/issues/11703) + +- Add a request-aware table able to detect accesses from different requests. + [#11017](https://github.com/Kong/kong/issues/11017) + +- A unique Request ID is now populated in the error log, access log, error templates, log serializer, and in a new X-Kong-Request-Id header (configurable for upstream/downstream using the `headers` and `headers_upstream` configuration options). + [#11663](https://github.com/Kong/kong/issues/11663) + +- Add support for optional Wasm filter configuration schemas + [#11568](https://github.com/Kong/kong/issues/11568) + +- Support JSON in Wasm filter configuration + [#11697](https://github.com/Kong/kong/issues/11697) + +- Support HTTP query parameters in expression routes. + [#11348](https://github.com/Kong/kong/pull/11348) + +##### Plugin + +- **response-ratelimiting**: add support for secret rotation with redis connection + [#10570](https://github.com/Kong/kong/issues/10570) + + +- **CORS**: Support the `Access-Control-Request-Private-Network` header in crossing-origin pre-light requests + [#11523](https://github.com/Kong/kong/issues/11523) + +- add scan_count to redis storage schema + [#11532](https://github.com/Kong/kong/issues/11532) + + +- **AWS-Lambda**: the AWS-Lambda plugin has been refactored by using `lua-resty-aws` as an + underlying AWS library. The refactor simplifies the AWS-Lambda plugin code base and + adding support for multiple IAM authenticating scenarios. + [#11350](https://github.com/Kong/kong/pull/11350) + +- **OpenTelemetry** and **Zipkin**: Support GCP X-Cloud-Trace-Context header + The field `header_type` now accepts the value `gcp` to propagate the + Google Cloud trace header + [#11254](https://github.com/Kong/kong/pull/11254) + +##### Clustering + +- **Clustering**: Allow configuring DP metadata labels for on-premise CP Gateway + [#11625](https://github.com/Kong/kong/issues/11625) + +#### Fixes +##### Configuration + +- The default value of `dns_no_sync` option has been changed to `on` + [#11871](https://github.com/Kong/kong/issues/11871) + +##### Core + +- Fix an issue that the TTL of the key-auth plugin didnt work in DB-less and Hybrid mode. + [#11464](https://github.com/Kong/kong/issues/11464) + +- Fix a problem that abnormal socket connection will be reused when querying Postgres database. + [#11480](https://github.com/Kong/kong/issues/11480) + +- Fix upstream ssl failure when plugins use response handler + [#11502](https://github.com/Kong/kong/issues/11502) + +- Fix an issue that protocol `tls_passthrough` can not work with expressions flavor + [#11538](https://github.com/Kong/kong/issues/11538) + +- Fix a bug that will cause a failure of sending tracing data to datadog when value of x-datadog-parent-id header in requests is a short dec string + [#11599](https://github.com/Kong/kong/issues/11599) + +- Apply Nginx patch for detecting HTTP/2 stream reset attacks early (CVE-2023-44487) + [#11743](https://github.com/Kong/kong/issues/11743) + +- fix the building failure when applying patches + [#11696](https://github.com/Kong/kong/issues/11696) + +- Vault references can be used in Dbless mode in declarative config + [#11845](https://github.com/Kong/kong/issues/11845) + + +- Properly warmup Vault caches on init + [#11827](https://github.com/Kong/kong/issues/11827) + + +- Vault resurrect time is respected in case a vault secret is deleted from a vault + [#11852](https://github.com/Kong/kong/issues/11852) + +- Fixed critical level logs when starting external plugin servers. Those logs cannot be suppressed due to the limitation of OpenResty. We choose to remove the socket availability detection feature. + [#11372](https://github.com/Kong/kong/pull/11372) + +- Fix an issue where a crashing Go plugin server process would cause subsequent + requests proxied through Kong to execute Go plugins with inconsistent configurations. + The issue only affects scenarios where the same Go plugin is applied to different Route + or Service entities. + [#11306](https://github.com/Kong/kong/pull/11306) + +- Fix an issue where cluster_cert or cluster_ca_cert is inserted into lua_ssl_trusted_certificate before being base64 decoded. + [#11385](https://github.com/Kong/kong/pull/11385) + +- Fix cache warmup mechanism not working in `acls` plugin groups config entity scenario. + [#11414](https://github.com/Kong/kong/pull/11414) + +- Fix an issue that queue stops processing when a hard error is encountered in the handler function. + [#11423](https://github.com/Kong/kong/pull/11423) + +- Fix an issue that query parameters are not forwarded in proxied request. + Thanks [@chirag-manwani](https://github.com/chirag-manwani) for contributing this change. + [#11328](https://github.com/Kong/kong/pull/11328) + +- Fix an issue that response status code is not real upstream status when using kong.response function. + [#11437](https://github.com/Kong/kong/pull/11437) + +- Removed a hardcoded proxy-wasm isolation level setting that was preventing the + `nginx_http_proxy_wasm_isolation` configuration value from taking effect. + [#11407](https://github.com/Kong/kong/pull/11407) + +##### PDK + +- Fix several issues in Vault and refactor the Vault code base: - Make DAOs to fallback to empty string when resolving Vault references fail - Use node level mutex when rotation references - Refresh references on config changes - Update plugin referenced values only once per request - Pass only the valid config options to vault implementations - Resolve multi-value secrets only once when rotating them - Do not start vault secrets rotation timer on control planes - Re-enable negative caching - Reimplement the kong.vault.try function - Remove references from rotation in case their configuration has changed + [#11652](https://github.com/Kong/kong/issues/11652) + +- Fix response body gets repeated when `kong.response.get_raw_body()` is called multiple times in a request lifecycle. + [#11424](https://github.com/Kong/kong/issues/11424) + +- Tracing: fix an issue that resulted in some parent spans to end before their children due to different precision of their timestamps + [#11484](https://github.com/Kong/kong/issues/11484) + +- Fix a bug related to data interference between requests in the kong.log.serialize function. + [#11566](https://github.com/Kong/kong/issues/11566) +##### Plugin + +- **Opentelemetry**: fix an issue that resulted in invalid parent IDs in the propagated tracing headers + [#11468](https://github.com/Kong/kong/issues/11468) + +- **AWS-Lambda**: let plugin-level proxy take effect on EKS IRSA credential provider + [#11551](https://github.com/Kong/kong/issues/11551) + +- Cache the AWS lambda service by those lambda service related fields + [#11821](https://github.com/Kong/kong/issues/11821) + +- **Opentelemetry**: fix an issue that resulted in traces with invalid parent IDs when `balancer` instrumentation was enabled + [#11830](https://github.com/Kong/kong/issues/11830) + + +- **tcp-log**: fix an issue of unnecessary handshakes when reusing TLS connection + [#11848](https://github.com/Kong/kong/issues/11848) + +- **OAuth2**: For OAuth2 plugin, `scope` has been taken into account as a new criterion of the request validation. When refreshing token with `refresh_token`, the scopes associated with the `refresh_token` provided in the request must be same with or a subset of the scopes configured in the OAuth2 plugin instance hit by the request. + [#11342](https://github.com/Kong/kong/pull/11342) + +- When the worker is in shutdown mode and more data is immediately available without waiting for `max_coalescing_delay`, queues are now cleared in batches. + Thanks [@JensErat](https://github.com/JensErat) for contributing this change. + [#11376](https://github.com/Kong/kong/pull/11376) + +- A race condition in the plugin queue could potentially crash the worker when `max_entries` was set to `max_batch_size`. + [#11378](https://github.com/Kong/kong/pull/11378) + +- **AWS-Lambda**: fix an issue that the AWS-Lambda plugin cannot extract a json encoded proxy integration response. + [#11413](https://github.com/Kong/kong/pull/11413) + +##### Default + +- Restore lapis & luarocks-admin bins + [#11578](https://github.com/Kong/kong/issues/11578) +### Kong-Manager + + + + + + +#### Features +##### Default + +- Add `JSON` and `YAML` formats in entity config cards. + [#111](https://github.com/Kong/kong-manager/issues/111) + + +- Plugin form fields now display descriptions from backend schema. + [#66](https://github.com/Kong/kong-manager/issues/66) + + +- Add the `protocols` field in plugin form. + [#93](https://github.com/Kong/kong-manager/issues/93) + + +- The upstream target list shows the `Mark Healthy` and `Mark Unhealthy` action items when certain conditions are met. + [#86](https://github.com/Kong/kong-manager/issues/86) + + +#### Fixes +##### Default + +- Fix incorrect port number in Port Details. + [#103](https://github.com/Kong/kong-manager/issues/103) + + +- Fix a bug where the `proxy-cache` plugin cannot be installed. + [#104](https://github.com/Kong/kong-manager/issues/104) + +## 3.4.2 + +### Kong + +#### Fixes +##### Core + +- Apply Nginx patch for detecting HTTP/2 stream reset attacks early (CVE-2023-44487) + [#11743](https://github.com/Kong/kong/issues/11743) + [CVE-2023](https://konghq.atlassian.net/browse/CVE-2023) [nginx-1](https://konghq.atlassian.net/browse/nginx-1) [SIR-435](https://konghq.atlassian.net/browse/SIR-435) + +## 3.4.1 + +### Kong + + +#### Additions + +##### Core + +- Support HTTP query parameters in expression routes. + [#11348](https://github.com/Kong/kong/pull/11348) + + +#### Dependencies + +##### Core + +- Fix incorrect LuaJIT LDP/STP fusion on ARM64 which may sometimes cause incorrect logic + [#11537](https://github.com/Kong/kong-ee/issues/11537) + + + +#### Fixes + +##### Core + +- Removed a hardcoded proxy-wasm isolation level setting that was preventing the + `nginx_http_proxy_wasm_isolation` configuration value from taking effect. + [#11407](https://github.com/Kong/kong/pull/11407) +- Fix an issue that the TTL of the key-auth plugin didnt work in DB-less and Hybrid mode. + [#11464](https://github.com/Kong/kong-ee/issues/11464) +- Fix a problem that abnormal socket connection will be reused when querying Postgres database. + [#11480](https://github.com/Kong/kong-ee/issues/11480) +- Fix upstream ssl failure when plugins use response handler + [#11502](https://github.com/Kong/kong-ee/issues/11502) +- Fix an issue that protocol `tls_passthrough` can not work with expressions flavor + [#11538](https://github.com/Kong/kong-ee/issues/11538) + +##### PDK + +- Fix several issues in Vault and refactor the Vault code base: - Make DAOs to fallback to empty string when resolving Vault references fail - Use node level mutex when rotation references - Refresh references on config changes - Update plugin referenced values only once per request - Pass only the valid config options to vault implementations - Resolve multi-value secrets only once when rotating them - Do not start vault secrets rotation timer on control planes - Re-enable negative caching - Reimplement the kong.vault.try function - Remove references from rotation in case their configuration has changed + +[#11402](https://github.com/Kong/kong-ee/issues/11402) +- Tracing: fix an issue that resulted in some parent spans to end before their children due to different precision of their timestamps + [#11484](https://github.com/Kong/kong-ee/issues/11484) + +##### Plugin + +- **Opentelemetry**: fix an issue that resulted in invalid parent IDs in the propagated tracing headers + [#11468](https://github.com/Kong/kong-ee/issues/11468) + +### Kong Manager + +#### Fixes + +- Fixed entity docs link. + [#92](https://github.com/Kong/kong-manager/pull/92) ## 3.4.0 diff --git a/build/openresty/BUILD.openresty.bazel b/build/openresty/BUILD.openresty.bazel index 9b86a74a25e5..4b6aa4292b10 100644 --- a/build/openresty/BUILD.openresty.bazel +++ b/build/openresty/BUILD.openresty.bazel @@ -293,7 +293,7 @@ configure_make( ], out_lib_dir = "", out_shared_libs = select({ - "@kong//:wasmx_flag": [ + "@kong//:wasmx_dynamic_mod": [ "nginx/modules/ngx_wasmx_module.so", ], "//conditions:default": [], diff --git a/changelog/unreleased/kong/hmac_www_authenticate.yml b/changelog/unreleased/kong/hmac_www_authenticate.yml new file mode 100644 index 000000000000..23e0e20ab91c --- /dev/null +++ b/changelog/unreleased/kong/hmac_www_authenticate.yml @@ -0,0 +1,3 @@ +message: "**hmac-auth**: Add WWW-Authenticate headers to 401 responses." +type: bugfix +scope: Plugin diff --git a/changelog/unreleased/kong/ldap_www_authenticate.yml b/changelog/unreleased/kong/ldap_www_authenticate.yml new file mode 100644 index 000000000000..bd1fbe096d9b --- /dev/null +++ b/changelog/unreleased/kong/ldap_www_authenticate.yml @@ -0,0 +1,3 @@ +message: "**ldap-auth**: Add WWW-Authenticate headers to all 401 responses." +type: bugfix +scope: Plugin diff --git a/kong-3.8.0-0.rockspec b/kong-3.8.0-0.rockspec index bec67ea7cce8..09613b9220c6 100644 --- a/kong-3.8.0-0.rockspec +++ b/kong-3.8.0-0.rockspec @@ -318,6 +318,7 @@ build = { ["kong.pdk.private.checks"] = "kong/pdk/private/checks.lua", ["kong.pdk.private.phases"] = "kong/pdk/private/phases.lua", ["kong.pdk.private.node"] = "kong/pdk/private/node.lua", + ["kong.pdk.private.rate_limiting"] = "kong/pdk/private/rate_limiting.lua", ["kong.pdk.client"] = "kong/pdk/client.lua", ["kong.pdk.client.tls"] = "kong/pdk/client/tls.lua", ["kong.pdk.ctx"] = "kong/pdk/ctx.lua", diff --git a/kong/clustering/compat/removed_fields.lua b/kong/clustering/compat/removed_fields.lua index 359d9b763797..ef370447bc9f 100644 --- a/kong/clustering/compat/removed_fields.lua +++ b/kong/clustering/compat/removed_fields.lua @@ -150,6 +150,12 @@ return { }, aws_lambda = { "empty_arrays_mode", - } + }, + ldap_auth = { + "realm", + }, + hmac_auth = { + "realm", + }, }, } diff --git a/kong/clustering/data_plane.lua b/kong/clustering/data_plane.lua index 177344bdacea..f6621c2a2341 100644 --- a/kong/clustering/data_plane.lua +++ b/kong/clustering/data_plane.lua @@ -8,7 +8,6 @@ local config_helper = require("kong.clustering.config_helper") local clustering_utils = require("kong.clustering.utils") local declarative = require("kong.db.declarative") local constants = require("kong.constants") -local pl_stringx = require("pl.stringx") local inspect = require("inspect") local assert = assert @@ -37,7 +36,7 @@ local PING_WAIT = PING_INTERVAL * 1.5 local _log_prefix = "[clustering] " local DECLARATIVE_EMPTY_CONFIG_HASH = constants.DECLARATIVE_EMPTY_CONFIG_HASH -local endswith = pl_stringx.endswith +local endswith = require("pl.stringx").endswith local function is_timeout(err) return err and sub(err, -7) == "timeout" diff --git a/kong/clustering/rpc/utils.lua b/kong/clustering/rpc/utils.lua index 544d2892932f..073d8a707694 100644 --- a/kong/clustering/rpc/utils.lua +++ b/kong/clustering/rpc/utils.lua @@ -1,5 +1,4 @@ local _M = {} -local pl_stringx = require("pl.stringx") local cjson = require("cjson") local snappy = require("resty.snappy") @@ -8,7 +7,7 @@ local string_sub = string.sub local assert = assert local cjson_encode = cjson.encode local cjson_decode = cjson.decode -local rfind = pl_stringx.rfind +local rfind = require("pl.stringx").rfind local snappy_compress = snappy.compress local snappy_uncompress = snappy.uncompress diff --git a/kong/cmd/health.lua b/kong/cmd/health.lua index ba8d37c758f2..3b8cee18b692 100644 --- a/kong/cmd/health.lua +++ b/kong/cmd/health.lua @@ -2,9 +2,10 @@ local log = require "kong.cmd.utils.log" local kill = require "kong.cmd.utils.kill" local pl_path = require "pl.path" local pl_tablex = require "pl.tablex" -local pl_stringx = require "pl.stringx" local conf_loader = require "kong.conf_loader" +local ljust = require("pl.stringx").ljust + local function execute(args) log.disable() -- retrieve default prefix or use given one @@ -27,7 +28,7 @@ local function execute(args) local count = 0 for k, v in pairs(pids) do local running = kill.is_running(v) - local msg = pl_stringx.ljust(k, 12, ".") .. (running and "running" or "not running") + local msg = ljust(k, 12, ".") .. (running and "running" or "not running") if running then count = count + 1 end diff --git a/kong/cmd/utils/inject_confs.lua b/kong/cmd/utils/inject_confs.lua index 9249d5c70e8f..90a478df3e4a 100644 --- a/kong/cmd/utils/inject_confs.lua +++ b/kong/cmd/utils/inject_confs.lua @@ -1,8 +1,8 @@ local conf_loader = require "kong.conf_loader" local pl_path = require "pl.path" -local pl_stringx = require "pl.stringx" local prefix_handler = require "kong.cmd.utils.prefix_handler" local log = require "kong.cmd.utils.log" +local strip = require("kong.tools.string").strip local fmt = string.format local compile_nginx_main_inject_conf = prefix_handler.compile_nginx_main_inject_conf @@ -42,7 +42,7 @@ local function convert_directive_path_to_absolute(prefix, nginx_conf, paths) return nil, err elseif m then - local path = pl_stringx.strip(m[2]) + local path = strip(m[2]) if path:sub(1, 1) ~= '/' then local absolute_path = prefix .. "/" .. path diff --git a/kong/cmd/utils/nginx_signals.lua b/kong/cmd/utils/nginx_signals.lua index fb9065466eb9..80f3e8643a1e 100644 --- a/kong/cmd/utils/nginx_signals.lua +++ b/kong/cmd/utils/nginx_signals.lua @@ -5,10 +5,12 @@ local meta = require "kong.meta" local pl_path = require "pl.path" local version = require "version" local pl_utils = require "pl.utils" -local pl_stringx = require "pl.stringx" local process_secrets = require "kong.cmd.utils.process_secrets" +local strip = require("kong.tools.string").strip + + local fmt = string.format local ipairs = ipairs local unpack = unpack @@ -115,7 +117,7 @@ function _M.find_nginx_bin(kong_conf) log.debug("finding executable absolute path from $PATH...") local ok, code, stdout, stderr = pl_utils.executeex("command -v nginx") if ok and code == 0 then - path_to_check = pl_stringx.strip(stdout) + path_to_check = strip(stdout) else log.error("could not find executable absolute path: %s", stderr) diff --git a/kong/cmd/utils/prefix_handler.lua b/kong/cmd/utils/prefix_handler.lua index 14ca40f81a1f..b1e6557f4ac5 100644 --- a/kong/cmd/utils/prefix_handler.lua +++ b/kong/cmd/utils/prefix_handler.lua @@ -15,7 +15,6 @@ local x509 = require "resty.openssl.x509" local x509_extension = require "resty.openssl.x509.extension" local x509_name = require "resty.openssl.x509.name" local pl_template = require "pl.template" -local pl_stringx = require "pl.stringx" local pl_tablex = require "pl.tablex" local pl_utils = require "pl.utils" local pl_file = require "pl.file" @@ -27,6 +26,10 @@ local bit = require "bit" local nginx_signals = require "kong.cmd.utils.nginx_signals" +local strip = require("kong.tools.string").strip +local split = require("kong.tools.string").split + + local getmetatable = getmetatable local makepath = pl_dir.makepath local tonumber = tonumber @@ -229,7 +232,7 @@ local function get_ulimit() if not ok then return nil, stderr end - local sanitized_limit = pl_stringx.strip(stdout) + local sanitized_limit = strip(stdout) if sanitized_limit:lower():match("unlimited") then return 65536 else @@ -725,7 +728,7 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ end local template_env = {} - nginx_conf_flags = nginx_conf_flags and pl_stringx.split(nginx_conf_flags, ",") or {} + nginx_conf_flags = nginx_conf_flags and split(nginx_conf_flags, ",") or {} for _, flag in ipairs(nginx_conf_flags) do template_env[flag] = true end diff --git a/kong/conf_loader/init.lua b/kong/conf_loader/init.lua index 40f574e43f66..13b908dc4c3c 100644 --- a/kong/conf_loader/init.lua +++ b/kong/conf_loader/init.lua @@ -4,7 +4,6 @@ local require = require local kong_default_conf = require "kong.templates.kong_defaults" local process_secrets = require "kong.cmd.utils.process_secrets" local pl_stringio = require "pl.stringio" -local pl_stringx = require "pl.stringx" local socket_url = require "socket.url" local conf_constants = require "kong.conf_loader.constants" local listeners = require "kong.conf_loader.listeners" @@ -16,18 +15,20 @@ local pl_config = require "pl.config" local pl_file = require "pl.file" local pl_path = require "pl.path" local tablex = require "pl.tablex" -local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy local log = require "kong.cmd.utils.log" local env = require "kong.cmd.utils.env" +local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy +local strip = require("kong.tools.string").strip + + local fmt = string.format local sub = string.sub local type = type local sort = table.sort local find = string.find local gsub = string.gsub -local strip = pl_stringx.strip local lower = string.lower local match = string.match local pairs = pairs diff --git a/kong/conf_loader/listeners.lua b/kong/conf_loader/listeners.lua index 7e638b3618e5..1f17a33e8fbf 100644 --- a/kong/conf_loader/listeners.lua +++ b/kong/conf_loader/listeners.lua @@ -1,5 +1,5 @@ -local pl_stringx = require "pl.stringx" local tools_ip = require "kong.tools.ip" +local strip = require("kong.tools.string").strip local type = type @@ -73,7 +73,7 @@ local function parse_option_flags(value, flags) end end - return pl_stringx.strip(value), result, pl_stringx.strip(sanitized) + return strip(value), result, strip(sanitized) end @@ -98,7 +98,7 @@ local function parse_listeners(values, flags) return nil, usage end - if pl_stringx.strip(values[1]) == "off" then + if strip(values[1]) == "off" then return list end diff --git a/kong/conf_loader/parse.lua b/kong/conf_loader/parse.lua index eeeac125c25d..5be3cadb2cab 100644 --- a/kong/conf_loader/parse.lua +++ b/kong/conf_loader/parse.lua @@ -1,7 +1,6 @@ local require = require -local pl_stringx = require "pl.stringx" local pl_path = require "pl.path" local socket_url = require "socket.url" local tablex = require "pl.tablex" @@ -10,15 +9,16 @@ local openssl_pkey = require "resty.openssl.pkey" local log = require "kong.cmd.utils.log" local nginx_signals = require "kong.cmd.utils.nginx_signals" local conf_constants = require "kong.conf_loader.constants" - - -local tools_system = require("kong.tools.system") -- for unit-testing -local tools_ip = require("kong.tools.ip") +local tools_system = require "kong.tools.system" -- for unit-testing +local tools_ip = require "kong.tools.ip" +local tools_string = require "kong.tools.string" local normalize_ip = tools_ip.normalize_ip local is_valid_ip_or_cidr = tools_ip.is_valid_ip_or_cidr -local try_decode_base64 = require("kong.tools.string").try_decode_base64 +local try_decode_base64 = tools_string.try_decode_base64 +local strip = tools_string.strip +local split = tools_string.split local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy local is_valid_uuid = require("kong.tools.uuid").is_valid_uuid @@ -40,7 +40,6 @@ local insert = table.insert local concat = table.concat local getenv = os.getenv local re_match = ngx.re.match -local strip = pl_stringx.strip local exists = pl_path.exists local isdir = pl_path.isdir @@ -93,7 +92,7 @@ local function parse_value(value, typ) -- must check type because pl will already convert comma -- separated strings to tables (but not when the arr has -- only one element) - value = setmetatable(pl_stringx.split(value, ","), nil) -- remove List mt + value = setmetatable(split(value, ","), nil) -- remove List mt for i = 1, #value do value[i] = strip(value[i]) @@ -232,7 +231,7 @@ local function check_and_parse(conf, opts) elseif v_schema.enum and not tablex.find(v_schema.enum, value) then errors[#errors + 1] = fmt("%s has an invalid value: '%s' (%s)", k, - tostring(value), concat(v_schema.enum, ", ")) + tostring(value), concat(v_schema.enum, ", ")) end @@ -444,7 +443,7 @@ local function check_and_parse(conf, opts) "nginx_stream_lua_ssl_conf_command"}) do if conf[key] then - local _, _, seclevel = string.find(conf[key], "@SECLEVEL=(%d+)") + local _, _, seclevel = find(conf[key], "@SECLEVEL=(%d+)") if seclevel ~= "0" then ngx.log(ngx.WARN, key, ": Default @SECLEVEL=0 overridden, TLSv1.1 unavailable") end diff --git a/kong/db/strategies/postgres/connector.lua b/kong/db/strategies/postgres/connector.lua index 12559403ef58..94e0f9ee0215 100644 --- a/kong/db/strategies/postgres/connector.lua +++ b/kong/db/strategies/postgres/connector.lua @@ -1,11 +1,10 @@ local logger = require "kong.cmd.utils.log" local pgmoon = require "pgmoon" local arrays = require "pgmoon.arrays" -local stringx = require "pl.stringx" local semaphore = require "ngx.semaphore" -local kong_global = require "kong.global" -local constants = require "kong.constants" -local db_utils = require "kong.db.utils" +local kong_global = require "kong.global" +local constants = require "kong.constants" +local db_utils = require "kong.db.utils" local setmetatable = setmetatable @@ -31,6 +30,7 @@ local sub = string.sub local utils_toposort = db_utils.topological_sort local insert = table.insert local table_merge = require("kong.tools.table").table_merge +local strip = require("kong.tools.string").strip local WARN = ngx.WARN @@ -867,7 +867,7 @@ function _mt:run_up_migration(name, up_sql) error("no connection") end - local sql = stringx.strip(up_sql) + local sql = strip(up_sql) if sub(sql, -1) ~= ";" then sql = sql .. ";" end diff --git a/kong/pdk/log.lua b/kong/pdk/log.lua index 2c3dabc1e9fe..6f9c07c56569 100644 --- a/kong/pdk/log.lua +++ b/kong/pdk/log.lua @@ -16,14 +16,18 @@ local ngx_re = require("ngx.re") local inspect = require("inspect") local phase_checker = require("kong.pdk.private.phases") local constants = require("kong.constants") +local clear_tab = require("table.clear") + local request_id_get = require("kong.tracing.request_id").get local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy local get_tls1_version_str = require("ngx.ssl").get_tls1_version_str local get_workspace_name = require("kong.workspaces").get_workspace_name + local sub = string.sub local type = type +local error = error local pairs = pairs local ipairs = ipairs local find = string.find @@ -38,11 +42,12 @@ local setmetatable = setmetatable local ngx = ngx local kong = kong local check_phase = phase_checker.check -local split = require("kong.tools.string").split local byte = string.byte -local EMPTY_TAB = require("pl.tablex").readonly({}) +local DOT_BYTE = byte(".") + + local _PREFIX = "[kong] " local _DEFAULT_FORMAT = "%file_src:%line_src %message" local _DEFAULT_NAMESPACED_FORMAT = "%file_src:%line_src [%namespace] %message" @@ -552,17 +557,6 @@ local _log_mt = { } -local function get_default_serialize_values() - if ngx.config.subsystem == "http" then - return { - { key = "request.headers.authorization", value = "REDACTED", mode = "replace" }, - { key = "request.headers.proxy-authorization", value = "REDACTED", mode = "replace" }, - } - end - - return {} -end - --- -- Sets a value to be used on the `serialize` custom table. -- @@ -606,27 +600,36 @@ end local function set_serialize_value(key, value, options) check_phase(phases_with_ctx) - options = options or {} - local mode = options.mode or "set" - if type(key) ~= "string" then error("key must be a string", 2) end + local mode = options and options.mode or "set" if mode ~= "set" and mode ~= "add" and mode ~= "replace" then error("mode must be 'set', 'add' or 'replace'", 2) end - local ongx = options.ngx or ngx + local data = { + key = key, + value = value, + mode = mode, + } + + local ongx = options and options.ngx or ngx local ctx = ongx.ctx - ctx.serialize_values = ctx.serialize_values or get_default_serialize_values() - ctx.serialize_values[#ctx.serialize_values + 1] = - { key = key, value = value, mode = mode } + local serialize_values = ctx.serialize_values + if serialize_values then + serialize_values[#serialize_values + 1] = data + else + ctx.serialize_values = { data } + end end local serialize do + local VISITED = {} + local function is_valid_value(v, visited) local t = type(v) if v == nil or t == "number" or t == "string" or t == "boolean" then @@ -637,17 +640,20 @@ do return false end - if visited[v] then + if not visited then + clear_tab(VISITED) + visited = VISITED + + elseif visited[v] then return false end + visited[v] = true for k, val in pairs(v) do t = type(k) - if (t ~= "string" - and t ~= "number" - and t ~= "boolean") - or not is_valid_value(val, visited) + if (t ~= "string" and t ~= "number" and t ~= "boolean") + or not is_valid_value(val, visited) then return false end @@ -656,43 +662,47 @@ do return true end + -- Modify returned table with values set with kong.log.set_serialize_values - local function edit_result(ctx, root) - local serialize_values = ctx.serialize_values or get_default_serialize_values() - local key, mode, new_value, subkeys, node, subkey, last_subkey, existing_value + local function edit_result(root, serialize_values) for _, item in ipairs(serialize_values) do - key, mode, new_value = item.key, item.mode, item.value - - if not is_valid_value(new_value, {}) then - error("value must be nil, a number, string, boolean or a non-self-referencial table containing numbers, string and booleans", 2) + local new_value = item.value + if not is_valid_value(new_value) then + error("value must be nil, a number, string, boolean or a non-self-referencial table containing numbers, string and booleans", 3) end - -- Split key by ., creating subtables when needed - subkeys = setmetatable(split(key, "."), nil) - node = root -- start in root, iterate with each subkey - for i = 1, #subkeys - 1 do -- note that last subkey is treated differently, below - subkey = subkeys[i] - if node[subkey] == nil then - if mode == "set" or mode == "add" then - node[subkey] = {} -- add subtables as needed - else - node = nil - break -- mode == replace; and we have a missing link on the "chain" + -- Split key by ., creating sub-tables when needed + local key = item.key + local mode = item.mode + local is_set_or_add = mode == "set" or mode == "add" + local node = root + local start = 1 + for i = 2, #key do + if byte(key, i) == DOT_BYTE then + local subkey = sub(key, start, i - 1) + start = i + 1 + if node[subkey] == nil then + if is_set_or_add then + node[subkey] = {} -- add sub-tables as needed + else + node = nil + break -- mode == replace; and we have a missing link on the "chain" + end + + elseif type(node[subkey]) ~= "table" then + error("The key '" .. key .. "' could not be used as a serialize value. " .. + "Subkey '" .. subkey .. "' is not a table. It's " .. tostring(node[subkey])) end - end - if type(node[subkey]) ~= "table" then - error("The key '" .. key .. "' could not be used as a serialize value. " .. - "Subkey '" .. subkey .. "' is not a table. It's " .. tostring(node[subkey])) + node = node[subkey] end - - node = node[subkey] end + if type(node) == "table" then - last_subkey = subkeys[#subkeys] - existing_value = node[last_subkey] + local last_subkey = sub(key, start) + local existing_value = node[last_subkey] if (mode == "set") - or (mode == "add" and existing_value == nil) + or (mode == "add" and existing_value == nil) or (mode == "replace" and existing_value ~= nil) then node[last_subkey] = new_value @@ -704,29 +714,32 @@ do end local function build_authenticated_entity(ctx) - local authenticated_entity - if ctx.authenticated_credential ~= nil then - authenticated_entity = { - id = ctx.authenticated_credential.id, - consumer_id = ctx.authenticated_credential.consumer_id, + local credential = ctx.authenticated_credential + if credential ~= nil then + local consumer_id = credential.consumer_id + if not consumer_id then + local consumer = ctx.authenticate_consumer + if consumer ~= nil then + consumer_id = consumer.id + end + end + + return { + id = credential.id, + consumer_id = consumer_id, } end - - return authenticated_entity end local function build_tls_info(var, override) - local tls_info local tls_info_ver = get_tls1_version_str() if tls_info_ver then - tls_info = { + return { version = tls_info_ver, cipher = var.ssl_cipher, client_verify = override or var.ssl_client_verify, } end - - return tls_info end local function to_decimal(str) @@ -796,42 +809,41 @@ do function serialize(options) check_phase(PHASES_LOG) - options = options or EMPTY_TAB - local ongx = options.ngx or ngx - local okong = options.kong or kong + local ongx = options and options.ngx or ngx + local okong = options and options.kong or kong local okong_request = okong.request local ctx = ongx.ctx local var = ongx.var - local request_uri = var.request_uri or "" - - local host_port = ctx.host_port or tonumber(var.server_port, 10) - + local request_uri = ctx.request_uri or var.request_uri or "" local upstream_uri = var.upstream_uri or "" if upstream_uri ~= "" and not find(upstream_uri, "?", nil, true) then - if byte(ctx.request_uri or var.request_uri, -1) == QUESTION_MARK then + if byte(request_uri, -1) == QUESTION_MARK then upstream_uri = upstream_uri .. "?" elseif var.is_args == "?" then upstream_uri = upstream_uri .. "?" .. (var.args or "") end end - local request_headers, response_headers = nil, nil -- THIS IS AN INTERNAL ONLY FLAG TO SKIP FETCHING HEADERS, -- AND THIS FLAG MIGHT BE REMOVED IN THE FUTURE -- WITHOUT ANY NOTICE AND DEPRECATION. - if not options.__skip_fetch_headers__ then + local request_headers + local response_headers + if not (options and options.__skip_fetch_headers__) then request_headers = okong_request.get_headers() response_headers = ongx.resp.get_headers() + if request_headers["authorization"] ~= nil then + request_headers["authorization"] = "REDACTED" + end + if request_headers["proxy-authorization"] ~= nil then + request_headers["proxy-authorization"] = "REDACTED" + end end - local upstream_status = var.upstream_status or ctx.buffered_status or "" - - local response_source = okong.response.get_source(ongx.ctx) - local response_source_name = TYPE_NAMES[response_source] - local url + local host_port = ctx.host_port or tonumber(var.server_port, 10) if host_port then url = var.scheme .. "://" .. var.host .. ":" .. host_port .. request_uri else @@ -850,7 +862,7 @@ do tls = build_tls_info(var, ctx.CLIENT_VERIFY_OVERRIDE), }, upstream_uri = upstream_uri, - upstream_status = upstream_status, + upstream_status = var.upstream_status or ctx.buffered_status or "", response = { status = ongx.status, headers = response_headers, @@ -862,42 +874,43 @@ do request = tonumber(var.request_time) * 1000, receive = ctx.KONG_RECEIVE_TIME or 0, }, - tries = (ctx.balancer_data or EMPTY_TAB).tries, + tries = ctx.balancer_data and ctx.balancer_data.tries, authenticated_entity = build_authenticated_entity(ctx), route = cycle_aware_deep_copy(ctx.route), service = cycle_aware_deep_copy(ctx.service), consumer = cycle_aware_deep_copy(ctx.authenticated_consumer), client_ip = var.remote_addr, started_at = okong_request.get_start_time(), - source = response_source_name, - + source = TYPE_NAMES[okong.response.get_source(ctx)], workspace = ctx.workspace, workspace_name = get_workspace_name(), } - return edit_result(ctx, root) + local serialize_values = ctx.serialize_values + if serialize_values then + root = edit_result(root, serialize_values) + end + + return root end else function serialize(options) check_phase(PHASES_LOG) - options = options or EMPTY_TAB - local ongx = options.ngx or ngx - local okong = options.kong or kong + local ongx = options and options.ngx or ngx + local okong = options and options.kong or kong local ctx = ongx.ctx local var = ongx.var - local host_port = ctx.host_port or tonumber(var.server_port, 10) - local root = { session = { tls = build_tls_info(var, ctx.CLIENT_VERIFY_OVERRIDE), received = to_decimal(var.bytes_received), sent = to_decimal(var.bytes_sent), status = ongx.status, - server_port = host_port, + server_port = ctx.host_port or tonumber(var.server_port, 10), }, upstream = { received = to_decimal(var.upstream_bytes_received), @@ -907,19 +920,23 @@ do kong = ctx.KONG_PROXY_LATENCY or ctx.KONG_RESPONSE_LATENCY or 0, session = var.session_time * 1000, }, - tries = (ctx.balancer_data or EMPTY_TAB).tries, + tries = ctx.balancer_data and ctx.balancer_data.tries, authenticated_entity = build_authenticated_entity(ctx), route = cycle_aware_deep_copy(ctx.route), service = cycle_aware_deep_copy(ctx.service), consumer = cycle_aware_deep_copy(ctx.authenticated_consumer), client_ip = var.remote_addr, started_at = okong.request.get_start_time(), - workspace = ctx.workspace, workspace_name = get_workspace_name(), } - return edit_result(ctx, root) + local serialize_values = ctx.serialize_values + if serialize_values then + root = edit_result(root, serialize_values) + end + + return root end end end diff --git a/kong/pdk/private/rate_limiting.lua b/kong/pdk/private/rate_limiting.lua new file mode 100644 index 000000000000..1710372a71df --- /dev/null +++ b/kong/pdk/private/rate_limiting.lua @@ -0,0 +1,100 @@ +local table_new = require("table.new") + +local type = type +local pairs = pairs +local assert = assert +local tostring = tostring +local resp_header = ngx.header + +-- determine the number of pre-allocated fields at runtime +local max_fields_n = 4 + +local _M = {} + + +local function _has_rl_ctx(ngx_ctx) + return ngx_ctx.__rate_limiting_context__ ~= nil +end + + +local function _create_rl_ctx(ngx_ctx) + assert(not _has_rl_ctx(ngx_ctx), "rate limiting context already exists") + local ctx = table_new(0, max_fields_n) + ngx_ctx.__rate_limiting_context__ = ctx + return ctx +end + + +local function _get_rl_ctx(ngx_ctx) + assert(_has_rl_ctx(ngx_ctx), "rate limiting context does not exist") + return ngx_ctx.__rate_limiting_context__ +end + + +local function _get_or_create_rl_ctx(ngx_ctx) + if not _has_rl_ctx(ngx_ctx) then + _create_rl_ctx(ngx_ctx) + end + + local rl_ctx = _get_rl_ctx(ngx_ctx) + return rl_ctx +end + + +function _M.store_response_header(ngx_ctx, key, value) + assert( + type(key) == "string", + "arg #2 `key` for function `store_response_header` must be a string" + ) + + local value_type = type(value) + assert( + value_type == "string" or value_type == "number", + "arg #3 `value` for function `store_response_header` must be a string or a number" + ) + + local rl_ctx = _get_or_create_rl_ctx(ngx_ctx) + rl_ctx[key] = value +end + + +function _M.get_stored_response_header(ngx_ctx, key) + assert( + type(key) == "string", + "arg #2 `key` for function `get_stored_response_header` must be a string" + ) + + if not _has_rl_ctx(ngx_ctx) then + return nil + end + + local rl_ctx = _get_rl_ctx(ngx_ctx) + return rl_ctx[key] +end + + +function _M.apply_response_headers(ngx_ctx) + if not _has_rl_ctx(ngx_ctx) then + return + end + + local rl_ctx = _get_rl_ctx(ngx_ctx) + local actual_fields_n = 0 + + for k, v in pairs(rl_ctx) do + resp_header[k] = tostring(v) + actual_fields_n = actual_fields_n + 1 + end + + if actual_fields_n > max_fields_n then + local msg = string.format( + "[private-rl-pdk] bumpping pre-allocated fields from %d to %d for performance reasons", + max_fields_n, + actual_fields_n + ) + ngx.log(ngx.INFO, msg) + max_fields_n = actual_fields_n + end +end + +return _M diff --git a/kong/plugins/aws-lambda/request-util.lua b/kong/plugins/aws-lambda/request-util.lua index 1a152b6778b8..5f936ce81859 100644 --- a/kong/plugins/aws-lambda/request-util.lua +++ b/kong/plugins/aws-lambda/request-util.lua @@ -3,14 +3,13 @@ local ngx_encode_base64 = ngx.encode_base64 local ngx_decode_base64 = ngx.decode_base64 local cjson = require "cjson.safe" -local pl_stringx = require("pl.stringx") local date = require("date") local get_request_id = require("kong.tracing.request_id").get local EMPTY = {} local isempty = require "table.isempty" -local split = pl_stringx.split +local split = require("kong.tools.string").split local ngx_req_get_headers = ngx.req.get_headers local ngx_req_get_uri_args = ngx.req.get_uri_args local ngx_get_http_version = ngx.req.http_version diff --git a/kong/plugins/hmac-auth/access.lua b/kong/plugins/hmac-auth/access.lua index 4df53921d525..39e3acb90fe4 100644 --- a/kong/plugins/hmac-auth/access.lua +++ b/kong/plugins/hmac-auth/access.lua @@ -275,24 +275,29 @@ local function set_consumer(consumer, credential) end end +local function unauthorized(message, www_auth_content) + return { status = 401, message = message, headers = { ["WWW-Authenticate"] = www_auth_content } } +end + local function do_authentication(conf) local authorization = kong_request.get_header(AUTHORIZATION) local proxy_authorization = kong_request.get_header(PROXY_AUTHORIZATION) + local www_auth_content = conf.realm and fmt('hmac realm="%s"', conf.realm) or 'hmac' -- If both headers are missing, return 401 if not (authorization or proxy_authorization) then - return false, { status = 401, message = "Unauthorized" } + return false, unauthorized("Unauthorized", www_auth_content) end -- validate clock skew if not (validate_clock_skew(X_DATE, conf.clock_skew) or validate_clock_skew(DATE, conf.clock_skew)) then - return false, { - status = 401, - message = "HMAC signature cannot be verified, a valid date or " .. - "x-date header is required for HMAC Authentication" - } + return false, unauthorized( + "HMAC signature cannot be verified, a valid date or " .. + "x-date header is required for HMAC Authentication", + www_auth_content + ) end -- retrieve hmac parameter from Proxy-Authorization header @@ -312,26 +317,26 @@ local function do_authentication(conf) local ok, err = validate_params(hmac_params, conf) if not ok then kong.log.debug(err) - return false, { status = 401, message = SIGNATURE_NOT_VALID } + return false, unauthorized(SIGNATURE_NOT_VALID, www_auth_content) end -- validate signature local credential = load_credential(hmac_params.username) if not credential then kong.log.debug("failed to retrieve credential for ", hmac_params.username) - return false, { status = 401, message = SIGNATURE_NOT_VALID } + return false, unauthorized(SIGNATURE_NOT_VALID, www_auth_content) end hmac_params.secret = credential.secret if not validate_signature(hmac_params) then - return false, { status = 401, message = SIGNATURE_NOT_SAME } + return false, unauthorized(SIGNATURE_NOT_SAME, www_auth_content) end -- If request body validation is enabled, then verify digest. if conf.validate_request_body and not validate_body() then kong.log.debug("digest validation failed") - return false, { status = 401, message = SIGNATURE_NOT_SAME } + return false, unauthorized(SIGNATURE_NOT_SAME, www_auth_content) end -- Retrieve consumer @@ -349,34 +354,52 @@ local function do_authentication(conf) return true end +local function set_anonymous_consumer(anonymous) + local consumer_cache_key = kong.db.consumers:cache_key(anonymous) + local consumer, err = kong.cache:get(consumer_cache_key, nil, + kong.client.load_consumer, + anonymous, true) + if err then + return error(err) + end -local _M = {} - + set_consumer(consumer) +end -function _M.execute(conf) - if conf.anonymous and kong_client.get_credential() then - -- we're already authenticated, and we're configured for using anonymous, - -- hence we're in a logical OR between auth methods and we're already done. +--- When conf.anonymous is enabled we are in "logical OR" authentication flow. +--- Meaning - either anonymous consumer is enabled or there are multiple auth plugins +--- and we need to passthrough on failed authentication. +local function logical_OR_authentication(conf) + if kong.client.get_credential() then + -- we're already authenticated and in "logical OR" between auth methods -- early exit return end + local ok, _ = do_authentication(conf) + if not ok then + set_anonymous_consumer(conf.anonymous) + end +end + +--- When conf.anonymous is not set we are in "logical AND" authentication flow. +--- Meaning - if this authentication fails the request should not be authorized +--- even though other auth plugins might have successfully authorized user. +local function logical_AND_authentication(conf) local ok, err = do_authentication(conf) if not ok then - if conf.anonymous then - -- get anonymous user - local consumer_cache_key = kong.db.consumers:cache_key(conf.anonymous) - local consumer, err = kong.cache:get(consumer_cache_key, nil, - kong_client.load_consumer, - conf.anonymous, true) - if err then - return error(err) - end + return kong.response.error(err.status, err.message, err.headers) + end +end - set_consumer(consumer) - else - return kong.response.error(err.status, err.message, err.headers) - end +local _M = {} + +function _M.execute(conf) + + if conf.anonymous then + return logical_OR_authentication(conf) + else + return logical_AND_authentication(conf) end end diff --git a/kong/plugins/hmac-auth/schema.lua b/kong/plugins/hmac-auth/schema.lua index a95b53bd62f9..3e83acf7d5cb 100644 --- a/kong/plugins/hmac-auth/schema.lua +++ b/kong/plugins/hmac-auth/schema.lua @@ -29,6 +29,7 @@ return { elements = { type = "string", one_of = ALGORITHMS }, default = ALGORITHMS, }, }, + { realm = { description = "When authentication fails the plugin sends `WWW-Authenticate` header with `realm` attribute value.", type = "string", required = false }, }, }, }, }, diff --git a/kong/plugins/ldap-auth/access.lua b/kong/plugins/ldap-auth/access.lua index fd79e6f2dccc..e75d92344860 100644 --- a/kong/plugins/ldap-auth/access.lua +++ b/kong/plugins/ldap-auth/access.lua @@ -232,25 +232,29 @@ local function set_consumer(consumer, credential) end end +local function unauthorized(message, authorization_scheme) + return { + status = 401, + message = message, + headers = { ["WWW-Authenticate"] = authorization_scheme } + } +end local function do_authentication(conf) local authorization_value = kong.request.get_header(AUTHORIZATION) local proxy_authorization_value = kong.request.get_header(PROXY_AUTHORIZATION) + local scheme = conf.header_type + if scheme == "ldap" then + -- ensure backwards compatibility (see GH PR #3656) + -- TODO: provide migration to capitalize older configurations + scheme = upper(scheme) + end + + local www_auth_content = conf.realm and fmt('%s realm="%s"', scheme, conf.realm) or scheme -- If both headers are missing, return 401 if not (authorization_value or proxy_authorization_value) then - local scheme = conf.header_type - if scheme == "ldap" then - -- ensure backwards compatibility (see GH PR #3656) - -- TODO: provide migration to capitalize older configurations - scheme = upper(scheme) - end - - return false, { - status = 401, - message = "Unauthorized", - headers = { ["WWW-Authenticate"] = scheme .. ' realm="kong"' } - } + return false, unauthorized("Unauthorized", www_auth_content) end local is_authorized, credential @@ -263,7 +267,7 @@ local function do_authentication(conf) end if not is_authorized then - return false, {status = 401, message = "Unauthorized" } + return false, unauthorized("Unauthorized", www_auth_content) end if conf.hide_credentials then @@ -277,30 +281,50 @@ local function do_authentication(conf) end -function _M.execute(conf) - if conf.anonymous and kong.client.get_credential() then - -- we're already authenticated, and we're configured for using anonymous, - -- hence we're in a logical OR between auth methods and we're already done. +local function set_anonymous_consumer(anonymous) + local consumer_cache_key = kong.db.consumers:cache_key(anonymous) + local consumer, err = kong.cache:get(consumer_cache_key, nil, + kong.client.load_consumer, + anonymous, true) + if err then + return error(err) + end + + set_consumer(consumer) +end + + +--- When conf.anonymous is enabled we are in "logical OR" authentication flow. +--- Meaning - either anonymous consumer is enabled or there are multiple auth plugins +--- and we need to passthrough on failed authentication. +local function logical_OR_authentication(conf) + if kong.client.get_credential() then + -- we're already authenticated and in "logical OR" between auth methods -- early exit return end + local ok, _ = do_authentication(conf) + if not ok then + set_anonymous_consumer(conf.anonymous) + end +end + +--- When conf.anonymous is not set we are in "logical AND" authentication flow. +--- Meaning - if this authentication fails the request should not be authorized +--- even though other auth plugins might have successfully authorized user. +local function logical_AND_authentication(conf) local ok, err = do_authentication(conf) if not ok then - if conf.anonymous then - -- get anonymous user - local consumer_cache_key = kong.db.consumers:cache_key(conf.anonymous) - local consumer, err = kong.cache:get(consumer_cache_key, nil, - kong.client.load_consumer, - conf.anonymous, true) - if err then - return error(err) - end + return kong.response.error(err.status, err.message, err.headers) + end +end - set_consumer(consumer) - else - return kong.response.error(err.status, err.message, err.headers) - end +function _M.execute(conf) + if conf.anonymous then + return logical_OR_authentication(conf) + else + return logical_AND_authentication(conf) end end diff --git a/kong/plugins/ldap-auth/schema.lua b/kong/plugins/ldap-auth/schema.lua index afd6c1acc25f..a5738d471cd9 100644 --- a/kong/plugins/ldap-auth/schema.lua +++ b/kong/plugins/ldap-auth/schema.lua @@ -24,6 +24,7 @@ return { { keepalive = { description = "An optional value in milliseconds that defines how long an idle connection to LDAP server will live before being closed.", type = "number", default = 60000 }, }, { anonymous = { description = "An optional string (consumer UUID or username) value to use as an “anonymous” consumer if authentication fails. If empty (default null), the request fails with an authentication failure `4xx`.", type = "string" }, }, { header_type = { description = "An optional string to use as part of the Authorization header", type = "string", default = "ldap" }, }, + { realm = { description = "When authentication fails the plugin sends `WWW-Authenticate` header with `realm` attribute value.", type = "string", required = false }, }, }, entity_checks = { { conditional = { diff --git a/kong/plugins/oauth2/secret.lua b/kong/plugins/oauth2/secret.lua index f8ee5bbe5ea6..5680334870d2 100644 --- a/kong/plugins/oauth2/secret.lua +++ b/kong/plugins/oauth2/secret.lua @@ -1,6 +1,3 @@ -local kong_string = require "kong.tools.string" - - local type = type local fmt = string.format local find = string.find @@ -11,8 +8,8 @@ local assert = assert local tonumber = tonumber local encode_base64 = ngx.encode_base64 local decode_base64 = ngx.decode_base64 -local strip = kong_string.strip -local split = kong_string.split +local strip = require("kong.tools.string").strip +local split = require("kong.tools.string").split local get_rand_bytes = require("kong.tools.rand").get_rand_bytes diff --git a/kong/plugins/rate-limiting/handler.lua b/kong/plugins/rate-limiting/handler.lua index 13a9bebadaca..a21890248333 100644 --- a/kong/plugins/rate-limiting/handler.lua +++ b/kong/plugins/rate-limiting/handler.lua @@ -2,6 +2,7 @@ local timestamp = require "kong.tools.timestamp" local policies = require "kong.plugins.rate-limiting.policies" local kong_meta = require "kong.meta" +local pdk_private_rl = require "kong.pdk.private.rate_limiting" local kong = kong @@ -16,6 +17,10 @@ local timer_at = ngx.timer.at local SYNC_RATE_REALTIME = -1 +local pdk_rl_store_response_header = pdk_private_rl.store_response_header +local pdk_rl_apply_response_headers = pdk_private_rl.apply_response_headers + + local EMPTY = {} local EXPIRATION = require "kong.plugins.rate-limiting.expiration" @@ -145,11 +150,9 @@ function RateLimitingHandler:access(conf) end if usage then - -- Adding headers + local ngx_ctx = ngx.ctx local reset - local headers if not conf.hide_client_headers then - headers = {} local timestamps local limit local window @@ -178,25 +181,23 @@ function RateLimitingHandler:access(conf) reset = max(1, window - floor((current_timestamp - timestamps[k]) / 1000)) end - headers[X_RATELIMIT_LIMIT[k]] = current_limit - headers[X_RATELIMIT_REMAINING[k]] = current_remaining + pdk_rl_store_response_header(ngx_ctx, X_RATELIMIT_LIMIT[k], current_limit) + pdk_rl_store_response_header(ngx_ctx, X_RATELIMIT_REMAINING[k], current_remaining) end - headers[RATELIMIT_LIMIT] = limit - headers[RATELIMIT_REMAINING] = remaining - headers[RATELIMIT_RESET] = reset + pdk_rl_store_response_header(ngx_ctx, RATELIMIT_LIMIT, limit) + pdk_rl_store_response_header(ngx_ctx, RATELIMIT_REMAINING, remaining) + pdk_rl_store_response_header(ngx_ctx, RATELIMIT_RESET, reset) end -- If limit is exceeded, terminate the request if stop then - headers = headers or {} - headers[RETRY_AFTER] = reset - return kong.response.error(conf.error_code, conf.error_message, headers) + pdk_rl_store_response_header(ngx_ctx, RETRY_AFTER, reset) + pdk_rl_apply_response_headers(ngx_ctx) + return kong.response.error(conf.error_code, conf.error_message) end - if headers then - kong.response.set_headers(headers) - end + pdk_rl_apply_response_headers(ngx_ctx) end if conf.sync_rate ~= SYNC_RATE_REALTIME and conf.policy == "redis" then @@ -211,4 +212,4 @@ function RateLimitingHandler:access(conf) end -return RateLimitingHandler +return RateLimitingHandler \ No newline at end of file diff --git a/kong/plugins/request-size-limiting/handler.lua b/kong/plugins/request-size-limiting/handler.lua index 4c0246d11f09..09ccd524187a 100644 --- a/kong/plugins/request-size-limiting/handler.lua +++ b/kong/plugins/request-size-limiting/handler.lua @@ -1,6 +1,6 @@ -- Copyright (C) Kong Inc. -local strip = require("pl.stringx").strip +local strip = require("kong.tools.string").strip local kong_meta = require "kong.meta" local tonumber = tonumber diff --git a/kong/plugins/response-ratelimiting/access.lua b/kong/plugins/response-ratelimiting/access.lua index 00078502c936..bba16660015e 100644 --- a/kong/plugins/response-ratelimiting/access.lua +++ b/kong/plugins/response-ratelimiting/access.lua @@ -1,5 +1,6 @@ local policies = require "kong.plugins.response-ratelimiting.policies" local timestamp = require "kong.tools.timestamp" +local pdk_private_rl = require "kong.pdk.private.rate_limiting" local kong = kong @@ -9,6 +10,10 @@ local error = error local tostring = tostring +local pdk_rl_store_response_header = pdk_private_rl.store_response_header +local pdk_rl_apply_response_headers = pdk_private_rl.apply_response_headers + + local EMPTY = {} local HTTP_TOO_MANY_REQUESTS = 429 local RATELIMIT_REMAINING = "X-RateLimit-Remaining" @@ -84,6 +89,7 @@ function _M.execute(conf) end -- Append usage headers to the upstream request. Also checks "block_on_first_violation". + local ngx_ctx = ngx.ctx for k in pairs(conf.limits) do local remaining for _, lv in pairs(usage[k]) do @@ -97,9 +103,11 @@ function _M.execute(conf) end end - kong.service.request.set_header(RATELIMIT_REMAINING .. "-" .. k, remaining) + pdk_rl_store_response_header(ngx_ctx, RATELIMIT_REMAINING .. "-" .. k, remaining) end + pdk_rl_apply_response_headers(ngx_ctx) + kong.ctx.plugin.usage = usage -- For later use end diff --git a/kong/plugins/response-ratelimiting/header_filter.lua b/kong/plugins/response-ratelimiting/header_filter.lua index 15b450ce685c..65885b627c51 100644 --- a/kong/plugins/response-ratelimiting/header_filter.lua +++ b/kong/plugins/response-ratelimiting/header_filter.lua @@ -1,4 +1,5 @@ local kong_string = require "kong.tools.string" +local pdk_private_rl = require "kong.pdk.private.rate_limiting" local kong = kong @@ -8,8 +9,12 @@ local pairs = pairs local ipairs = ipairs local tonumber = tonumber local math_max = math.max + + local strip = kong_string.strip local split = kong_string.split +local pdk_rl_store_response_header = pdk_private_rl.store_response_header +local pdk_rl_apply_response_headers = pdk_private_rl.apply_response_headers local RATELIMIT_LIMIT = "X-RateLimit-Limit" @@ -63,16 +68,16 @@ function _M.execute(conf) end local stop + local ngx_ctx = ngx.ctx for limit_name in pairs(usage) do for period_name, lv in pairs(usage[limit_name]) do if not conf.hide_client_headers then - -- increment_value for this current request local limit_hdr = RATELIMIT_LIMIT .. "-" .. limit_name .. "-" .. period_name local remain_hdr = RATELIMIT_REMAINING .. "-" .. limit_name .. "-" .. period_name - kong.response.set_header(limit_hdr, lv.limit) - local remain = math_max(0, lv.remaining - (increments[limit_name] and increments[limit_name] or 0)) - kong.response.set_header(remain_hdr, remain) + + pdk_rl_store_response_header(ngx_ctx, limit_hdr, lv.limit) + pdk_rl_store_response_header(ngx_ctx, remain_hdr, remain) end if increments[limit_name] and increments[limit_name] > 0 and lv.remaining <= 0 then @@ -81,6 +86,8 @@ function _M.execute(conf) end end + pdk_rl_apply_response_headers(ngx_ctx) + kong.response.clear_header(conf.header_name) -- If limit is exceeded, terminate the request diff --git a/kong/plugins/response-transformer/header_transformer.lua b/kong/plugins/response-transformer/header_transformer.lua index 4ed1863ba6cd..3f81bcbe69d5 100644 --- a/kong/plugins/response-transformer/header_transformer.lua +++ b/kong/plugins/response-transformer/header_transformer.lua @@ -1,7 +1,7 @@ local isempty = require "table.isempty" local mime_type = require "kong.tools.mime_type" local ngx_re = require "ngx.re" -local pl_stringx = require "pl.stringx" + local kong = kong local type = type @@ -10,8 +10,8 @@ local noop = function() end local ipairs = ipairs local parse_mime_type = mime_type.parse_mime_type local mime_type_includes = mime_type.includes -local split = ngx_re.split -local strip = pl_stringx.strip +local re_split = ngx_re.split +local strip = require("kong.tools.string").strip local _M = {} @@ -44,7 +44,7 @@ local function is_json_body(content_type) if not content_type then return false end - local content_types = split(content_type, ",") + local content_types = re_split(content_type, ",") local expected_media_type = { type = "application", subtype = "json" } for _, content_type in ipairs(content_types) do local t, subtype = parse_mime_type(strip(content_type)) diff --git a/kong/tools/emmy_debugger.lua b/kong/tools/emmy_debugger.lua index e4bb1be2fe07..e53c98d09555 100644 --- a/kong/tools/emmy_debugger.lua +++ b/kong/tools/emmy_debugger.lua @@ -35,7 +35,7 @@ local function find_source(path) end end - ngx.log(ngx.ERR, "source file " .. path .. " not found in " .. env_prefix .. "_EMMY_DEBUGGER_SOURCE_PATH") + ngx.log(ngx.ERR, "source file ", path, " not found in ", env_prefix, "_EMMY_DEBUGGER_SOURCE_PATH") return path end @@ -71,24 +71,24 @@ local function init(config_) end if not pl_path.isabs(debugger) then - ngx.log(ngx.ERR, env_prefix .. "_EMMY_DEBUGGER (" .. debugger .. ") must be an absolute path") + ngx.log(ngx.ERR, env_prefix, "_EMMY_DEBUGGER (", debugger, ") must be an absolute path") return end if not pl_path.exists(debugger) then - ngx.log(ngx.ERR, env_prefix .. "_EMMY_DEBUGGER (" .. debugger .. ") file not found") + ngx.log(ngx.ERR, env_prefix, "_EMMY_DEBUGGER (", debugger, ") file not found") return end local ext = pl_path.extension(debugger) if ext ~= ".so" and ext ~= ".dylib" then - ngx.log(ngx.ERR, env_prefix .. "_EMMY_DEBUGGER (" .. debugger .. ") must be a .so (Linux) or .dylib (macOS) file") + ngx.log(ngx.ERR, env_prefix, "_EMMY_DEBUGGER (", debugger, ") must be a .so (Linux) or .dylib (macOS) file") return end if ngx.worker.id() > 0 and not multi_worker then - ngx.log(ngx.ERR, env_prefix .. "_EMMY_DEBUGGER is only supported in the first worker process, suggest setting KONG_NGINX_WORKER_PROCESSES to 1") + ngx.log(ngx.ERR, env_prefix, "_EMMY_DEBUGGER is only supported in the first worker process, suggest setting KONG_NGINX_WORKER_PROCESSES to 1") return end - ngx.log(ngx.NOTICE, "loading EmmyLua debugger " .. debugger) + ngx.log(ngx.NOTICE, "loading EmmyLua debugger ", debugger) ngx.log(ngx.WARN, "The EmmyLua integration for Kong is a feature solely for your convenience during development. Kong assumes no liability as a result of using the integration and does not endorse it’s usage. Issues related to usage of EmmyLua integration should be directed to the respective project instead.") local dbg = load_debugger(debugger) diff --git a/kong/tools/http.lua b/kong/tools/http.lua index 613661d68d2f..d95648ae9138 100644 --- a/kong/tools/http.lua +++ b/kong/tools/http.lua @@ -1,6 +1,5 @@ local pl_path = require "pl.path" local pl_file = require "pl.file" -local tools_str = require "kong.tools.string" local type = type @@ -13,9 +12,9 @@ local sort = table.sort local concat = table.concat local fmt = string.format local re_match = ngx.re.match -local join = tools_str.join -local split = tools_str.split -local strip = tools_str.strip +local join = require("kong.tools.string").join +local split = require("kong.tools.string").split +local strip = require("kong.tools.string").strip local _M = {} diff --git a/kong/tools/ip.lua b/kong/tools/ip.lua index 786bf8d6460e..2cb2c9119c5a 100644 --- a/kong/tools/ip.lua +++ b/kong/tools/ip.lua @@ -1,5 +1,5 @@ local ipmatcher = require "resty.ipmatcher" -local pl_stringx = require "pl.stringx" + local type = type @@ -11,7 +11,7 @@ local sub = string.sub local fmt = string.format local lower = string.lower local find = string.find -local split = pl_stringx.split +local split = require("kong.tools.string").split local _M = {} diff --git a/kong/tools/queue.lua b/kong/tools/queue.lua index 725db7ddbf14..dc6a8fb6a108 100644 --- a/kong/tools/queue.lua +++ b/kong/tools/queue.lua @@ -297,8 +297,18 @@ end function Queue.can_enqueue(queue_conf, entry) local queue = queues[_make_queue_key(queue_conf.name)] if not queue then - -- treat non-existing queues as not full as they will be created on demand - return false + -- treat non-existing queues having enough capacity. + -- WARNING: The limitation is that if the `entry` is a string and the `queue.max_bytes` is set, + -- and also the `#entry` is larger than `queue.max_bytes`, + -- this function will incorrectly return `true` instead of `false`. + -- This is a limitation of the current implementation. + -- All capacity checking functions need a Queue instance to work correctly. + -- constructing a Queue instance just for this function is not efficient, + -- so we just return `true` here. + -- This limitation should not happen in normal usage, + -- as user should be aware of the queue capacity settings + -- to avoid such situation. + return true end return _can_enqueue(queue, entry) diff --git a/kong/tools/rand.lua b/kong/tools/rand.lua index cfb4bfbf3409..94ea0615cd78 100644 --- a/kong/tools/rand.lua +++ b/kong/tools/rand.lua @@ -106,10 +106,14 @@ _M.get_rand_bytes = get_rand_bytes -- @return string The random string (a chunk of base64ish-encoded random bytes) local random_string do - local char = string.char local rand = math.random + local byte = string.byte local encode_base64 = ngx.encode_base64 + local OUTPUT = require("string.buffer").new(32) + local SLASH_BYTE = byte("/") + local PLUS_BYTE = byte("+") + -- generate a random-looking string by retrieving a chunk of bytes and -- replacing non-alphanumeric characters with random alphanumeric replacements -- (we dont care about deriving these bytes securely) @@ -117,15 +121,29 @@ do -- previous implementation (stripping a UUID of its hyphens), while significantly -- expanding the size of the keyspace. random_string = function() + OUTPUT:reset() + -- get 24 bytes, which will return a 32 char string after encoding -- this is done in attempt to maintain backwards compatibility as -- much as possible while improving the strength of this function - return encode_base64(get_rand_bytes(24, true)) - :gsub("/", char(rand(48, 57))) -- 0 - 10 - :gsub("+", char(rand(65, 90))) -- A - Z - :gsub("=", char(rand(97, 122))) -- a - z - end + -- TODO: in future we may want to have variable length option to this + local str = encode_base64(get_rand_bytes(24, true), true) + + for i = 1, 32 do + local b = byte(str, i) + if b == SLASH_BYTE then + OUTPUT:putf("%c", rand(65, 90)) -- A-Z + elseif b == PLUS_BYTE then + OUTPUT:putf("%c", rand(97, 122)) -- a-z + else + OUTPUT:putf("%c", b) + end + end + + str = OUTPUT:get() + return str + end end _M.random_string = random_string diff --git a/kong/tools/string.lua b/kong/tools/string.lua index 1920d7e970b7..ef2d844e62d1 100644 --- a/kong/tools/string.lua +++ b/kong/tools/string.lua @@ -1,13 +1,20 @@ local pl_stringx = require "pl.stringx" -local type = type -local ipairs = ipairs -local tostring = tostring -local lower = string.lower -local fmt = string.format -local find = string.find -local gsub = string.gsub +local type = type +local ipairs = ipairs +local tostring = tostring +local lower = string.lower +local sub = string.sub +local fmt = string.format +local find = string.find +local gsub = string.gsub +local byte = string.byte + + +local SPACE_BYTE = byte(" ") +local TAB_BYTE = byte("\t") +local CR_BYTE = byte("\r") local _M = {} @@ -24,16 +31,52 @@ _M.split = pl_stringx.split --- strips whitespace from a string. -- @function strip -_M.strip = function(str) - if str == nil then +_M.strip = function(value) + if value == nil then return "" end - str = tostring(str) - if #str > 200 then - return str:gsub("^%s+", ""):reverse():gsub("^%s+", ""):reverse() - else - return str:match("^%s*(.-)%s*$") + + -- TODO: do we want to operate on non-string values (kept for backward compatibility)? + if type(value) ~= "string" then + value = tostring(value) or "" + end + + if value == "" then + return "" end + + local len = #value + local s = 1 -- position of the leftmost non-whitespace char + for i = 1, len do + local b = byte(value, i) + if b == SPACE_BYTE or (b >= TAB_BYTE and b <= CR_BYTE) then + s = s + 1 + else + break + end + end + + if s > len then + return "" + end + + local e = len -- position of the rightmost non-whitespace char + if s < e then + for i = e, 1, -1 do + local b = byte(value, i) + if b == SPACE_BYTE or (b >= TAB_BYTE and b <= CR_BYTE) then + e = e - 1 + else + break + end + end + end + + if s ~= 1 or e ~= len then + value = sub(value, s, e) + end + + return value end @@ -180,4 +223,3 @@ _M.replace_dashes_lower = replace_dashes_lower return _M - diff --git a/spec/01-unit/27-queue_spec.lua b/spec/01-unit/27-queue_spec.lua index ec166c295a4e..548be6b42bbe 100644 --- a/spec/01-unit/27-queue_spec.lua +++ b/spec/01-unit/27-queue_spec.lua @@ -812,8 +812,11 @@ describe("plugin queue", function() ) end + -- should be true if the queue does not exist + assert.is_true(Queue.can_enqueue(queue_conf)) + assert.is_false(Queue.is_full(queue_conf)) - assert.is_false(Queue.can_enqueue(queue_conf, "One")) + assert.is_true(Queue.can_enqueue(queue_conf, "One")) enqueue(queue_conf, "One") assert.is_false(Queue.is_full(queue_conf)) @@ -835,8 +838,11 @@ describe("plugin queue", function() max_retry_delay = 60, } + -- should be true if the queue does not exist + assert.is_true(Queue.can_enqueue(queue_conf)) + assert.is_false(Queue.is_full(queue_conf)) - assert.is_false(Queue.can_enqueue(queue_conf, "1")) + assert.is_true(Queue.can_enqueue(queue_conf, "1")) enqueue(queue_conf, "1") assert.is_false(Queue.is_full(queue_conf)) @@ -857,6 +863,9 @@ describe("plugin queue", function() max_retry_delay = 60, } + -- should be true if the queue does not exist + assert.is_true(Queue.can_enqueue(queue_conf)) + enqueue(queue_conf, "1") assert.is_false(Queue.is_full(queue_conf)) diff --git a/spec/02-integration/02-cmd/04-version_spec.lua b/spec/02-integration/02-cmd/04-version_spec.lua index e94aa7bbb2c9..3cf0e2578766 100644 --- a/spec/02-integration/02-cmd/04-version_spec.lua +++ b/spec/02-integration/02-cmd/04-version_spec.lua @@ -1,11 +1,14 @@ -local pl_stringx = require "pl.stringx" local helpers = require "spec.helpers" local meta = require "kong.meta" + +local strip = require("kong.tools.string").strip + + describe("kong version", function() it("outputs Kong version", function() local _, _, stdout = assert(helpers.kong_exec("version")) - assert.equal(meta._VERSION, pl_stringx.strip(stdout)) + assert.equal(meta._VERSION, strip(stdout)) end) it("--all outputs all deps versions", function() local _, _, stdout = assert(helpers.kong_exec("version -a")) diff --git a/spec/02-integration/02-cmd/10-migrations_spec.lua b/spec/02-integration/02-cmd/10-migrations_spec.lua index 938d5a325f4e..981cc9e4006b 100644 --- a/spec/02-integration/02-cmd/10-migrations_spec.lua +++ b/spec/02-integration/02-cmd/10-migrations_spec.lua @@ -2,7 +2,7 @@ local helpers = require "spec.helpers" local DB = require "kong.db.init" local tb_clone = require "table.clone" local shell = require "resty.shell" -local strip = require "kong.tools.string".strip +local strip = require("kong.tools.string").strip -- Current number of migrations to execute in a new install @@ -445,4 +445,3 @@ for _, strategy in helpers.each_strategy() do end) end - diff --git a/spec/02-integration/04-admin_api/01-admin_api_spec.lua b/spec/02-integration/04-admin_api/01-admin_api_spec.lua index 19316c60ddf5..0434dec63c01 100644 --- a/spec/02-integration/04-admin_api/01-admin_api_spec.lua +++ b/spec/02-integration/04-admin_api/01-admin_api_spec.lua @@ -1,9 +1,11 @@ local helpers = require "spec.helpers" local utils = require "pl.utils" -local stringx = require "pl.stringx" local http = require "resty.http" +local strip = require("kong.tools.string").strip + + local function count_server_blocks(filename) local file = assert(utils.readfile(filename)) local _, count = file:gsub("[%\n%s]+server%s{","") @@ -16,10 +18,10 @@ local function get_listeners(filename) local result = {} for block in file:gmatch("[%\n%s]+server%s+(%b{})") do local server = {} - local server_name = stringx.strip(block:match("[%\n%s]server_name%s(.-);")) + local server_name = strip(block:match("[%\n%s]server_name%s(.-);")) result[server_name] = server for listen in block:gmatch("[%\n%s]listen%s(.-);") do - listen = stringx.strip(listen) + listen = strip(listen) table.insert(server, listen) server[listen] = #server end diff --git a/spec/02-integration/05-proxy/01-proxy_spec.lua b/spec/02-integration/05-proxy/01-proxy_spec.lua index 110c90cb423f..53f31a050f43 100644 --- a/spec/02-integration/05-proxy/01-proxy_spec.lua +++ b/spec/02-integration/05-proxy/01-proxy_spec.lua @@ -1,9 +1,11 @@ local helpers = require "spec.helpers" local utils = require "pl.utils" -local stringx = require "pl.stringx" local http = require "resty.http" +local strip = require("kong.tools.string").strip + + local function count_server_blocks(filename) local file = assert(utils.readfile(filename)) local _, count = file:gsub("[%\n%s]+server%s{%s*\n","") @@ -16,11 +18,11 @@ local function get_listeners(filename) local result = {} for block in file:gmatch("[%\n%s]+server%s+(%b{})") do local server_name = block:match("[%\n%s]server_name%s(.-);") - server_name = server_name and stringx.strip(server_name) or "stream" + server_name = server_name and strip(server_name) or "stream" local server = result[server_name] or {} result[server_name] = server for listen in block:gmatch("[%\n%s]listen%s(.-);") do - listen = stringx.strip(listen) + listen = strip(listen) table.insert(server, listen) server[listen] = #server end diff --git a/spec/02-integration/09-hybrid_mode/09-config-compat_spec.lua b/spec/02-integration/09-hybrid_mode/09-config-compat_spec.lua index 18465456a086..f7789afb00b1 100644 --- a/spec/02-integration/09-hybrid_mode/09-config-compat_spec.lua +++ b/spec/02-integration/09-hybrid_mode/09-config-compat_spec.lua @@ -659,6 +659,37 @@ describe("CP/DP config compat transformations #" .. strategy, function() -- cleanup admin.plugins:remove({ id = key_auth.id }) end) + + it("[ldap-auth] removes realm for versions below 3.8", function() + local ldap_auth = admin.plugins:insert { + name = "ldap-auth", + config = { + ldap_host = "localhost", + base_dn = "test", + attribute = "test", + realm = "test", + } + } + local expected_ldap_auth_prior_38 = cycle_aware_deep_copy(ldap_auth) + expected_ldap_auth_prior_38.config.realm = nil + do_assert(uuid(), "3.7.0", expected_ldap_auth_prior_38) + -- cleanup + admin.plugins:remove({ id = ldap_auth.id }) + end) + + it("[hmac-auth] removes realm for versions below 3.8", function() + local hmac_auth = admin.plugins:insert { + name = "hmac-auth", + config = { + realm = "test" + } + } + local expected_hmac_auth_prior_38 = cycle_aware_deep_copy(hmac_auth) + expected_hmac_auth_prior_38.config.realm = nil + do_assert(uuid(), "3.7.0", expected_hmac_auth_prior_38) + -- cleanup + admin.plugins:remove({ id = hmac_auth.id }) + end) end) describe("compatibility test for response-transformer plugin", function() diff --git a/spec/02-integration/14-tracing/04-trace-ids-log_spec.lua b/spec/02-integration/14-tracing/04-trace-ids-log_spec.lua index aff12255a3d9..b17fcbfa59aa 100644 --- a/spec/02-integration/14-tracing/04-trace-ids-log_spec.lua +++ b/spec/02-integration/14-tracing/04-trace-ids-log_spec.lua @@ -1,7 +1,8 @@ local helpers = require "spec.helpers" local cjson = require "cjson.safe" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" + +local strip = require("kong.tools.string").strip local FILE_LOG_PATH = os.tmpname() @@ -11,7 +12,6 @@ local trace_id_hex_128 = "4bf92000000000000000000000000001" local span_id = "0000000000000003" local trace_id_hex_pattern = "^%x+$" - local tracing_headers = { { type = "b3", @@ -85,7 +85,7 @@ local function wait_json_log() .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/03-plugins/04-file-log/01-log_spec.lua b/spec/03-plugins/04-file-log/01-log_spec.lua index 1230dfe9ac5a..0fface7e4cd1 100644 --- a/spec/03-plugins/04-file-log/01-log_spec.lua +++ b/spec/03-plugins/04-file-log/01-log_spec.lua @@ -1,11 +1,11 @@ local cjson = require "cjson" local helpers = require "spec.helpers" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" local pl_path = require "pl.path" local fmt = string.format local random_string = require("kong.tools.rand").random_string local uuid = require("kong.tools.uuid").uuid +local strip = require("kong.tools.string").strip local FILE_LOG_PATH = os.tmpname() @@ -85,7 +85,7 @@ local function wait_for_json_log_entry() .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/03-plugins/05-syslog/01-log_spec.lua b/spec/03-plugins/05-syslog/01-log_spec.lua index 9f913dd36c6c..a5d2ab0c586f 100644 --- a/spec/03-plugins/05-syslog/01-log_spec.lua +++ b/spec/03-plugins/05-syslog/01-log_spec.lua @@ -1,7 +1,10 @@ local helpers = require "spec.helpers" local uuid = require "kong.tools.uuid" local cjson = require "cjson" -local pl_stringx = require "pl.stringx" + + +local strip = require("kong.tools.string").strip +local split = require("kong.tools.string").split for _, strategy in helpers.each_strategy() do @@ -138,7 +141,7 @@ for _, strategy in helpers.each_strategy() do local ok, _, stdout = helpers.execute("uname") assert(ok, "failed to retrieve platform name") - platform = pl_stringx.strip(stdout) + platform = strip(stdout) assert(helpers.start_kong({ database = strategy, @@ -207,7 +210,7 @@ for _, strategy in helpers.each_strategy() do local _, _, stdout = assert(helpers.execute("sudo find /var/log -type f -mmin -5 | grep syslog")) assert.True(#stdout > 0) - local files = pl_stringx.split(stdout, "\n") + local files = split(stdout, "\n") assert.True(#files > 0) if files[#files] == "" then diff --git a/spec/03-plugins/11-correlation-id/01-access_spec.lua b/spec/03-plugins/11-correlation-id/01-access_spec.lua index 65de363f8d18..b281b6b7a0f8 100644 --- a/spec/03-plugins/11-correlation-id/01-access_spec.lua +++ b/spec/03-plugins/11-correlation-id/01-access_spec.lua @@ -1,7 +1,10 @@ local helpers = require "spec.helpers" local cjson = require "cjson" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" + + +local strip = require("kong.tools.string").strip + local UUID_PATTERN = "%x%x%x%x%x%x%x%x%-%x%x%x%x%-%x%x%x%x%-%x%x%x%x%-%x%x%x%x%x%x%x%x%x%x%x%x" local UUID_COUNTER_PATTERN = UUID_PATTERN .. "#%d" @@ -18,7 +21,7 @@ local function wait_json_log() .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/03-plugins/19-hmac-auth/03-access_spec.lua b/spec/03-plugins/19-hmac-auth/03-access_spec.lua index 4e3a1920d0fe..c771500426a4 100644 --- a/spec/03-plugins/19-hmac-auth/03-access_spec.lua +++ b/spec/03-plugins/19-hmac-auth/03-access_spec.lua @@ -47,7 +47,8 @@ for _, strategy in helpers.each_strategy() do name = "hmac-auth", route = { id = route1.id }, config = { - clock_skew = 3000 + clock_skew = 3000, + realm = "test-realm" } } @@ -175,19 +176,40 @@ for _, strategy in helpers.each_strategy() do end) describe("HMAC Authentication", function() - it("should not be authorized when the hmac credentials are missing", function() - local date = os.date("!%a, %d %b %Y %H:%M:%S GMT") - local res = assert(proxy_client:send { - method = "POST", - body = {}, - headers = { - ["HOST"] = "hmacauth.test", - date = date - } - }) - local body = assert.res_status(401, res) - body = cjson.decode(body) - assert.equal("Unauthorized", body.message) + describe("when realm is set", function () + it("should not be authorized when the hmac credentials are missing", function() + local date = os.date("!%a, %d %b %Y %H:%M:%S GMT") + local res = assert(proxy_client:send { + method = "POST", + body = {}, + headers = { + ["HOST"] = "hmacauth.test", + date = date + } + }) + local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) + body = cjson.decode(body) + assert.equal("Unauthorized", body.message) + end) + end) + + describe("when realm is not set", function () + it("should return a 401 with an invalid authorization header", function() + local date = os.date("!%a, %d %b %Y %H:%M:%S GMT") + local res = assert(proxy_client:send { + method = "GET", + path = "/request", + body = {}, + headers = { + ["HOST"] = "hmacauth6.test", + date = date, + ["proxy-authorization"] = "this is no hmac token at all is it?", + }, + }) + assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) + end) end) it("rejects gRPC call without credentials", function() @@ -211,6 +233,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -228,6 +251,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("HMAC signature does not match", body.message) end) @@ -242,6 +266,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal([[HMAC signature cannot be verified, ]] .. [[a valid date or x-date header is]] @@ -260,6 +285,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -276,6 +302,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -293,6 +320,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -310,6 +338,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -326,6 +355,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -341,6 +371,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("Unauthorized", body.message) end) @@ -361,6 +392,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.not_nil(body.message) assert.matches("HMAC signature cannot be verified", body.message) @@ -381,6 +413,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.not_nil(body.message) assert.matches("HMAC signature cannot be verified", body.message) @@ -633,6 +666,7 @@ for _, strategy in helpers.each_strategy() do }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -659,6 +693,7 @@ for _, strategy in helpers.each_strategy() do }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -686,6 +721,7 @@ for _, strategy in helpers.each_strategy() do }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -711,6 +747,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -736,6 +773,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -761,6 +799,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -786,6 +825,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal(SIGNATURE_NOT_VALID, body.message) end) @@ -834,6 +874,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) end) it("should pass the right headers to the upstream server", function() @@ -905,6 +946,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal([[HMAC signature cannot be verified, a valid date or]] .. [[ x-date header is required for HMAC Authentication]], body.message) @@ -930,6 +972,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac realm="test-realm"', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal([[HMAC signature cannot be verified, a valid date or]] .. [[ x-date header is required for HMAC Authentication]], body.message) @@ -1071,6 +1114,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("HMAC signature does not match", body.message) end) @@ -1253,6 +1297,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("HMAC signature does not match", body.message) end) @@ -1280,6 +1325,7 @@ for _, strategy in helpers.each_strategy() do }, }) local body = assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("HMAC signature does not match", body.message) end) @@ -1307,6 +1353,7 @@ for _, strategy in helpers.each_strategy() do } }) local body = assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) body = cjson.decode(body) assert.equal("HMAC signature does not match", body.message) end) @@ -1354,6 +1401,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) encodedSignature = ngx.encode_base64( hmac_sha1_binary("secret", "date: " @@ -1373,6 +1421,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) end) it("should pass with GET with request-line having query param", function() @@ -1587,6 +1636,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) end) it("should pass with GET with hmac-sha384", function() @@ -1653,6 +1703,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) end) it("should return a 401 with an invalid authorization header", function() @@ -1668,6 +1719,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.res_status(401, res) + assert.equal('hmac', res.headers["WWW-Authenticate"]) end) it("should pass with hmac-sha1", function() @@ -1831,6 +1883,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.response(res).has.status(401) + assert.equal('hmac', res.headers["WWW-Authenticate"]) end) it("fails 401, with only the second credential provided", function() @@ -1844,6 +1897,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.response(res).has.status(401) + assert.equal('Key', res.headers["WWW-Authenticate"]) end) it("fails 401, with no credential provided", function() @@ -1855,6 +1909,7 @@ for _, strategy in helpers.each_strategy() do }, }) assert.response(res).has.status(401) + assert.equal('Key', res.headers["WWW-Authenticate"]) end) end) diff --git a/spec/03-plugins/20-ldap-auth/01-access_spec.lua b/spec/03-plugins/20-ldap-auth/01-access_spec.lua index f106076e58b6..af5dd2a6cd83 100644 --- a/spec/03-plugins/20-ldap-auth/01-access_spec.lua +++ b/spec/03-plugins/20-ldap-auth/01-access_spec.lua @@ -74,6 +74,10 @@ for _, ldap_strategy in pairs(ldap_strategies) do hosts = { "ldap7.test" }, } + local route8 = bp.routes:insert { + hosts = { "ldap8.test" }, + } + assert(bp.routes:insert { protocols = { "grpc" }, paths = { "/hello.HelloService/" }, @@ -110,6 +114,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do attribute = "uid", hide_credentials = true, cache_ttl = 2, + realm = "test-ldap", } } @@ -177,6 +182,20 @@ for _, ldap_strategy in pairs(ldap_strategies) do } } + bp.plugins:insert { + route = { id = route8.id }, + name = "ldap-auth", + config = { + ldap_host = ldap_host_aws, + ldap_port = 389, + start_tls = ldap_strategy.start_tls, + base_dn = "ou=scientists,dc=ldap,dc=mashape,dc=com", + attribute = "uid", + header_type = "Basic", + realm = "test-ldap", + } + } + assert(helpers.start_kong({ database = strategy, nginx_conf = "spec/fixtures/custom_nginx.template", @@ -211,8 +230,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) - local value = assert.response(res).has.header("www-authenticate") - assert.are.equal('LDAP realm="kong"', value) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) local json = assert.response(res).has.jsonbody() assert.equal("Unauthorized", json.message) end) @@ -249,6 +267,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) local json = assert.response(res).has.jsonbody() assert.equal("Unauthorized", json.message) end) @@ -262,6 +281,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) local json = assert.response(res).has.jsonbody() assert.equal("Unauthorized", json.message) end) @@ -291,7 +311,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do end) it("fails if credential type is invalid in post request", function() - local r = assert(proxy_client:send { + local res = assert(proxy_client:send { method = "POST", path = "/request", body = {}, @@ -301,7 +321,8 @@ for _, ldap_strategy in pairs(ldap_strategies) do ["content-type"] = "application/x-www-form-urlencoded", } }) - assert.response(r).has.status(401) + assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("passes if credential is valid and starts with space in post request", function() local res = assert(proxy_client:send { @@ -349,6 +370,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("authorization fails with correct status with wrong very long password", function() local res = assert(proxy_client:send { @@ -360,6 +382,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("authorization fails if credential has multiple encoded usernames or passwords separated by ':' in get request", function() local res = assert(proxy_client:send { @@ -371,6 +394,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("does not pass if credential is invalid in get request", function() local res = assert(proxy_client:send { @@ -382,6 +406,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("does not hide credential sent along with authorization header to upstream server", function() local res = assert(proxy_client:send { @@ -408,6 +433,18 @@ for _, ldap_strategy in pairs(ldap_strategies) do assert.response(res).has.status(200) assert.request(res).has.no.header("authorization") end) + it("does not pass if credential is invalid in get request and passes www-authenticate realm information", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/request", + headers = { + host = "ldap2.test", + authorization = "ldap " .. ngx.encode_base64("einstein:wrong_password") + } + }) + assert.response(res).has.status(401) + assert.equal('LDAP realm="test-ldap"', res.headers["WWW-Authenticate"]) + end) it("passes if custom credential type is given in post request", function() local r = assert(proxy_client:send { method = "POST", @@ -432,12 +469,27 @@ for _, ldap_strategy in pairs(ldap_strategies) do assert.response(res).has.status(401) local value = assert.response(res).has.header("www-authenticate") - assert.equal('Basic realm="kong"', value) + assert.equal('Basic', value) + local json = assert.response(res).has.jsonbody() + assert.equal("Unauthorized", json.message) + end) + it("injects conf.header_type in WWW-Authenticate header and realm if provided", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/get", + headers = { + host = "ldap8.test", + } + }) + assert.response(res).has.status(401) + + local value = assert.response(res).has.header("www-authenticate") + assert.equal('Basic realm="test-ldap"', value) local json = assert.response(res).has.jsonbody() assert.equal("Unauthorized", json.message) end) it("fails if custom credential type is invalid in post request", function() - local r = assert(proxy_client:send { + local res = assert(proxy_client:send { method = "POST", path = "/request", body = {}, @@ -447,7 +499,8 @@ for _, ldap_strategy in pairs(ldap_strategies) do ["content-type"] = "application/x-www-form-urlencoded", } }) - assert.response(r).has.status(401) + assert.response(res).has.status(401) + assert.equal('Basic', res.headers["WWW-Authenticate"]) end) it("passes if credential is valid in get request using global plugin", function() local res = assert(proxy_client:send { @@ -676,6 +729,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('LDAP', res.headers["WWW-Authenticate"]) end) it("fails 401, with only the second credential provided", function() @@ -688,6 +742,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('Key', res.headers["WWW-Authenticate"]) end) it("fails 401, with no credential provided", function() @@ -699,6 +754,7 @@ for _, ldap_strategy in pairs(ldap_strategies) do } }) assert.response(res).has.status(401) + assert.equal('Key', res.headers["WWW-Authenticate"]) end) end) diff --git a/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua b/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua index 7fc751546d11..a3df462b3d8d 100644 --- a/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua @@ -1,7 +1,10 @@ local helpers = require "spec.helpers" local cjson = require "cjson" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" + + +local strip = require("kong.tools.string").strip + local PLUGIN_NAME = "ai-proxy" local MOCK_PORT = helpers.get_available_port() @@ -21,7 +24,7 @@ local function wait_for_json_log_entry(FILE_LOG_PATH) .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua b/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua index e96e570f3887..d846b4f5c55a 100644 --- a/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua +++ b/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua @@ -1,7 +1,8 @@ local helpers = require "spec.helpers" local cjson = require "cjson" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" + +local strip = require("kong.tools.string").strip local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-request-transformer" @@ -17,7 +18,7 @@ local function wait_for_json_log_entry(FILE_LOG_PATH) .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua b/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua index 806c824ace0f..d5e4a4ecb611 100644 --- a/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua +++ b/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua @@ -1,7 +1,8 @@ local helpers = require "spec.helpers" local cjson = require "cjson" local pl_file = require "pl.file" -local pl_stringx = require "pl.stringx" + +local strip = require("kong.tools.string").strip local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-response-transformer" @@ -17,7 +18,7 @@ local function wait_for_json_log_entry(FILE_LOG_PATH) .eventually(function() local data = assert(pl_file.read(FILE_LOG_PATH)) - data = pl_stringx.strip(data) + data = strip(data) assert(#data > 0, "log file is empty") data = data:match("%b{}") diff --git a/spec/06-third-party/01-deck/01-deck-integration_spec.lua b/spec/06-third-party/01-deck/01-deck-integration_spec.lua new file mode 100644 index 000000000000..3297bee2e32a --- /dev/null +++ b/spec/06-third-party/01-deck/01-deck-integration_spec.lua @@ -0,0 +1,378 @@ +local helpers = require "spec.helpers" +local pl_tablex = require "pl.tablex" +local cjson = require "cjson" +local ssl_fixtures = require "spec.fixtures.ssl" + +local pl_pairmap = pl_tablex.pairmap + +local ADMIN_LISTEN = "127.0.0.1:9001" +local DECK_TAG = "latest" + + +-- some plugins define required config fields that do not have default values. +-- This table defines values for such fields to obtain a minimal configuration +-- to set up each plugin. +local function get_plugins_configs(service) + return { + ["tcp-log"] = { + name = "tcp-log", + config = { + host = "127.0.0.1", + port = 10000, + } + }, + ["post-function"] = { + name = "post-function", + config = { + access = { "print('hello, world')" }, + } + }, + ["pre-function"] = { + name = "pre-function", + config = { + access = { "print('hello, world')" }, + } + }, + ["acl"] = { + name = "acl", + config = { + allow = { "test group" } + } + }, + ["oauth2"] = { + name = "oauth2", + config = { + enable_password_grant = true + } + }, + ["azure-functions"] = { + name = "azure-functions", + config = { + appname = "test", + functionname = "test", + } + }, + ["udp-log"] = { + name = "udp-log", + config = { + host = "test.test", + port = 8123 + } + }, + ["ip-restriction"] = { + name = "ip-restriction", + config = { + allow = { "0.0.0.0" } + } + }, + ["file-log"] = { + name = "file-log", + config = { + path = "/tmp/log.out" + } + }, + ["http-log"] = { + name = "http-log", + config = { + http_endpoint = "http://test.test" + } + }, + ["acme"] = { + name = "acme", + config = { + account_email = "test@test.test" + }, + }, + ["rate-limiting"] = { + name = "rate-limiting", + config = { + second = 1 + }, + }, + ["ai-request-transformer"] = { + name = "ai-request-transformer", + config = { + prompt = "test", + llm = { + route_type = "llm/v1/chat", + auth = { + header_name = "Authorization", + header_value = "Bearer cohere-key", + }, + model = { + name = "command", + provider = "cohere", + }, + }, + }, + }, + ["ai-prompt-guard"] = { + name = "ai-prompt-guard", + config = { + allow_patterns = { "test" }, + }, + }, + ["response-ratelimiting"] = { + name = "response-ratelimiting", + config = { + limits = { + test = { + second = 1, + }, + }, + }, + }, + ["proxy-cache"] = { + name = "proxy-cache", + config = { + strategy = "memory" + }, + }, + ["opentelemetry"] = { + name = "opentelemetry", + config = { + endpoint = "http://test.test" + }, + }, + ["loggly"] = { + name = "loggly", + config = { + key = "123" + }, + }, + ["ai-proxy"] = { + name = "ai-proxy", + config = { + route_type = "llm/v1/chat", + auth = { + header_name = "Authorization", + header_value = "Bearer openai-key", + }, + model = { + name = "gpt-3.5-turbo", + provider = "openai", + options = { + upstream_url = "http://test.test" + }, + }, + }, + }, + ["ai-prompt-template"] = { + name = "ai-prompt-template", + config = { + templates = { + [1] = { + name = "developer-chat", + template = "foo", + }, + } + }, + }, + ["ai-prompt-decorator"] = { + name = "ai-prompt-decorator", + config = { + prompts = { + prepend = { + [1] = { + role = "system", + content = "Prepend text 1 here.", + } + } + }, + }, + }, + ["ldap-auth"] = { + name = "ldap-auth", + config = { + base_dn = "ou=scientists,dc=ldap,dc=mashape,dc=com", + attribute = "uid", + ldap_host = "test" + }, + }, + ["ai-response-transformer"] = { + name = "ai-response-transformer", + config = { + prompt = "test", + llm = { + model = { + provider = "cohere" + }, + auth = { + header_name = "foo", + header_value = "bar" + }, + route_type = "llm/v1/chat", + }, + }, + }, + ["standard-webhooks"] = { + name = "standard-webhooks", + config = { + secret_v1 = "test", + }, + } + } +end + + +-- pending plugins are not yet supported by deck +local pending = {} + + +-- returns a list-like table of all plugins +local function get_all_plugins() + return pl_pairmap( + function(k, v) + return type(k) ~= "number" and k or v + end, + require("kong.constants").BUNDLED_PLUGINS + ) +end + + +local function get_docker_run_cmd(deck_command, config_dir, config_file) + local cmd = "docker run -u $(id -u):$(id -g) " .. + "-v " .. config_dir .. ":/tmp/cfg " .. + "--network host " .. + "kong/deck:" .. DECK_TAG .. + " gateway " .. deck_command .. + " --kong-addr http://" .. ADMIN_LISTEN + + if deck_command == "dump" then + cmd = cmd .. " --with-id -o" + end + + return cmd .. " /tmp/cfg/" .. config_file +end + + +for _, strategy in helpers.each_strategy({ "postgres" }) do + describe("Deck tests", function() + local admin_client, cleanup + local plugins = get_all_plugins() + local configured_plugins_num = 0 + + local kong_env = { + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + plugins = table.concat(plugins, ","), + admin_listen = ADMIN_LISTEN, + } + + lazy_setup(function() + local bp = helpers.get_db_utils(strategy, nil, plugins) + + -- services and plugins + local service = bp.services:insert { + name = "example-service", + host = "example.com" + } + local plugins_configs = get_plugins_configs(service) + for _, plugin in ipairs(plugins) do + if not pending[plugin] then + local ok, err + ok, err = pcall( + bp.plugins.insert, + bp.plugins, + plugins_configs[plugin] or { name = plugin } + ) + + -- if this assertion fails make sure the plugin is configured + -- correctly with the required fields in the `get_plugins_configs` + -- function above + assert(ok, "failed configuring plugin: " .. plugin .. " with error: " + .. tostring(err)) + configured_plugins_num = configured_plugins_num + 1 + end + end + + -- other entities + bp.routes:insert { + hosts = { "example.com" }, + service = service, + } + local certificate = bp.certificates:insert { + cert = ssl_fixtures.cert_alt_alt, + key = ssl_fixtures.key_alt_alt, + cert_alt = ssl_fixtures.cert_alt_alt_ecdsa, + key_alt = ssl_fixtures.key_alt_alt_ecdsa, + } + bp.snis:insert { + name = "example.test", + certificate = certificate, + } + bp.ca_certificates:insert { + cert = ssl_fixtures.cert_ca, + } + local upstream = bp.upstreams:insert() + bp.targets:insert({ + upstream = upstream, + target = "api-1:80", + }) + bp.consumers:insert { + username = "consumer" + } + bp.vaults:insert({ + name = "env", + prefix = "my-env-vault", + }) + + assert(helpers.start_kong(kong_env)) + admin_client = helpers.admin_client() + + -- pull deck image + local result = { os.execute("docker pull kong/deck:" .. DECK_TAG) } + assert.same({ true, "exit", 0 }, result) + end) + + lazy_teardown(function() + if admin_client then + admin_client:close() + end + + helpers.stop_kong() + if cleanup then + cleanup() + end + end) + + it("execute `gateway dump` and `gateway sync` commands successfully", function() + local config_file = "deck-config.yml" + local config_dir + config_dir, cleanup = helpers.make_temp_dir() + + -- dump the config + local result = { os.execute(get_docker_run_cmd("dump", config_dir, config_file)) } + assert.same({ true, "exit", 0 }, result) + + -- confirm the config file was created + local f = io.open(config_dir .. "/" .. config_file, "r") + assert(f and f:close()) + assert.not_nil(f) + + -- reset db + helpers.get_db_utils(strategy, nil, plugins) + helpers.restart_kong(kong_env) + + -- confirm db reset (no plugins are configured) + local res = assert(admin_client:send { + method = "GET", + path = "/plugins/", + }) + local configured_plugins = cjson.decode(assert.res_status(200, res)) + assert.equals(0, #configured_plugins.data) + + -- sync the config + result = { os.execute(get_docker_run_cmd("sync", config_dir, config_file)) } + assert.same({ true, "exit", 0 }, result) + + -- confirm sync happened (all expected plugins are configured) + res = assert(admin_client:send { + method = "GET", + path = "/plugins/", + }) + configured_plugins = cjson.decode(assert.res_status(200, res)) + assert.equals(configured_plugins_num, #configured_plugins.data) + end) + end) +end diff --git a/spec/fixtures/https_server.lua b/spec/fixtures/https_server.lua index b3c61f4496a6..04872ece2497 100644 --- a/spec/fixtures/https_server.lua +++ b/spec/fixtures/https_server.lua @@ -9,13 +9,15 @@ local pl_dir = require "pl.dir" local pl_file = require "pl.file" local pl_template = require "pl.template" local pl_path = require "pl.path" -local pl_stringx = require "pl.stringx" local uuid = require "resty.jit-uuid" local http_client = require "resty.http" local cjson = require "cjson" local shell = require "resty.shell" +local Template = require("pl.stringx").Template + + -- we need this to get random UUIDs math.randomseed(os.time()) @@ -62,7 +64,7 @@ local function create_conf(params) return nil, err end - local compiled_tpl = pl_stringx.Template(tpl:render(params, { ipairs = ipairs })) + local compiled_tpl = Template(tpl:render(params, { ipairs = ipairs })) local conf_filename = params.base_path .. "/nginx.conf" local conf, err = io.open (conf_filename, "w") if err then diff --git a/spec/fixtures/mock_upstream.lua b/spec/fixtures/mock_upstream.lua index 76c9a58369fb..0584ae850d53 100644 --- a/spec/fixtures/mock_upstream.lua +++ b/spec/fixtures/mock_upstream.lua @@ -1,8 +1,8 @@ local cjson_safe = require "cjson.safe" local cjson = require "cjson" local ws_server = require "resty.websocket.server" -local pl_stringx = require "pl.stringx" local pl_file = require "pl.file" +local strip = require("kong.tools.string").strip local split = require("kong.tools.string").split @@ -28,7 +28,7 @@ local function parse_multipart_form_params(body, content_type) local params = {} local part, from, to, part_value, part_name, part_headers, first_header for i = 1, #parts_split do - part = pl_stringx.strip(parts_split[i]) + part = strip(parts_split[i]) if part ~= '' and part ~= '--' then from, to, err = ngx.re.find(part, '^\\r$', 'ojm') @@ -39,7 +39,7 @@ local function parse_multipart_form_params(body, content_type) part_value = part:sub(to + 2, #part) -- +2: trim leading line jump part_headers = part:sub(1, from - 1) first_header = split(part_headers, '\\n')[1] - if pl_stringx.startswith(first_header:lower(), "content-disposition") then + if first_header:lower():sub(1, 19) == "content-disposition" then local m, err = ngx.re.match(first_header, 'name="(.*?)"', "oj") if err or not m or not m[1] then diff --git a/spec/helpers.lua b/spec/helpers.lua index 71eab581cc76..7292b55e06c4 100644 --- a/spec/helpers.lua +++ b/spec/helpers.lua @@ -49,7 +49,6 @@ local dc_blueprints = require "spec.fixtures.dc_blueprints" local conf_loader = require "kong.conf_loader" local kong_global = require "kong.global" local Blueprints = require "spec.fixtures.blueprints" -local pl_stringx = require "pl.stringx" local constants = require "kong.constants" local pl_tablex = require "pl.tablex" local pl_utils = require "pl.utils" @@ -78,6 +77,9 @@ local resty_signal = require "resty.signal" local lfs = require "lfs" local luassert = require "luassert.assert" local uuid = require("kong.tools.uuid").uuid +local colors = require "ansicolors" +local strip = require("kong.tools.string").strip +local splitlines = require("pl.stringx").splitlines ffi.cdef [[ int setenv(const char *name, const char *value, int overwrite); @@ -2473,6 +2475,33 @@ local deep_sort do end +local function copy_errlog(errlog_path) + local file_path = "Unknown path" + local line_number = "Unknown line" + local errlog_cache_dir = os.getenv("SPEC_ERRLOG_CACHE_DIR") or "/tmp/kong_errlog_cache" + + local ok, err = pl_dir.makepath(errlog_cache_dir) + assert(ok, "makepath failed: " .. tostring(err)) + + local info = debug.getinfo(4, "Sl") + if info then + file_path = info.source:gsub("^@", "") + line_number = info.currentline + end + + if string.find(file_path, '/', nil, true) then + file_path = string.gsub(file_path, '/', '_') + end + file_path = errlog_cache_dir .. "/" .. file_path:gsub("%.lua$", "_") .. "line_" .. line_number .. '.log' + + ok, err = pl_file.copy(errlog_path, file_path) + if ok then + print(colors("%{yellow}Log saved as: " .. file_path .. "%{reset}")) + else + print(colors("%{red}Failed to save error log for test " .. file_path .. ": " .. err)) + end +end + --- Assertion to check the status-code of a http response. -- @function status -- @param expected the expected status code @@ -2499,12 +2528,14 @@ local function res_status(state, args) if expected ~= res.status then local body, err = res:read_body() if not body then body = "Error reading body: " .. err end - table.insert(args, 1, pl_stringx.strip(body)) + table.insert(args, 1, strip(body)) table.insert(args, 1, res.status) table.insert(args, 1, expected) args.n = 3 if res.status == 500 then + copy_errlog(conf.nginx_err_logs) + -- on HTTP 500, we can try to read the server's error logs -- for debugging purposes (very useful for travis) local str = pl_file.read(conf.nginx_err_logs) @@ -2512,7 +2543,7 @@ local function res_status(state, args) return false -- no err logs to read in this prefix end - local lines_t = pl_stringx.splitlines(str) + local lines_t = splitlines(str) local str_t = {} -- filter out debugs as they are not usually useful in this context for i = 1, #lines_t do @@ -2536,12 +2567,12 @@ local function res_status(state, args) local body, err = res:read_body() local output = body if not output then output = "Error reading body: " .. err end - output = pl_stringx.strip(output) + output = strip(output) table.insert(args, 1, output) table.insert(args, 1, res.status) table.insert(args, 1, expected) args.n = 3 - return true, {pl_stringx.strip(body)} + return true, { strip(body) } end end say:set("assertion.res_status.negative", [[ @@ -3345,7 +3376,7 @@ function kong_exec(cmd, env, returns, env_vars) do local function cleanup(t) if t then - t = pl_stringx.strip(t) + t = strip(t) if t:sub(-1,-1) == ";" then t = t:sub(1, -2) end diff --git a/t/01-pdk/02-log/00-phase_checks.t b/t/01-pdk/02-log/00-phase_checks.t index cef91c1755c6..fc49d609f34f 100644 --- a/t/01-pdk/02-log/00-phase_checks.t +++ b/t/01-pdk/02-log/00-phase_checks.t @@ -237,5 +237,3 @@ qq{ GET /t --- no_error_log [error] - - diff --git a/t/01-pdk/02-log/05-set_serialize_value.t b/t/01-pdk/02-log/05-set_serialize_value.t index 2776ddf612b7..d05a3445d15c 100644 --- a/t/01-pdk/02-log/05-set_serialize_value.t +++ b/t/01-pdk/02-log/05-set_serialize_value.t @@ -35,6 +35,7 @@ mode false mode must be 'set', 'add' or 'replace' [error] + === TEST 2: kong.log.serialize() rejects invalid values, including self-referencial tables --- http_config eval: $t::Util::HttpConfig --- config @@ -64,6 +65,7 @@ self_ref false value must be nil, a number, string, boolean or a non-self-refere [error] + === TEST 3: kong.log.set_serialize_value stores changes on ngx.ctx.serialize_values --- http_config eval: $t::Util::HttpConfig --- config @@ -73,15 +75,15 @@ self_ref false value must be nil, a number, string, boolean or a non-self-refere local pdk = PDK.new() pdk.log.set_serialize_value("val1", 1) - assert(#ngx.ctx.serialize_values == 3, "== 3 ") + assert(#ngx.ctx.serialize_values == 1, "== 1 ") -- Supports several operations over the same variable pdk.log.set_serialize_value("val1", 2) - assert(#ngx.ctx.serialize_values == 4, "== 4") + assert(#ngx.ctx.serialize_values == 2, "== 2") -- Other variables also supported pdk.log.set_serialize_value("val2", 1) - assert(#ngx.ctx.serialize_values == 5, "== 5") + assert(#ngx.ctx.serialize_values == 3, "== 3") } } --- request @@ -91,6 +93,7 @@ GET /t [error] + === TEST 4: kong.log.set_serialize_value() sets, adds and replaces values with simple keys --- http_config eval: $t::Util::HttpConfig --- config @@ -139,6 +142,7 @@ add existing value does not set it true [error] + === TEST 5: kong.log.set_serialize_value sets, adds and replaces values with keys with dots --- http_config eval: $t::Util::HttpConfig --- config @@ -248,6 +252,8 @@ complex true --- no_error_log [error] + + === TEST 7: kong.log.set_serialize_value() setting values to nil --- http_config eval: $t::Util::HttpConfig --- config @@ -287,6 +293,8 @@ request.headers.authorization true --- no_error_log [error] + + === TEST 8: kong.log.serialize() redactes authorization headers by default --- http_config eval: $t::Util::HttpConfig --- config @@ -325,5 +333,3 @@ Proxy-Authorization REDACTED PROXY_AUTHORIZATION REDACTED --- no_error_log [error] - - diff --git a/t/01-pdk/04-request/21-get_uri_captures.t b/t/01-pdk/04-request/21-get_uri_captures.t index bd2d977df935..576623e1e474 100644 --- a/t/01-pdk/04-request/21-get_uri_captures.t +++ b/t/01-pdk/04-request/21-get_uri_captures.t @@ -39,5 +39,3 @@ GET /t/01/ok uri_captures: tag: /ok, 0: /t/01/ok, 1: /01/ok, 2: /01, 3: /ok --- no_error_log [error] - - diff --git a/t/01-pdk/08-response/05-set_header.t b/t/01-pdk/08-response/05-set_header.t index ed4cf1fea607..130f735c809b 100644 --- a/t/01-pdk/08-response/05-set_header.t +++ b/t/01-pdk/08-response/05-set_header.t @@ -280,7 +280,8 @@ Transfer-Encoding: chunked manually setting Transfer-Encoding. Ignored. -=== TEST 8: response.set_header() with header table + +=== TEST 9: response.set_header() with header table --- http_config eval: $t::Util::HttpConfig --- config location = /t { diff --git a/t/01-pdk/08-response/11-exit.t b/t/01-pdk/08-response/11-exit.t index 4a6f7a624c92..d86ea99088e7 100644 --- a/t/01-pdk/08-response/11-exit.t +++ b/t/01-pdk/08-response/11-exit.t @@ -1178,5 +1178,3 @@ Content-Type: application/json; charset=utf-8 {"n":9007199254740992} --- no_error_log [error] - - diff --git a/t/01-pdk/08-response/13-error.t b/t/01-pdk/08-response/13-error.t index 047850b44b24..b65924f8f179 100644 --- a/t/01-pdk/08-response/13-error.t +++ b/t/01-pdk/08-response/13-error.t @@ -454,6 +454,7 @@ grpc-message: ResourceExhausted [error] + === TEST 15: service.response.error() honors values of multiple Accept headers --- http_config eval: $t::Util::HttpConfig --- config diff --git a/t/01-pdk/10-nginx/02-get_statistics.t b/t/01-pdk/10-nginx/02-get_statistics.t index 087cdb2bca4a..2b0ac7d535b8 100644 --- a/t/01-pdk/10-nginx/02-get_statistics.t +++ b/t/01-pdk/10-nginx/02-get_statistics.t @@ -20,7 +20,6 @@ run_tests(); __DATA__ - === TEST 1: nginx.get_statistics() returns Nginx connections and requests states --- http_config eval: $t::Util::HttpConfig @@ -54,4 +53,4 @@ connections_waiting: \d+ connections_accepted: \d+ connections_handled: \d+/ --- no_error_log -[error] \ No newline at end of file +[error] diff --git a/t/01-pdk/15-tracing/01-context.t b/t/01-pdk/15-tracing/01-context.t index d1e1b1210524..d8a0f5bf0451 100644 --- a/t/01-pdk/15-tracing/01-context.t +++ b/t/01-pdk/15-tracing/01-context.t @@ -32,6 +32,7 @@ nil [error] + === TEST 2: tracer.set_active_span() sets active span --- http_config eval: $t::Util::HttpConfig --- config @@ -57,6 +58,7 @@ access [error] + === TEST 3: tracer.active_span() get tracer from active span --- http_config eval: $t::Util::HttpConfig --- config diff --git a/t/01-pdk/16-rl-ctx.t b/t/01-pdk/16-rl-ctx.t new file mode 100644 index 000000000000..2c233fd9c01f --- /dev/null +++ b/t/01-pdk/16-rl-ctx.t @@ -0,0 +1,200 @@ +use strict; +use warnings FATAL => 'all'; +use Test::Nginx::Socket::Lua; +do "./t/Util.pm"; + +plan tests => repeat_each() * (blocks() * 4) - 1; + +run_tests(); + +__DATA__ + +=== TEST 1: should work in rewrite phase +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + rewrite_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + pdk_rl.store_response_header(ngx.ctx, "X-1", 1) + pdk_rl.store_response_header(ngx.ctx, "X-2", 2) + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + + pdk_rl.apply_response_headers(ngx.ctx) + } + + content_by_lua_block { + ngx.say("ok") + } + + log_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + } + } +--- request +GET /t +--- response_headers +X-1: 1 +X-2: 2 +--- no_error_log +[error] + + + +=== TEST 2: should work in access phase +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + access_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + pdk_rl.store_response_header(ngx.ctx, "X-1", 1) + pdk_rl.store_response_header(ngx.ctx, "X-2", 2) + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + + pdk_rl.apply_response_headers(ngx.ctx) + } + + content_by_lua_block { + ngx.say("ok") + } + + log_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + } + } +--- request +GET /t +--- response_headers +X-1: 1 +X-2: 2 +--- no_error_log +[error] + + +=== TEST 3: should work in header_filter phase +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + header_filter_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + pdk_rl.store_response_header(ngx.ctx, "X-1", 1) + pdk_rl.store_response_header(ngx.ctx, "X-2", 2) + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + + pdk_rl.apply_response_headers(ngx.ctx) + } + + content_by_lua_block { + ngx.say("ok") + } + + log_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + + local value = pdk_rl.get_stored_response_header(ngx.ctx, "X-1") + assert(value == 1, "unexpected value: " .. value) + + value = pdk_rl.get_stored_response_header(ngx.ctx, "X-2") + assert(value == 2, "unexpected value: " .. value) + } + } +--- request +GET /t +--- response_headers +X-1: 1 +X-2: 2 +--- no_error_log +[error] + + + +=== TEST 4: should not accept invalid arguments +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + rewrite_by_lua_block { + local pdk_rl = require("kong.pdk.private.rate_limiting") + local ok, err + + ok, err = pcall(pdk_rl.store_response_header, ngx.ctx, nil, 1) + assert(not ok, "pcall should fail") + assert( + err:find("arg #2 `key` for function `store_response_header` must be a string", nil, true), + "unexpected error message: " .. err + ) + for _k, v in ipairs({ 1, true, {}, function() end, ngx.null }) do + ok, err = pcall(pdk_rl.store_response_header, ngx.ctx, v, 1) + assert(not ok, "pcall should fail") + assert( + err:find("arg #2 `key` for function `store_response_header` must be a string", nil, true), + "unexpected error message: " .. err + ) + end + + ok, err = pcall(pdk_rl.store_response_header, ngx.ctx, "X-1", nil) + assert(not ok, "pcall should fail") + assert( + err:find("arg #3 `value` for function `store_response_header` must be a string or a number", nil, true), + "unexpected error message: " .. err + ) + for _k, v in ipairs({ true, {}, function() end, ngx.null }) do + ok, err = pcall(pdk_rl.store_response_header, ngx.ctx, "X-1", v) + assert(not ok, "pcall should fail") + assert( + err:find("arg #3 `value` for function `store_response_header` must be a string or a number", nil, true), + "unexpected error message: " .. err + ) + end + + ok, err = pcall(pdk_rl.get_stored_response_header, ngx.ctx, nil) + assert(not ok, "pcall should fail") + assert( + err:find("arg #2 `key` for function `get_stored_response_header` must be a string", nil, true), + "unexpected error message: " .. err + ) + for _k, v in ipairs({ 1, true, {}, function() end, ngx.null }) do + ok, err = pcall(pdk_rl.get_stored_response_header, ngx.ctx, v) + assert(not ok, "pcall should fail") + assert( + err:find("arg #2 `key` for function `get_stored_response_header` must be a string", nil, true), + "unexpected error message: " .. err + ) + end + } + + content_by_lua_block { + ngx.print("ok") + } + } +--- request +GET /t +--- response_body eval +"ok" +--- no_error_log +[error] diff --git a/t/03-dns-client/02-timer-usage.t b/t/03-dns-client/02-timer-usage.t index 73c35ccb1c4e..43572ad5058d 100644 --- a/t/03-dns-client/02-timer-usage.t +++ b/t/03-dns-client/02-timer-usage.t @@ -8,6 +8,7 @@ no_shuffle(); run_tests(); __DATA__ + === TEST 1: stale result triggers async timer --- config location = /t { diff --git a/t/04-patch/01-ngx-buf-double-free.t b/t/04-patch/01-ngx-buf-double-free.t index 10612e7912ec..699083db27c2 100644 --- a/t/04-patch/01-ngx-buf-double-free.t +++ b/t/04-patch/01-ngx-buf-double-free.t @@ -7,6 +7,7 @@ repeat_each(2); run_tests(); __DATA__ + === TEST 1: one buf was linked to multiple ngx_chain_t nodes --- config location /t { diff --git a/t/04-patch/02-ngx-read-body-block.t b/t/04-patch/02-ngx-read-body-block.t index a086b125704d..cfdc6cc650ff 100644 --- a/t/04-patch/02-ngx-read-body-block.t +++ b/t/04-patch/02-ngx-read-body-block.t @@ -28,6 +28,8 @@ true err: nil [error] [alert] + + === TEST 2: ngx.req.read_body() should work for HTTP2 POST requests that doesn't carry the content-length header --- config location = /test { @@ -46,4 +48,4 @@ Content-Length: true err: nil --- no_error_log [error] -[alert] \ No newline at end of file +[alert]