diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh deleted file mode 100755 index 447936f73ff6..000000000000 --- a/.ci/run_tests.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash -set -e - -function cyan() { - echo -e "\033[1;36m$*\033[0m" -} - -function red() { - echo -e "\033[1;31m$*\033[0m" -} - -function get_failed { - if [ ! -z "$FAILED_TEST_FILES_FILE" -a -f "$FAILED_TEST_FILES_FILE" ] - then - cat < $FAILED_TEST_FILES_FILE - else - echo "$@" - fi -} - -BUSTED_ARGS="--keep-going -o htest -v --exclude-tags=flaky,ipv6" -if [ ! -z "$FAILED_TEST_FILES_FILE" ] -then - BUSTED_ARGS="--helper=spec/busted-log-failed.lua $BUSTED_ARGS" -fi - -if [ "$KONG_TEST_DATABASE" == "postgres" ]; then - export TEST_CMD="bin/busted $BUSTED_ARGS,off" - - psql -v ON_ERROR_STOP=1 -h localhost --username "$KONG_TEST_PG_USER" <<-EOSQL - CREATE user ${KONG_TEST_PG_USER}_ro; - GRANT CONNECT ON DATABASE $KONG_TEST_PG_DATABASE TO ${KONG_TEST_PG_USER}_ro; - \c $KONG_TEST_PG_DATABASE; - GRANT USAGE ON SCHEMA public TO ${KONG_TEST_PG_USER}_ro; - ALTER DEFAULT PRIVILEGES FOR ROLE $KONG_TEST_PG_USER IN SCHEMA public GRANT SELECT ON TABLES TO ${KONG_TEST_PG_USER}_ro; -EOSQL - -elif [ "$KONG_TEST_DATABASE" == "cassandra" ]; then - echo "Cassandra is no longer supported" - exit 1 - -else - export TEST_CMD="bin/busted $BUSTED_ARGS,postgres,db" -fi - -if [ "$TEST_SUITE" == "integration" ]; then - if [[ "$TEST_SPLIT" == first* ]]; then - # GitHub Actions, run first batch of integration tests - files=$(ls -d spec/02-integration/* | sort | grep -v 05-proxy) - files=$(get_failed $files) - eval "$TEST_CMD" $files - - elif [[ "$TEST_SPLIT" == second* ]]; then - # GitHub Actions, run second batch of integration tests - # Note that the split here is chosen carefully to result - # in a similar run time between the two batches, and should - # be adjusted if imbalance become significant in the future - files=$(ls -d spec/02-integration/* | sort | grep 05-proxy) - files=$(get_failed $files) - eval "$TEST_CMD" $files - - else - # Non GitHub Actions - eval "$TEST_CMD" $(get_failed spec/02-integration/) - fi -fi - -if [ "$TEST_SUITE" == "dbless" ]; then - eval "$TEST_CMD" $(get_failed spec/02-integration/02-cmd \ - spec/02-integration/05-proxy \ - spec/02-integration/04-admin_api/02-kong_routes_spec.lua \ - spec/02-integration/04-admin_api/15-off_spec.lua \ - spec/02-integration/08-status_api/01-core_routes_spec.lua \ - spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua \ - spec/02-integration/11-dbless \ - spec/02-integration/20-wasm) -fi -if [ "$TEST_SUITE" == "plugins" ]; then - set +ex - rm -f .failed - - if [[ "$TEST_SPLIT" == first* ]]; then - # GitHub Actions, run first batch of plugin tests - PLUGINS=$(get_failed $(ls -d spec/03-plugins/* | head -n22)) - - elif [[ "$TEST_SPLIT" == second* ]]; then - # GitHub Actions, run second batch of plugin tests - # Note that the split here is chosen carefully to result - # in a similar run time between the two batches, and should - # be adjusted if imbalance become significant in the future - PLUGINS=$(get_failed $(ls -d spec/03-plugins/* | tail -n+23)) - - else - # Non GitHub Actions - PLUGINS=$(get_failed $(ls -d spec/03-plugins/*)) - fi - - for p in $PLUGINS; do - echo - cyan "--------------------------------------" - cyan $(basename $p) - cyan "--------------------------------------" - echo - - $TEST_CMD $p || echo "* $p" >> .failed - done - - if [[ "$TEST_SPLIT" != first* ]]; then - cat kong-*.rockspec | grep kong- | grep -v zipkin | grep -v sidecar | grep "~" | grep -v kong-prometheus-plugin | while read line ; do - REPOSITORY=`echo $line | sed "s/\"/ /g" | awk -F" " '{print $1}'` - VERSION=`luarocks show $REPOSITORY | grep $REPOSITORY | head -1 | awk -F" " '{print $2}' | cut -f1 -d"-"` - REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-prometheus-plugin/kong-plugin-prometheus/g'` - REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-proxy-cache-plugin/kong-plugin-proxy-cache/g'` - - echo - cyan "--------------------------------------" - cyan $REPOSITORY $VERSION - cyan "--------------------------------------" - echo - - git clone https://github.com/Kong/$REPOSITORY.git --branch $VERSION --single-branch /tmp/test-$REPOSITORY || \ - git clone https://github.com/Kong/$REPOSITORY.git --branch v$VERSION --single-branch /tmp/test-$REPOSITORY - sed -i 's/grpcbin:9000/localhost:15002/g' /tmp/test-$REPOSITORY/spec/*.lua - sed -i 's/grpcbin:9001/localhost:15003/g' /tmp/test-$REPOSITORY/spec/*.lua - cp -R /tmp/test-$REPOSITORY/spec/fixtures/* spec/fixtures/ || true - pushd /tmp/test-$REPOSITORY - luarocks make - popd - - $TEST_CMD /tmp/test-$REPOSITORY/spec/ || echo "* $REPOSITORY" >> .failed - - done - fi - - if [ -f .failed ]; then - echo - red "--------------------------------------" - red "Plugin tests failed:" - red "--------------------------------------" - cat .failed - exit 1 - else - exit 0 - fi -fi -if [ "$TEST_SUITE" == "pdk" ]; then - prove -I. -r t -fi -if [ "$TEST_SUITE" == "unit" ]; then - unset KONG_TEST_NGINX_USER KONG_PG_PASSWORD KONG_TEST_PG_PASSWORD - scripts/autodoc - bin/busted -v -o htest spec/01-unit - make lint -fi diff --git a/.ci/runtimes.json b/.ci/runtimes.json new file mode 100644 index 000000000000..d2b4b4ac9c5a --- /dev/null +++ b/.ci/runtimes.json @@ -0,0 +1 @@ +[{"suite":"plugins","filename":"spec/03-plugins/06-statsd/01-log_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/18-upstream_tls_spec.lua","expectedDuration":"154.40"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/04-round-robin_spec.lua","expectedDuration":"45.76"},{"suite":"plugins","filename":"spec/03-plugins/23-rate-limiting/04-access_spec.lua","expectedDuration":"139.37"},{"suite":"integration","filename":"spec/02-integration/05-proxy/06-ssl_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/05-cache-invalidation_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/02-start_stop_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/08-router_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/23-context_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/02-router_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/01-sync_spec.lua","expectedDuration":"26.29"},{"suite":"plugins","filename":"spec/03-plugins/03-http-log/01-log_spec.lua","expectedDuration":"24.28"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/05-ocsp_spec.lua","expectedDuration":"21.35"},{"suite":"plugins","filename":"spec/03-plugins/36-request-transformer/02-access_spec.lua","expectedDuration":"19.72"},{"suite":"integration","filename":"spec/02-integration/01-helpers/01-helpers_spec.lua","expectedDuration":"18.20"},{"suite":"integration","filename":"spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua","expectedDuration":"14.98"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/06-restart_spec.lua","expectedDuration":"14.46"},{"suite":"plugins","filename":"spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua","expectedDuration":"13.30"},{"suite":"unit","filename":"spec/01-unit/09-balancer/04-round_robin_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/07-latency_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/11-reports_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/02-udp-log/01-udp-log_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/03-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/14-tracing/03-tracer-pdk_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/09-key-auth/03-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/11-handler_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/06-worker_events_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/05-redis_storage_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/16-jwt/03-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/09-websockets_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/33-serverless-functions/03-dbless_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/25-oauth2/02-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/11-dbless/01-respawn_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/34-zipkin/zipkin_queue_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/18-acl/01-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/01-vault_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/10-basic-auth/04-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/09-filter-meta_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/03-http-log/04-legacy_queue_sharing_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/15-response-transformer/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/32-query-params_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/35-azure-functions/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/06-postgres_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/11-declarative_config/04-on-the-fly-migration_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/11-declarative_config/01-validate_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/03-plugins_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/11-reports_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/11-declarative_config/02-process_auto_fields_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/21-dns-client/01-utils_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/13-cors/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/31-proxy-cache/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/13-cluster_status_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/28-inject_confs_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/03-arguments_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/05-cache-invalidation_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/07-dao/01-plugins_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/25-upstream_keepalive_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/15-off_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/07-reports_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/06-timestamp_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/15-workspaces_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/31-proxy-cache/05-cache_key_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/24-runloop_certificate_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/19-hooks_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/06-statsd/03-allow_status_codes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua","expectedDuration":"253.25"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/09-node-id-persistence_spec.lua","expectedDuration":"30.43"},{"suite":"plugins","filename":"spec/03-plugins/23-rate-limiting/02-policies_spec.lua","expectedDuration":"24.18"},{"suite":"plugins","filename":"spec/03-plugins/25-oauth2/03-access_spec.lua","expectedDuration":"23.83"},{"suite":"plugins","filename":"spec/03-plugins/28-grpc-gateway/01-proxy_spec.lua","expectedDuration":"20.30"},{"suite":"plugins","filename":"spec/03-plugins/31-proxy-cache/02-access_spec.lua","expectedDuration":"18.79"},{"suite":"unit","filename":"spec/01-unit/27-queue_spec.lua","expectedDuration":"16.35"},{"suite":"integration","filename":"spec/02-integration/02-cmd/06-restart_spec.lua","expectedDuration":"14.63"},{"suite":"integration","filename":"spec/02-integration/05-proxy/33-request-id-header_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/02-log_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/27-unbuffered_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/20-ldap-auth/02-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/07-resurrect_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/06-clustering_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/06-regression_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/06-certificates_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/03-mock_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/06-invalidations/01-cluster_events_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/23-cors_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/31-proxy-cache/04-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/01-legacy_queue_parameter_warning_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/11-dbless/02-workers_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/01-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/17-admin_gui/01-admin-gui-path_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/10-services_routes_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/12-request-size-limiting/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/10-basic-auth/02-api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/16-jwt/02-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/05-cache_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/20-timers_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/03-custom-serve_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/13-plugin-endpoints_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/21-admin-api-keys_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/11-db_transformations_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/03-conf_loader_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/07-tags_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/24-response-rate-limiting/02-policies_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/10-declarative_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/01-otlp_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/23-vaults_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/04-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/33-serverless-functions/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/36-request-transformer/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/08-status_api/02-targets_routes_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/27-unbuffered_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/01-admin-api_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/06-clustering_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/18-upstream_tls_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/14-server_tokens_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/03-runtime_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/11-dbless/02-workers_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/03-typedefs_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/09-balancer/02-least_connections_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/11-snis_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/14-consumers_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/12-topological_sort_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/22-grpc-utils_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/25-msgpack_rpc_spec.lua","expectedDuration":"0.00"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua","expectedDuration":"41.69"},{"suite":"integration","filename":"spec/02-integration/02-cmd/10-migrations_spec.lua","expectedDuration":"40.56"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/04-exporter_spec.lua","expectedDuration":"27.07"},{"suite":"integration","filename":"spec/02-integration/14-tracing/01-instrumentations_spec.lua","expectedDuration":"24.56"},{"suite":"integration","filename":"spec/02-integration/05-proxy/28-stream_plugins_triggering_spec.lua","expectedDuration":"22.19"},{"suite":"integration","filename":"spec/02-integration/02-cmd/11-config_spec.lua","expectedDuration":"20.82"},{"suite":"plugins","filename":"spec/03-plugins/08-datadog/01-log_spec.lua","expectedDuration":"18.77"},{"suite":"plugins","filename":"spec/03-plugins/27-aws-lambda/06-request-util_spec.lua","expectedDuration":"17.51"},{"suite":"integration","filename":"spec/02-integration/02-cmd/13-signals_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/08-declarative_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/14-vault_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/19-grpc_proxy_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/12-plugins-conf_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/08-targets_routes_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/03-reload_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/01-helpers/03-http_mock_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/17-admin_gui/03-reports_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/01-admin-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/09-routes_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/02-least-connections_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/09-balancer/03-consistent_hashing_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/01-ctx_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/07-health_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/17-foreign-entity_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/05-metrics_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/26-tracing/01-tracer_pdk_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/08-declarative_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/05-syslog/01-log_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/15-response-transformer/05-big_response_body_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/30-session/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/06-refresh-secrets_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/32-grpc-web/01-proxy_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/36-request-transformer/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/19-vaults_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/09-prepare_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/12-hybrid_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/05-check_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/04-version_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/09-balancer/01-generic_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/02-metaschema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/06-routes_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/03-http-log/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/02-env_vault_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/19-hybrid/03-compat_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/19-hmac-auth/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/21-grpc_plugins_triggering_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/07-latency_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/33-request-aware-table_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/02-db_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/06-stream_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/27-lua-ssl-trusted-cert_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/09-websockets_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/26-tracing/02-propagation_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/02-rockspec_meta_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/14-dns_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/15-response-transformer/02-body_transformer_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/07-api_helpers_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/27-aws-lambda/05-aws-serializer_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/26-tracing/03-request-id_spec.lua","expectedDuration":"0.00"},{"suite":"integration","filename":"spec/02-integration/05-proxy/25-upstream_keepalive_spec.lua","expectedDuration":"41.70"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/03-consistent-hashing_spec.lua","expectedDuration":"34.52"},{"suite":"integration","filename":"spec/02-integration/02-cmd/03-reload_spec.lua","expectedDuration":"26.10"},{"suite":"unit","filename":"spec/01-unit/21-dns-client/02-client_spec.lua","expectedDuration":"22.58"},{"suite":"dbless","filename":"spec/02-integration/04-admin_api/15-off_spec.lua","expectedDuration":"20.47"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/01-proxy_spec.lua","expectedDuration":"18.97"},{"suite":"integration","filename":"spec/02-integration/05-proxy/04-plugins_triggering_spec.lua","expectedDuration":"16.46"},{"suite":"plugins","filename":"spec/03-plugins/30-session/02-kong_storage_adapter_spec.lua","expectedDuration":"14.26"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/05-otelcol_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/15-upstream-status-header_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/08-quit_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/05-dns_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/26-udp_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/17-ip-restriction/02-access_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/18-acl/02-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/03-consumers_routes_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/34-zipkin/zipkin_no_endpoint_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/22-reports_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/20-ttl-cleanup_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/03-runtime_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/24-buffered_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/03-pki_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/01-client_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/11-dbless/01-respawn_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/06-hybrid_mode_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/19-hmac-auth/02-api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/07-loggly/01-log_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/11-correlation-id/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/21-bot-detection/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/04-echo_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/15-response-transformer/04-filter_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/21-truncated_arguments_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/12-stream_api/01-stream_api_endpoint_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/10-db_unique_foreign_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/06-hybrid-mode_metrics_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/18-keys_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/19-key-sets_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/01-cmds_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/29-admin_gui/01-admin_gui_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/10-log_serializer_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/16-jwt/06-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/24-response-rate-limiting/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/03-http-log/03-schem-vault_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/29-admin_gui/02-admin_gui_template_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/11-dbless/03-config_persistence_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/08-uri_encoding_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/02-start_stop_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/19-grpc_proxy_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/31-stream_tls_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/30-max-args_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/09-balancer/05-worker_consistency_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/19-hybrid/02-clustering_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/20-sandbox_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/15-utils_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/30-session/03-session_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/18-tools_uri_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/10-basic-auth/01-crypto_spec.lua","expectedDuration":"0.00"},{"suite":"integration","filename":"spec/02-integration/21-request-debug/01-request-debug_spec.lua","expectedDuration":"56.23"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/04-status_api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/02-access_spec.lua","expectedDuration":"35.66"},{"suite":"integration","filename":"spec/02-integration/05-proxy/33-request-aware-table_spec.lua","expectedDuration":"33.04"},{"suite":"plugins","filename":"spec/03-plugins/27-aws-lambda/08-sam-integration_spec.lua","expectedDuration":"21.91"},{"suite":"integration","filename":"spec/02-integration/14-tracing/04-trace-ids-log_spec.lua","expectedDuration":"21.28"},{"suite":"integration","filename":"spec/02-integration/06-invalidations/04-balancer_cache_correctness_spec.lua","expectedDuration":"20.86"},{"suite":"integration","filename":"spec/02-integration/05-proxy/01-proxy_spec.lua","expectedDuration":"18.93"},{"suite":"plugins","filename":"spec/03-plugins/24-response-rate-limiting/05-integration_spec.lua","expectedDuration":"15.97"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/07-optional_fields_spec.lua","expectedDuration":"14.79"},{"suite":"plugins","filename":"spec/03-plugins/20-ldap-auth/01-access_spec.lua","expectedDuration":"14.78"},{"suite":"integration","filename":"spec/02-integration/05-proxy/30-max-args_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/04-file-log/01-log_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/03-propagation_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/15-connection_pool_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/02-kong_routes_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/04-prefix_handler_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/18-acl/03-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/11-postgres-ro_spec.lua","expectedDuration":"6.52"},{"suite":"plugins","filename":"spec/03-plugins/09-key-auth/01-api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/19-hmac-auth/03-access_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/21-bot-detection/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/05-recreate-request_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/14-tracing/02-propagation_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/09-prepare_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/25-oauth2/04-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/10-basic-auth/05-declarative_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/07-upstream_timeouts_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/15-plugins-iterator/02-correctness_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/14-request-termination/03-integration_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/11-config_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/16-custom_nginx_directive_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/29-collect-plugin-errors_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/16-custom_nginx_directive_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/04-db_cluster_mutex_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/16-jwt/01-jwt_parser_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/02-db_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/05-check_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/04-version_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/01-cmds_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/13-cluster_status_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/06-statsd/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/27-aws-lambda/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/14-request-termination/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/07-plugins_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/02-db-errors_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/24-buffered_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/05-dns_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/20-wasm/04-proxy-wasm_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/28-stream_plugins_triggering_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/03-upstream_headers_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/26-udp_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/11-handler_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/09-upstreams_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/16-runloop_handler_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/15-utils_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/09-no_broadcast_crud_event_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/20-ldap-auth/03-decode_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/17-concurrency_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/26-mime-type_spec.lua","expectedDuration":"0.00"},{"suite":"integration","filename":"spec/02-integration/05-proxy/31-stream_tls_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/22-debug_spec.lua","expectedDuration":"35.81"},{"suite":"plugins","filename":"spec/03-plugins/23-rate-limiting/05-integration_spec.lua","expectedDuration":"30.83"},{"suite":"integration","filename":"spec/02-integration/05-proxy/21-grpc_plugins_triggering_spec.lua","expectedDuration":"24.60"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/02-router_spec.lua","expectedDuration":"22.94"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/04-round-robin_spec.lua","expectedDuration":"20.62"},{"suite":"plugins","filename":"spec/03-plugins/27-aws-lambda/99-access_spec.lua","expectedDuration":"18.71"},{"suite":"plugins","filename":"spec/03-plugins/09-key-auth/02-access_spec.lua","expectedDuration":"17.27"},{"suite":"unit","filename":"spec/01-unit/09-balancer/06-latency_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/33-request-id-header_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/24-reconfiguration-completion_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/08-quit_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/11-dbless/03-config_persistence_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/11-status_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/12-hybrid_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/16-jwt/04-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/04-cp_cluster_sync_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/08-lazy_export_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/10-balancer/06-stream_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/33-serverless-functions/02-access_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/29-acme/02-kong_storage_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/01-tcp-log/01-tcp-log_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/02-cmd/07-health_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/07-reports_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/07-upstreams_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/09-query-semaphore_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/09-config-compat_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/31-proxy-cache/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/23-rate-limiting/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/14-request-termination/02-access_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/08-uri_encoding_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/13-error_handlers_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/13-error_handlers_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/17-admin_gui/02-log_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/02-db_core_entities_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/11-declarative_config/03-flatten_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/25-oauth2/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/22-ca_certificates_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/21-services_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/10-migrations_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/23-rate-limiting/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/37-opentelemetry/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/20-ldap-auth/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/05-recreate-request_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/14-vault_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/33-serverless-functions/04-phases_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/04-plugins_triggering_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/32-query-params_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/29-collect-plugin-errors_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/24-response-rate-limiting/04-access_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/22-reports_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/05-services_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/05-utils_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/07-dao/02-tags_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/04-file-log/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/10-migrations_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/28-plugins-iterator/02-lookup_cfg_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/34-zipkin/request_tags_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/14-server_tokens_spec.lua","expectedDuration":"42.10"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/03-consistent-hashing_spec.lua","expectedDuration":"35.45"},{"suite":"plugins","filename":"spec/03-plugins/34-zipkin/zipkin_spec.lua","expectedDuration":"26.18"},{"suite":"integration","filename":"spec/02-integration/06-invalidations/03-plugins_iterator_invalidation_spec.lua","expectedDuration":"23.89"},{"suite":"integration","filename":"spec/02-integration/10-go_plugins/01-reports_spec.lua","expectedDuration":"20.97"},{"suite":"integration","filename":"spec/02-integration/05-proxy/03-upstream_headers_spec.lua","expectedDuration":"19.23"},{"suite":"integration","filename":"spec/02-integration/15-plugins-iterator/01-precedence_spec.lua","expectedDuration":"18.18"},{"suite":"integration","filename":"spec/02-integration/16-queues/01-shutdown_spec.lua","expectedDuration":"15.09"},{"suite":"dbless","filename":"spec/02-integration/02-cmd/13-signals_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/08-declarative_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/15-upstream-status-header_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/08-status_api/01-core_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/10-basic-auth/03-access_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/04-admin_api/02-kong_routes_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/09-filter-meta_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/12-error_default_type_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/20-wasm/04-proxy-wasm_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/05-proxy/27-lua-ssl-trusted-cert_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/13-vaults/05-ttl_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/03-cluster_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/14-tags_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/01-admin_api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/16-ca_certificates_routes_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/13-cors/01-access_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/29-lua_cjson_large_str_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/04-admin_api/04-plugins_routes_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/19-hmac-auth/04-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/21-bot-detection/02-invalidations_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/01-db_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/24-response-rate-limiting/03-api_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/04-plugin-config_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/07-sdk/05-pdk_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/21-dns-client/03-client_cache_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/06-migrations_state_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/14-dao_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/01-helpers/02-blueprints_spec.lua","expectedDuration":"NaN"},{"suite":"integration","filename":"spec/02-integration/03-db/12-dao_hooks_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/08-cache_warmup_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/17-ip-restriction/01-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/08-datadog/02-schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/34-zipkin/schema_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/12-plugins_order_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/13-plugins_version_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/10-balancer/02-least-connections_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/12-error_default_type_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/07-upstream_timeouts_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/23-context_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/08-status_api/01-core_routes_spec.lua","expectedDuration":"NaN"},{"suite":"dbless","filename":"spec/02-integration/05-proxy/06-ssl_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/07-db_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/04-dao_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/08-targets_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/15-response-transformer/01-header_transformer_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/01-db/01-schema/04-entities_schema_spec.lua","expectedDuration":"NaN"},{"suite":"plugins","filename":"spec/03-plugins/26-prometheus/08-unit_spec.lua","expectedDuration":"NaN"},{"suite":"unit","filename":"spec/01-unit/28-plugins-iterator/01-compound_key_spec.lua","expectedDuration":"NaN"}] \ No newline at end of file diff --git a/.ci/test_suites.json b/.ci/test_suites.json new file mode 100644 index 000000000000..eb6b15e5909e --- /dev/null +++ b/.ci/test_suites.json @@ -0,0 +1,34 @@ +[ + { + "name": "unit", + "exclude_tags": "flaky,ipv6", + "specs": ["spec/01-unit/"] + }, + { + "name": "integration", + "exclude_tags": "flaky,ipv6,off", + "environment": { + "KONG_TEST_DATABASE": "postgres" + }, + "specs": ["spec/02-integration/"] + }, + { + "name": "dbless", + "exclude_tags": "flaky,ipv6,postgres,db", + "specs": [ + "spec/02-integration/02-cmd/", + "spec/02-integration/05-proxy/", + "spec/02-integration/04-admin_api/02-kong_routes_spec.lua", + "spec/02-integration/04-admin_api/15-off_spec.lua", + "spec/02-integration/08-status_api/01-core_routes_spec.lua", + "spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua", + "spec/02-integration/11-dbless/", + "spec/02-integration/20-wasm/" + ] + }, + { + "name": "plugins", + "exclude_tags": "flaky,ipv6", + "specs": ["spec/03-plugins/"] + } +] diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8b3c77ccf375..e202558a7db9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -33,6 +33,7 @@ concurrency: env: BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build KONG_TEST_COVERAGE: ${{ inputs.coverage == true || github.event_name == 'schedule' }} + RUNNER_COUNT: 7 jobs: build: @@ -40,22 +41,11 @@ jobs: with: relative-build-root: bazel-bin/build - lint-doc-and-unit-tests: - name: Lint, Doc and Unit tests + lint-and-doc-tests: + name: Lint and Doc tests runs-on: ubuntu-22.04 needs: build - services: - postgres: - image: postgres:13 - env: - POSTGRES_USER: kong - POSTGRES_DB: kong - POSTGRES_HOST_AUTH_METHOD: trust - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8 - steps: - name: Checkout Kong source code uses: actions/checkout@v4 @@ -93,41 +83,51 @@ jobs: - name: Check labeler configuration run: scripts/check-labeler.pl .github/labeler.yml - - name: Unit tests - env: - KONG_TEST_PG_DATABASE: kong - KONG_TEST_PG_USER: kong - run: | - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - TEST_CMD="bin/busted -v -o htest spec/01-unit" - if [[ $KONG_TEST_COVERAGE = true ]]; then - TEST_CMD="$TEST_CMD --coverage" - fi - $TEST_CMD + schedule: + name: Schedule busted tests to run + runs-on: ubuntu-22.04 + needs: build - - name: Archive coverage stats file + env: + WORKFLOW_ID: ${{ github.run_id }} + + outputs: + runners: ${{ steps.generate-runner-array.outputs.RUNNERS }} + + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Schedule tests + uses: Kong/gateway-test-scheduler/schedule@main + with: + test-suites-file: .ci/test_suites.json + test-file-runtime-file: .ci/runtimes.json + output-prefix: test-chunk. + runner-count: ${{ env.RUNNER_COUNT }} + + - name: Upload schedule files uses: actions/upload-artifact@v3 - if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} + continue-on-error: true with: - name: luacov-stats-out-${{ github.job }}-${{ github.run_id }} - retention-days: 1 - path: | - luacov.stats.out + name: schedule-test-files + path: test-chunk.* + retention-days: 7 - - name: Get kernel message - if: failure() + - name: Generate runner array + id: generate-runner-array run: | - sudo dmesg -T + echo "RUNNERS=[$(echo $(seq 1 $(( $RUNNER_COUNT ))))]" | sed -e 's/ /, /g' >> $GITHUB_OUTPUT - integration-tests-postgres: - name: Postgres ${{ matrix.suite }} - ${{ matrix.split }} tests + busted-tests: + name: Busted test runner ${{ matrix.runner }} runs-on: ubuntu-22.04 - needs: build + needs: [build,schedule] + strategy: fail-fast: false matrix: - suite: [integration, plugins] - split: [first, second] + runner: ${{ fromJSON(needs.schedule.outputs.runners) }} services: postgres: @@ -179,7 +179,6 @@ jobs: echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts - name: Enable SSL for Redis - if: ${{ matrix.suite == 'plugins' }} run: | docker cp ${{ github.workspace }} kong_redis:/workspace docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh @@ -202,47 +201,54 @@ jobs: docker logs opentelemetry-collector - name: Install AWS SAM cli tool - if: ${{ matrix.suite == 'plugins' }} run: | curl -L -s -o /tmp/aws-sam-cli.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip unzip -o /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli sudo /tmp/aws-sam-cli/install --update - - name: Update PATH - run: | - echo "$BUILD_ROOT/kong-dev/bin" >> $GITHUB_PATH - echo "$BUILD_ROOT/kong-dev/openresty/nginx/sbin" >> $GITHUB_PATH - echo "$BUILD_ROOT/kong-dev/openresty/bin" >> $GITHUB_PATH - - - name: Debug (nginx) - run: | - echo nginx: $(which nginx) - nginx -V 2>&1 | sed -re 's/ --/\n--/g' - ldd $(which nginx) - - - name: Debug (luarocks) + - name: Create kong_ro user in Postgres run: | - echo luarocks: $(which luarocks) - luarocks --version - luarocks config + psql -v ON_ERROR_STOP=1 -h localhost --username kong <<\EOD + CREATE user kong_ro; + GRANT CONNECT ON DATABASE kong TO kong_ro; + \c kong; + GRANT USAGE ON SCHEMA public TO kong_ro; + ALTER DEFAULT PRIVILEGES FOR ROLE kong IN SCHEMA public GRANT SELECT ON TABLES TO kong_ro; + EOD - name: Tune up postgres max_connections run: | # arm64 runners may use more connections due to more worker cores psql -hlocalhost -Ukong kong -tAc 'alter system set max_connections = 5000;' - - name: Generate test rerun filename + - name: Download test schedule file + uses: actions/download-artifact@v3 + continue-on-error: true + with: + name: schedule-test-files + + - name: Generate helper environment variables run: | - echo FAILED_TEST_FILES_FILE=$(echo '${{ github.run_id }}-${{ matrix.suite }}-${{ matrix.split }}' | tr A-Z a-z | sed -Ee 's/[^a-z0-9]+/-/g').txt >> $GITHUB_ENV + echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV + echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV + - name: Build & install dependencies + run: | + make dev - name: Download test rerun information uses: actions/download-artifact@v3 continue-on-error: true with: - name: ${{ env.FAILED_TEST_FILES_FILE }} + name: test-rerun-info-${{ matrix.runner }} - - name: Tests + - name: Download test runtime statistics from previous runs + uses: actions/download-artifact@v3 + continue-on-error: true + with: + name: test-runtime-statistics-${{ matrix.runner }} + + - name: Run Tests env: KONG_TEST_PG_DATABASE: kong KONG_TEST_PG_USER: kong @@ -250,108 +256,44 @@ jobs: KONG_SPEC_TEST_GRPCBIN_PORT: "15002" KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json - TEST_SUITE: ${{ matrix.suite }} - TEST_SPLIT: ${{ matrix.split }} - run: | - make dev # required to install other dependencies like bin/grpcurl - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - .ci/run_tests.sh + DD_ENV: ci + DD_SERVICE: kong-ce-ci + DD_CIVISIBILITY_MANUAL_API_ENABLED: 1 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_TRACE_GIT_METADATA_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + uses: Kong/gateway-test-scheduler/runner@main + with: + tests-to-run-file: test-chunk.${{ matrix.runner }}.json + failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }} + test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }} + setup-venv: . ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - name: Upload test rerun information if: always() uses: actions/upload-artifact@v3 with: - name: ${{ env.FAILED_TEST_FILES_FILE }} + name: test-rerun-info-${{ matrix.runner }} path: ${{ env.FAILED_TEST_FILES_FILE }} retention-days: 2 - - name: Archive coverage stats file + - name: Upload test runtime statistics for offline scheduling + if: always() uses: actions/upload-artifact@v3 - if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} with: - name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.suite }}-${{ contains(matrix.split, 'first') && '1' || '2' }} - retention-days: 1 - path: | - luacov.stats.out - - - name: Get kernel message - if: failure() - run: | - sudo dmesg -T - - integration-tests-dbless: - name: DB-less integration tests - runs-on: ubuntu-22.04 - needs: build - - services: - grpcbin: - image: kong/grpcbin - ports: - - 15002:9000 - - 15003:9001 - - steps: - - name: Checkout Kong source code - uses: actions/checkout@v4 - - - name: Lookup build cache - id: cache-deps - uses: actions/cache@v3 - with: - path: ${{ env.BUILD_ROOT }} - key: ${{ needs.build.outputs.cache-key }} - - - name: Build WASM Test Filters - uses: ./.github/actions/build-wasm-test-filters - - - name: Add gRPC test host names - run: | - echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts - echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts - - - name: Run OpenTelemetry Collector - run: | - mkdir -p ${{ github.workspace }}/tmp/otel - touch ${{ github.workspace }}/tmp/otel/file_exporter.json - sudo chmod 777 -R ${{ github.workspace }}/tmp/otel - docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \ - -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \ - -v ${{ github.workspace }}/tmp/otel:/etc/otel \ - --name opentelemetry-collector -d \ - otel/opentelemetry-collector-contrib:0.52.0 \ - --config=/etc/otel-collector-config.yaml - sleep 2 - docker logs opentelemetry-collector - - - name: Tests - env: - KONG_TEST_PG_DATABASE: kong - KONG_TEST_PG_USER: kong - KONG_TEST_DATABASE: 'off' - KONG_SPEC_TEST_GRPCBIN_PORT: "15002" - KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" - KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json - TEST_SUITE: dbless - run: | - make dev # required to install other dependencies like bin/grpcurl - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - .ci/run_tests.sh + name: test-runtime-statistics-${{ matrix.runner }} + path: ${{ env.TEST_FILE_RUNTIME_FILE }} + retention-days: 7 - name: Archive coverage stats file uses: actions/upload-artifact@v3 if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} with: - name: luacov-stats-out-${{ github.job }}-${{ github.run_id }} + name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.runner }} retention-days: 1 path: | luacov.stats.out - - name: Get kernel message - if: failure() - run: | - sudo dmesg -T - pdk-tests: name: PDK tests runs-on: ubuntu-22.04 @@ -388,7 +330,7 @@ jobs: export PDK_LUACOV=1 fi eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) - .ci/run_tests.sh + prove -I. -r t - name: Archive coverage stats file uses: actions/upload-artifact@v3 @@ -404,9 +346,9 @@ jobs: run: | sudo dmesg -T - aggregator: - needs: [lint-doc-and-unit-tests,pdk-tests,integration-tests-postgres,integration-tests-dbless] - name: Luacov stats aggregator + cleanup-and-aggregate-stats: + needs: [lint-and-doc-tests,pdk-tests,busted-tests] + name: Cleanup and Luacov stats aggregator if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} runs-on: ubuntu-22.04 diff --git a/.github/workflows/update-test-runtime-statistics.yml b/.github/workflows/update-test-runtime-statistics.yml new file mode 100644 index 000000000000..0a24a01cde3e --- /dev/null +++ b/.github/workflows/update-test-runtime-statistics.yml @@ -0,0 +1,75 @@ +name: Update test runtime statistics file for test scheduling +on: + workflow_dispatch: + schedule: + - cron: "1 0 * * SAT" + # push rule below needed for testing only + push: + branches: + - feat/test-run-scheduler + +jobs: + process-statistics: + name: Download statistics from GitHub and combine them + runs-on: ubuntu-22.04 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + with: + token: ${{ secrets.PAT }} + + - name: Process statistics + uses: Kong/gateway-test-scheduler/analyze@main + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + with: + workflow-name: build_and_test.yml + test-file-runtime-file: .ci/runtimes.json + artifact-name-regexp: "^test-runtime-statistics-\\d+$" + + - name: Commit + env: + GIT_COMMITTER_NAME: "GitHub Actions Bot" + GIT_COMMITTER_EMAIL: "" + GIT_AUTHOR_NAME: "GitHub Actions Bot" + GIT_AUTHOR_EMAIL: "" + run: | + git commit -m "chore(ci): updated test file runtime file" .ci/runtimes.json + + - name: Create PR message + run: | + FIRST_LINE=$(git diff-tree -r -c -M -C --no-commit-id HEAD | head -n 1) + BLOB_HASH=$(echo $FIRST_LINE | awk '{print $4}') + SIZE=$(echo $BLOB_HASH | git cat-file --batch-check | grep "blob" | awk '{ print $3 }') + PRINTABLE_SIZE=$(numfmt --to=iec --suffix=B --format="%.1f" "$SIZE") + + cat > /tmp/commit-message <" + git push -f + git checkout main + git branch -d chore/update-runtimes + \`\`\` + EOD + + - name: Delete previous branch if it exists + run: git push origin :chore/update-runtimes + + - name: Push to branch + run: git push -f origin HEAD:chore/update-runtimes + + - name: Create PR + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + run: | + gh pr create --title "chore(ci): update test runtimes" \ + --body-file /tmp/commit-message \ + --base master \ + --head chore/update-runtimes diff --git a/spec/01-unit/19-hybrid/03-compat_spec.lua b/spec/01-unit/19-hybrid/03-compat_spec.lua index 48085ab24ecf..b2a0030aa0f0 100644 --- a/spec/01-unit/19-hybrid/03-compat_spec.lua +++ b/spec/01-unit/19-hybrid/03-compat_spec.lua @@ -390,7 +390,7 @@ describe("kong.clustering.compat", function() end end) - it(function() + it("has_update", function() local config = { config_table = declarative.export_config() } local has_update = compat.update_compatible_payload(config, "3.0.0", "test_") assert.truthy(has_update) @@ -561,7 +561,7 @@ describe("kong.clustering.compat", function() config = { config_table = declarative.export_config() } end) - it(function() + it("plugin.use_srv_name", function() local has_update, result = compat.update_compatible_payload(config, "3.0.0", "test_") assert.truthy(has_update) result = cjson_decode(inflate_gzip(result)).config_table diff --git a/spec/01-unit/29-admin_gui/02-admin_gui_template_spec.lua b/spec/01-unit/29-admin_gui/02-admin_gui_template_spec.lua index 9a3df93ab523..de4c337fda36 100644 --- a/spec/01-unit/29-admin_gui/02-admin_gui_template_spec.lua +++ b/spec/01-unit/29-admin_gui/02-admin_gui_template_spec.lua @@ -57,6 +57,7 @@ describe("admin_gui template", function() setup(function() prefix_handler.prepare_prefixed_interface_dir("/usr/local/kong", "gui", conf) + os.execute("mkdir -p " .. mock_prefix) assert(pl_path.isdir(mock_prefix)) end) @@ -138,6 +139,7 @@ describe("admin_gui template", function() setup(function() prefix_handler.prepare_prefixed_interface_dir("/usr/local/kong", "gui", conf) + os.execute("mkdir -p " .. mock_prefix) assert(pl_path.isdir(mock_prefix)) end) @@ -183,7 +185,7 @@ describe("admin_gui template", function() conf.prefix = mock_prefix if not pl_path.exists(usr_interface_path) then - assert(pl_path.mkdir(usr_interface_path)) + os.execute("mkdir -p " .. usr_interface_path) end end) diff --git a/spec/02-integration/17-admin_gui/02-log_spec.lua b/spec/02-integration/17-admin_gui/02-log_spec.lua index 226ff7d17901..e1b0176129ee 100644 --- a/spec/02-integration/17-admin_gui/02-log_spec.lua +++ b/spec/02-integration/17-admin_gui/02-log_spec.lua @@ -6,6 +6,7 @@ for _, strategy in helpers.each_strategy() do describe("Admin API - GUI logs - kong_admin #" .. strategy, function () lazy_setup(function () + helpers.get_db_utils(strategy) -- clear db assert(helpers.start_kong({ strategy = strategy, prefix = "servroot", diff --git a/spec/03-plugins/37-opentelemetry/05-otelcol_spec.lua b/spec/03-plugins/37-opentelemetry/05-otelcol_spec.lua index c27f4d3663bc..931eebcb9dda 100644 --- a/spec/03-plugins/37-opentelemetry/05-otelcol_spec.lua +++ b/spec/03-plugins/37-opentelemetry/05-otelcol_spec.lua @@ -75,6 +75,7 @@ for _, strategy in helpers.each_strategy() do lazy_setup(function() -- clear file + os.execute("mkdir -p $(dirname " .. OTELCOL_FILE_EXPORTER_PATH .. ")") os.execute("cat /dev/null > " .. OTELCOL_FILE_EXPORTER_PATH) setup_instrumentations("all") end) diff --git a/spec/busted-ci-helper.lua b/spec/busted-ci-helper.lua new file mode 100644 index 000000000000..ff85767086ff --- /dev/null +++ b/spec/busted-ci-helper.lua @@ -0,0 +1,59 @@ +-- busted-log-failed.lua + +-- Log which test files run by busted had failures or errors in a +-- file. The file to use for logging is specified in the +-- FAILED_TEST_FILES_FILE environment variable. This is used to +-- reduce test rerun times for flaky tests. + +local busted = require 'busted' +local cjson = require 'cjson' +local socket_unix = require 'socket.unix' + +local busted_event_path = os.getenv("BUSTED_EVENT_PATH") + +-- Function to recursively copy a table, skipping keys associated with functions +local function copyTable(original, copied) + copied = copied or {} + + for key, value in pairs(original) do + if type(value) == "table" then + copied[key] = copyTable(value, {}) + elseif type(value) ~= "function" then + copied[key] = value + end + end + + return copied +end + +if busted_event_path then + local sock = assert(socket_unix()) + assert(sock:connect(busted_event_path)) + + local events = {{ 'suite', 'reset' }, + { 'suite', 'start' }, + { 'suite', 'end' }, + { 'file', 'start' }, + { 'file', 'end' }, + { 'test', 'start' }, + { 'test', 'end' }, + { 'pending' }, + { 'failure', 'it' }, + { 'error', 'it' }, + { 'failure' }, + { 'error' }} + for _, event in ipairs(events) do + busted.subscribe(event, function (...) + local args = {} + for i, original in ipairs{...} do + if type(original) == "table" then + args[i] = copyTable(original) + elseif type(original) ~= "function" then + args[i] = original + end + end + + sock:send(cjson.encode({ event = event[1] .. (event[2] and ":" .. event[2] or ""), args = args }) .. "\n") + end) + end +end diff --git a/spec/busted-log-failed.lua b/spec/busted-log-failed.lua deleted file mode 100644 index 7bfe6804b83f..000000000000 --- a/spec/busted-log-failed.lua +++ /dev/null @@ -1,33 +0,0 @@ --- busted-log-failed.lua - --- Log which test files run by busted had failures or errors in a --- file. The file to use for logging is specified in the --- FAILED_TEST_FILES_FILE environment variable. This is used to --- reduce test rerun times for flaky tests. - -local busted = require 'busted' -local failed_files_file = assert(os.getenv("FAILED_TEST_FILES_FILE"), - "FAILED_TEST_FILES_FILE environment variable not set") - -local FAILED_FILES = {} - -busted.subscribe({ 'failure' }, function(element, parent, message, debug) - FAILED_FILES[element.trace.source] = true -end) - -busted.subscribe({ 'error' }, function(element, parent, message, debug) - FAILED_FILES[element.trace.source] = true -end) - -busted.subscribe({ 'suite', 'end' }, function(suite, count, total) - local output = assert(io.open(failed_files_file, "w")) - if next(FAILED_FILES) then - for failed_file in pairs(FAILED_FILES) do - if failed_file:sub(1, 1) == '@' then - failed_file = failed_file:sub(2) - end - assert(output:write(failed_file .. "\n")) - end - end - output:close() -end) diff --git a/spec/fixtures/aws-sam.lua b/spec/fixtures/aws-sam.lua index 5aa67f972eab..6316f7c574c4 100644 --- a/spec/fixtures/aws-sam.lua +++ b/spec/fixtures/aws-sam.lua @@ -1,4 +1,5 @@ --AWS SAM Local Test Helper +local ngx_pipe = require "ngx.pipe" local helpers = require "spec.helpers" local utils = require "spec.helpers.perf.utils" local fmt = string.format @@ -26,6 +27,9 @@ function _M.is_sam_installed() end +local sam_proc + + function _M.start_local_lambda() local port = helpers.get_available_port() if not port then @@ -33,9 +37,16 @@ function _M.start_local_lambda() end -- run in background - local _ = ngx.thread.spawn(function() - utils.execute("sam local start-lambda --template-file=spec/fixtures/sam-app/template.yaml --port " .. port) - end) + local err + sam_proc, err = ngx_pipe.spawn({"sam", + "local", + "start-lambda", + "--template-file", "spec/fixtures/sam-app/template.yaml", + "--port", port + }) + if not sam_proc then + return nil, err + end local ret, err = utils.execute("pgrep -f 'sam local'") if err then @@ -47,9 +58,12 @@ end function _M.stop_local_lambda() - local ret, err = utils.execute("pkill -f sam") - if err then - return nil, fmt("Stop SAM CLI failed(code: %s): %s", err, ret) + if sam_proc then + local ok, err = sam_proc:kill(15) + if not ok then + return nil, fmt("Stop SAM CLI failed: %s", err) + end + sam_proc = nil end return true