From b059eb3ff5d7e15bd933b4325d56d166c9f5cba7 Mon Sep 17 00:00:00 2001 From: Qi Date: Mon, 4 Mar 2024 13:15:21 +0000 Subject: [PATCH] old PoC --- .bazelrc | 4 + .github/workflows/build_and_test.yml | 438 ------- .github/workflows/release.yml | 195 --- Dockerfile | 27 + WORKSPACE | 12 + build/BUILD.bazel | 10 + build/openresty/repositories.bzl | 2 + build/openresty/resty_protobuf/BUILD.bazel | 0 .../resty_protobuf_repositories.bzl | 13 + docker-entrypoint.sh | 77 ++ kong-3.8.0-0.rockspec | 7 + kong/etrace/hooks/dns.lua | 64 + kong/etrace/hooks/init.lua | 30 + kong/etrace/hooks/redis.lua | 57 + kong/etrace/init.lua | 582 +++++++++ kong/etrace/queue.lua | 129 ++ kong/globalpatches.lua | 3 + kong/init.lua | 75 ++ kong/runloop/handler.lua | 4 + kong/templates/nginx.lua | 1 + lua-resty-protobuf/.bazelrc | 1 + lua-resty-protobuf/.bazelversion | 1 + .../.github/workflows/bazel.yml | 40 + lua-resty-protobuf/.gitignore | 2 + lua-resty-protobuf/BUILD | 36 + lua-resty-protobuf/Cargo.lock | 1151 +++++++++++++++++ lua-resty-protobuf/Cargo.toml | 20 + lua-resty-protobuf/Makefile | 14 + lua-resty-protobuf/WORKSPACE | 13 + lua-resty-protobuf/build/BUILD.bazel | 1 + lua-resty-protobuf/build/README.md | 44 + lua-resty-protobuf/build/crates.bzl | 6 + lua-resty-protobuf/build/deps.bzl | 39 + lua-resty-protobuf/build/repos.bzl | 19 + lua-resty-protobuf/cbindgen.toml | 5 + .../lualib/resty/kong/kproto/cdefs.lua | 333 +++++ .../lualib/resty/kong/kproto/log.lua | 76 ++ .../lualib/resty/kong/kproto/metric.lua | 55 + .../lualib/resty/kong/kproto/trace.lua | 92 ++ lua-resty-protobuf/src/ffi/logs.rs | 141 ++ lua-resty-protobuf/src/ffi/metrics.rs | 87 ++ lua-resty-protobuf/src/ffi/mod.rs | 4 + lua-resty-protobuf/src/ffi/trace.rs | 187 +++ lua-resty-protobuf/src/ffi/utils.rs | 13 + lua-resty-protobuf/src/lib.rs | 413 ++++++ .../fixtures/ubuntu-22.04-amd64.txt | 16 + 46 files changed, 3906 insertions(+), 633 deletions(-) delete mode 100644 .github/workflows/build_and_test.yml create mode 100644 Dockerfile create mode 100644 build/openresty/resty_protobuf/BUILD.bazel create mode 100644 build/openresty/resty_protobuf/resty_protobuf_repositories.bzl create mode 100644 docker-entrypoint.sh create mode 100644 kong/etrace/hooks/dns.lua create mode 100644 kong/etrace/hooks/init.lua create mode 100644 kong/etrace/hooks/redis.lua create mode 100644 kong/etrace/init.lua create mode 100644 kong/etrace/queue.lua create mode 100644 lua-resty-protobuf/.bazelrc create mode 100644 lua-resty-protobuf/.bazelversion create mode 100644 lua-resty-protobuf/.github/workflows/bazel.yml create mode 100644 lua-resty-protobuf/.gitignore create mode 100644 lua-resty-protobuf/BUILD create mode 100644 lua-resty-protobuf/Cargo.lock create mode 100644 lua-resty-protobuf/Cargo.toml create mode 100644 lua-resty-protobuf/Makefile create mode 100644 lua-resty-protobuf/WORKSPACE create mode 100644 lua-resty-protobuf/build/BUILD.bazel create mode 100644 lua-resty-protobuf/build/README.md create mode 100644 lua-resty-protobuf/build/crates.bzl create mode 100644 lua-resty-protobuf/build/deps.bzl create mode 100644 lua-resty-protobuf/build/repos.bzl create mode 100644 lua-resty-protobuf/cbindgen.toml create mode 100644 lua-resty-protobuf/lualib/resty/kong/kproto/cdefs.lua create mode 100644 lua-resty-protobuf/lualib/resty/kong/kproto/log.lua create mode 100644 lua-resty-protobuf/lualib/resty/kong/kproto/metric.lua create mode 100644 lua-resty-protobuf/lualib/resty/kong/kproto/trace.lua create mode 100644 lua-resty-protobuf/src/ffi/logs.rs create mode 100644 lua-resty-protobuf/src/ffi/metrics.rs create mode 100644 lua-resty-protobuf/src/ffi/mod.rs create mode 100644 lua-resty-protobuf/src/ffi/trace.rs create mode 100644 lua-resty-protobuf/src/ffi/utils.rs create mode 100644 lua-resty-protobuf/src/lib.rs diff --git a/.bazelrc b/.bazelrc index 7a1625ac970f..b59d548b4488 100644 --- a/.bazelrc +++ b/.bazelrc @@ -40,6 +40,10 @@ build --features=-debug_prefix_map_pwd_is_dot build --action_env=BUILD_NAME=kong-dev build --action_env=INSTALL_DESTDIR=MANAGED build --strip=never +build --compilation_mode=opt +build --copt="-g" +build --//:debug=false +build --//:licensing=false # Release flags build:release --//:debug=false diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index bc58a03990d5..000000000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,438 +0,0 @@ -name: Build & Test -on: - pull_request: - paths-ignore: - # ignore markdown files (CHANGELOG.md, README.md, etc.) - - '**/*.md' - - '.github/workflows/release.yml' - - 'changelog/**' - - 'kong.conf.default' - push: - paths-ignore: - # ignore markdown files (CHANGELOG.md, README.md, etc.) - - '**/*.md' - # ignore PRs for the generated COPYRIGHT file - - 'COPYRIGHT' - branches: - - master - - release/* - - test-please/* - workflow_dispatch: - inputs: - coverage: - description: 'Coverage enabled' - required: false - type: boolean - default: false - -# cancel previous runs if new commits are pushed to the PR, but run for each commit on master -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build - KONG_TEST_COVERAGE: ${{ inputs.coverage == true || github.event_name == 'schedule' }} - RUNNER_COUNT: 7 - -jobs: - build: - uses: ./.github/workflows/build.yml - with: - relative-build-root: bazel-bin/build - - lint-and-doc-tests: - name: Lint and Doc tests - runs-on: ubuntu-22.04 - needs: build - - steps: - - name: Bump max open files - run: | - sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - - - name: Checkout Kong source code - uses: actions/checkout@v4 - - - name: Lookup build cache - id: cache-deps - uses: actions/cache@v4 - with: - path: ${{ env.BUILD_ROOT }} - key: ${{ needs.build.outputs.cache-key }} - - - name: Check test-helpers doc generation - run: | - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - pushd ./spec && ldoc . - - - name: Check autodoc generation - run: | - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - scripts/autodoc - - - name: Lint Lua code - run: | - make lint - - - name: Validate rockspec file - run: | - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - scripts/validate-rockspec - - - name: Check spec file misspelling - run: | - scripts/check_spec_files_spelling.sh - - - name: Check labeler configuration - run: scripts/check-labeler.pl .github/labeler.yml - - schedule: - name: Schedule busted tests to run - runs-on: ubuntu-22.04 - needs: build - - env: - WORKFLOW_ID: ${{ github.run_id }} - - outputs: - runners: ${{ steps.generate-runner-array.outputs.RUNNERS }} - - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Download runtimes file - uses: Kong/gh-storage/download@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - repo-path: Kong/gateway-action-storage/main/.ci/runtimes.json - - - name: Schedule tests - uses: Kong/gateway-test-scheduler/schedule@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3 - with: - test-suites-file: .ci/test_suites.json - test-file-runtime-file: .ci/runtimes.json - output-prefix: test-chunk. - runner-count: ${{ env.RUNNER_COUNT }} - static-mode: ${{ github.run_attempt > 1 }} - - - name: Upload schedule files - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - name: schedule-test-files - path: test-chunk.* - retention-days: 7 - - - name: Generate runner array - id: generate-runner-array - run: | - echo "RUNNERS=[$(seq -s "," 1 $(( "$RUNNER_COUNT" )))]" >> "$GITHUB_OUTPUT" - - busted-tests: - name: Busted test runner ${{ matrix.runner }} - runs-on: ubuntu-22.04 - needs: [build,schedule] - - strategy: - fail-fast: false - matrix: - runner: ${{ fromJSON(needs.schedule.outputs.runners) }} - - services: - postgres: - image: postgres:13 - env: - POSTGRES_USER: kong - POSTGRES_DB: kong - POSTGRES_HOST_AUTH_METHOD: trust - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8 - - grpcbin: - image: kong/grpcbin - ports: - - 15002:9000 - - 15003:9001 - - redis: - image: redis - ports: - - 6379:6379 - - 6380:6380 - options: >- - --name kong_redis - - zipkin: - image: openzipkin/zipkin:2 - ports: - - 9411:9411 - - steps: - - name: Bump max open files - run: | - sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - - - name: Checkout Kong source code - uses: actions/checkout@v4 - - - name: Lookup build cache - id: cache-deps - uses: actions/cache@v4 - with: - path: ${{ env.BUILD_ROOT }} - key: ${{ needs.build.outputs.cache-key }} - - - name: Build WASM Test Filters - uses: ./.github/actions/build-wasm-test-filters - - - name: Add gRPC test host names - run: | - echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts - echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts - - - name: Enable SSL for Redis - run: | - docker cp ${{ github.workspace }} kong_redis:/workspace - docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh - docker restart kong_redis - docker logs kong_redis - - - name: Run OpenTelemetry Collector - run: | - mkdir -p ${{ github.workspace }}/tmp/otel - touch ${{ github.workspace }}/tmp/otel/file_exporter.json - sudo chmod 777 -R ${{ github.workspace }}/tmp/otel - docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \ - -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \ - -v ${{ github.workspace }}/tmp/otel:/etc/otel \ - --name opentelemetry-collector -d \ - otel/opentelemetry-collector-contrib:0.52.0 \ - --config=/etc/otel-collector-config.yaml - sleep 2 - docker logs opentelemetry-collector - - - name: Install AWS SAM cli tool - run: | - curl -L -s -o /tmp/aws-sam-cli.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip - unzip -o /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli - sudo /tmp/aws-sam-cli/install --update - - - name: Update PATH - run: | - echo "$BUILD_ROOT/kong-dev/bin" >> $GITHUB_PATH - echo "$BUILD_ROOT/kong-dev/openresty/nginx/sbin" >> $GITHUB_PATH - echo "$BUILD_ROOT/kong-dev/openresty/bin" >> $GITHUB_PATH - - - name: Debug (nginx) - run: | - echo nginx: $(which nginx) - nginx -V 2>&1 | sed -re 's/ --/\n--/g' - ldd $(which nginx) - - - name: Debug (luarocks) - run: | - echo luarocks: $(which luarocks) - luarocks --version - luarocks config - - - name: Tune up postgres max_connections - run: | - # arm64 runners may use more connections due to more worker cores - psql -hlocalhost -Ukong kong -tAc 'alter system set max_connections = 5000;' - - - name: Download test schedule file - uses: actions/download-artifact@v3 - with: - name: schedule-test-files - - - name: Generate helper environment variables - run: | - echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV - echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV - echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV - - - name: Build & install dependencies - run: | - make dev - - - name: Download test rerun information - uses: actions/download-artifact@v3 - continue-on-error: true - with: - name: test-rerun-info-${{ matrix.runner }} - - - name: Download test runtime statistics from previous runs - uses: actions/download-artifact@v3 - continue-on-error: true - with: - name: test-runtime-statistics-${{ matrix.runner }} - - - name: Run Tests - env: - KONG_TEST_PG_DATABASE: kong - KONG_TEST_PG_USER: kong - KONG_TEST_DATABASE: postgres - KONG_SPEC_TEST_GRPCBIN_PORT: "15002" - KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003" - KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json - DD_ENV: ci - DD_SERVICE: kong-ce-ci - DD_CIVISIBILITY_MANUAL_API_ENABLED: 1 - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_TRACE_GIT_METADATA_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }} - uses: Kong/gateway-test-scheduler/runner@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3 - with: - tests-to-run-file: test-chunk.${{ matrix.runner }}.json - failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }} - test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }} - setup-venv-path: ${{ env.BUILD_ROOT }} - - - name: Upload error logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: busted-test-errlogs-${{ matrix.runner }} - path: ${{ env.SPEC_ERRLOG_CACHE_DIR }} - retention-days: 1 - - - name: Upload test rerun information - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-rerun-info-${{ matrix.runner }} - path: ${{ env.FAILED_TEST_FILES_FILE }} - retention-days: 2 - - - name: Upload test runtime statistics for offline scheduling - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-runtime-statistics-${{ matrix.runner }} - path: ${{ env.TEST_FILE_RUNTIME_FILE }} - retention-days: 7 - - - name: Archive coverage stats file - uses: actions/upload-artifact@v3 - if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} - with: - name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.runner }} - retention-days: 1 - path: | - luacov.stats.out - - - name: Get kernel message - if: failure() - run: | - sudo dmesg -T - - pdk-tests: - name: PDK tests - runs-on: ubuntu-22.04 - needs: build - - steps: - - name: Bump max open files - run: | - sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf - - - name: Checkout Kong source code - uses: actions/checkout@v4 - - - name: Lookup build cache - id: cache-deps - uses: actions/cache@v4 - with: - path: ${{ env.BUILD_ROOT }} - key: ${{ needs.build.outputs.cache-key }} - - - name: Install Test::Nginx - run: | - CPAN_DOWNLOAD=./cpanm - mkdir -p $CPAN_DOWNLOAD - curl -o $CPAN_DOWNLOAD/cpanm https://cpanmin.us - chmod +x $CPAN_DOWNLOAD/cpanm - - echo "Installing CPAN dependencies..." - $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) - $CPAN_DOWNLOAD/cpanm --notest Test::Nginx - - - name: Generate environment variables - run: | - echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV - - - name: Tests - env: - TEST_SUITE: pdk - run: | - source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh - if [[ $KONG_TEST_COVERAGE = true ]]; then - export PDK_LUACOV=1 - fi - eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib) - prove -I. -r t - - - name: Upload error logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: PDK-test-errlogs - path: ${{ env.SPEC_ERRLOG_CACHE_DIR }} - retention-days: 1 - - - name: Archive coverage stats file - uses: actions/upload-artifact@v3 - if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} - with: - name: luacov-stats-out-${{ github.job }}-${{ github.run_id }} - retention-days: 1 - path: | - luacov.stats.out - - - name: Get kernel message - if: failure() - run: | - sudo dmesg -T - - cleanup-and-aggregate-stats: - needs: [lint-and-doc-tests,pdk-tests,busted-tests] - name: Cleanup and Luacov stats aggregator - if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }} - runs-on: ubuntu-22.04 - - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Install requirements - run: | - sudo apt-get update && sudo apt-get install -y luarocks - sudo luarocks install luacov - sudo luarocks install luafilesystem - - # Download all archived coverage stats files - - uses: actions/download-artifact@v3 - - - name: Stats aggregation - shell: bash - run: | - lua .ci/luacov-stats-aggregator.lua "luacov-stats-out-" "luacov.stats.out" ${{ github.workspace }}/ - # The following prints a report with each file sorted by coverage percentage, and the total coverage - printf "\n\nCoverage File\n\n" - awk -v RS='Coverage\n-+\n' 'NR>1{print $0}' luacov.report.out | grep -vE "^-|^$" > summary.out - cat summary.out | grep -v "^Total" | awk '{printf "%7d%% %s\n", $4, $1}' | sort -n - cat summary.out | grep "^Total" | awk '{printf "%7d%% %s\n", $4, $1}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c73fb2d3eafb..4c517f33209b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -277,38 +277,6 @@ jobs: path: bazel-bin/pkg retention-days: 3 - verify-manifest-packages: - needs: [metadata, build-packages] - name: Verify Manifest - Package ${{ matrix.label }} - runs-on: ubuntu-22.04 - - strategy: - fail-fast: false - matrix: - include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-packages'] }}" - - steps: - - uses: actions/checkout@v3 - - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.label }}-packages - path: bazel-bin/pkg - - - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' # caching pip dependencies - - - name: Verify - run: | - cd scripts/explain_manifest - pip install -r requirements.txt - pkg=$(ls ../../bazel-bin/pkg/kong* |head -n1) - python ./main.py -f filelist.txt -p $pkg -o test.txt -s ${{ matrix.check-manifest-suite }} - build-images: name: Build Images - ${{ matrix.label }} needs: [metadata, build-packages] @@ -413,169 +381,6 @@ jobs: Docker image available `${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ needs.metadata.outputs.commit-sha }}` Artifacts available https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - verify-manifest-images: - needs: [metadata, build-images] - name: Verify Manifest - Image ${{ matrix.label }} - runs-on: ubuntu-22.04 - if: github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') - - strategy: - fail-fast: false - matrix: - include: "${{ fromJSON(needs.metadata.outputs.matrix)['build-images'] }}" - - steps: - - uses: actions/checkout@v4 - - - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' # caching pip dependencies - - - name: Verify - run: | - cd scripts/explain_manifest - # docker image verify requires sudo to set correct permissions, so we - # also install deps for root - sudo -E pip install -r requirements.txt - IMAGE=${{ env.PRERELEASE_DOCKER_REPOSITORY }}:${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }} - - sudo -E python ./main.py --image $IMAGE -f docker_image_filelist.txt -s docker-image - - if [[ ! -z "${{ matrix.docker-platforms }}" ]]; then - DOCKER_DEFAULT_PLATFORM=linux/arm64 sudo -E python ./main.py --image $IMAGE -f docker_image_filelist.txt -s docker-image - fi - - scan-images: - name: Scan Images - ${{ matrix.label }} - needs: [metadata, build-images] - runs-on: ubuntu-22.04 - if: |- - always() - && fromJSON(needs.metadata.outputs.matrix)['scan-vulnerabilities'] != '' - && needs.build-images.result == 'success' - && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) - strategy: - fail-fast: false - matrix: - include: "${{ fromJSON(needs.metadata.outputs.matrix)['scan-vulnerabilities'] }}" - env: - IMAGE: ${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }} - steps: - - name: Install regctl - uses: regclient/actions/regctl-installer@main - - - name: Login to Docker Hub - if: ${{ env.HAS_ACCESS_TO_GITHUB_TOKEN }} - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v2.1.0 - with: - username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} - password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} - - # TODO: Refactor matrix file to support and parse platforms specific to distro - # Workaround: Look for specific amd64 and arm64 hardcooded architectures - - name: Parse Architecture Specific Image Manifest Digests - id: image_manifest_metadata - run: | - manifest_list_exists="$( - if regctl manifest get "${IMAGE}" --format raw-body --require-list -v panic &> /dev/null; then - echo true - else - echo false - fi - )" - echo "manifest_list_exists=$manifest_list_exists" - echo "manifest_list_exists=$manifest_list_exists" >> $GITHUB_OUTPUT - - amd64_sha="$(regctl image digest "${IMAGE}" --platform linux/amd64 || echo '')" - arm64_sha="$(regctl image digest "${IMAGE}" --platform linux/arm64 || echo '')" - echo "amd64_sha=$amd64_sha" - echo "amd64_sha=$amd64_sha" >> $GITHUB_OUTPUT - echo "arm64_sha=$arm64_sha" - echo "arm64_sha=$arm64_sha" >> $GITHUB_OUTPUT - - - name: Scan AMD64 Image digest - id: sbom_action_amd64 - if: steps.image_manifest_metadata.outputs.amd64_sha != '' - uses: Kong/public-shared-actions/security-actions/scan-docker-image@v2 - with: - asset_prefix: kong-${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }}-linux-amd64 - image: ${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }} - - - name: Scan ARM64 Image digest - if: steps.image_manifest_metadata.outputs.manifest_list_exists == 'true' && steps.image_manifest_metadata.outputs.arm64_sha != '' - id: sbom_action_arm64 - uses: Kong/public-shared-actions/security-actions/scan-docker-image@v2 - with: - asset_prefix: kong-${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }}-linux-arm64 - image: ${{ needs.metadata.outputs.prerelease-docker-repository }}:${{ needs.metadata.outputs.commit-sha }}-${{ matrix.label }} - - release-packages: - name: Release Packages - ${{ matrix.label }} - ${{ needs.metadata.outputs.release-desc }} - needs: [metadata, build-packages, build-images] - runs-on: ubuntu-22.04 - if: fromJSON(needs.metadata.outputs.matrix)['release-packages'] != '' - timeout-minutes: 5 # PULP takes a while to publish - environment: release - - strategy: - # limit to 3 jobs at a time - max-parallel: 3 - fail-fast: false - matrix: - include: "${{ fromJSON(needs.metadata.outputs.matrix)['release-packages'] }}" - - steps: - - uses: actions/checkout@v4 - - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.artifact-from }}-packages - path: bazel-bin/pkg - - - name: Set package architecture - id: pkg-arch - run: | - arch='amd64' - if [[ '${{ matrix.label }}' == *'arm64' ]]; then - arch='arm64' - fi - echo "arch=$arch" - echo "arch=$arch" >> $GITHUB_OUTPUT - - - name: Upload Packages - env: - ARCHITECTURE: ${{ steps.pkg-arch.outputs.arch }} - OFFICIAL_RELEASE: ${{ github.event.inputs.official }} - ARTIFACT_VERSION: ${{ matrix.artifact-version }} - ARTIFACT_TYPE: ${{ matrix.artifact-type }} - ARTIFACT: ${{ matrix.artifact }} - INPUT_VERSION: ${{ github.event.inputs.version }} - PACKAGE_TYPE: ${{ matrix.package }} - KONG_RELEASE_LABEL: ${{ needs.metadata.outputs.release-label }} - VERBOSE: ${{ runner.debug == '1' && '1' || '' }} - CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} - CLOUDSMITH_DRY_RUN: '' - IGNORE_CLOUDSMITH_FAILURES: ${{ vars.IGNORE_CLOUDSMITH_FAILURES }} - USE_CLOUDSMITH: ${{ vars.USE_CLOUDSMITH }} - run: | - sha256sum bazel-bin/pkg/* - - # set the version input as tags passed to release-scripts - # note: release-scripts rejects user tags if missing internal flag - # - # this can be a comma-sepratated list of tags to apply - if [[ "$OFFICIAL_RELEASE" == 'false' ]]; then - if echo "$INPUT_VERSION" | grep -qs -E 'rc|alpha|beta|nightly'; then - PACKAGE_TAGS="$INPUT_VERSION" - export PACKAGE_TAGS - fi - fi - - scripts/release-kong.sh - release-images: name: Release Images - ${{ matrix.label }} - ${{ needs.metadata.outputs.release-desc }} needs: [metadata, build-images] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..2a7a1d1a62e9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM kong:3.6.0-ubuntu + +COPY poc.patch /tmp/poc.patch +COPY lua-resty-protobuf /tmp/lua-resty-protobuf +COPY --chown=root:root --chmod=744 docker-entrypoint.sh /docker-entrypoint.sh + +USER root:root +RUN apt-get update -y \ + && apt-get install -y \ + automake \ + build-essential \ + cmake \ + git \ + libprotobuf-dev \ + protobuf-compiler \ + libabsl-dev \ + && echo "Successfully installed protobuf" \ + && cd /tmp/lua-resty-protobuf \ + && make install \ + && gcc -O2 -g3 consumer.c -o /usr/local/bin/consumer \ + && echo "Successfully installed lua-resty-protobuf" \ + && cd /usr/local/share/lua/5.1 \ + && git apply /tmp/poc.patch \ + && echo "Successfully patched kong" \ + && rm -rf /tmp/poc.patch /tmp/lua-resty-protobuf + +USER kong:kong diff --git a/WORKSPACE b/WORKSPACE index ae97c320d94e..92fef11703e1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -52,6 +52,18 @@ load("@atc_router//build:crates.bzl", "atc_router_crates") atc_router_crates() +load("@resty_protobuf//build:repos.bzl", "resty_protobuf_repositories") + +resty_protobuf_repositories() + +load("@resty_protobuf//build:deps.bzl", "resty_protobuf_dependencies") + +resty_protobuf_dependencies(cargo_home_isolated = False) # TODO: set cargo_home_isolated=True for release + +load("@resty_protobuf//build:crates.bzl", "resty_protobuf_crates") + +resty_protobuf_crates() + load("//build:repositories.bzl", "build_repositories") build_repositories() diff --git a/build/BUILD.bazel b/build/BUILD.bazel index 3af4f65ac45a..f4a88f05ec3d 100644 --- a/build/BUILD.bazel +++ b/build/BUILD.bazel @@ -100,6 +100,7 @@ lualib_deps = [ luaclib_deps = [ "@atc_router", + "@resty_protobuf", ] [ @@ -239,6 +240,15 @@ kong_genrule( # TODO: remove this after lua-resty-websocket becomes a patch or merged to upstream rm -rf ${BUILD_DESTDIR}/openresty/lualib/resty/websocket + RESTY_PROTOBUF=${WORKSPACE_PATH}/$(location @resty_protobuf) + cp $RESTY_PROTOBUF ${BUILD_DESTDIR}/openresty/lualib/. + + cp -r $(locations @protoc//:all_srcs) ${BUILD_DESTDIR}/kong/. + + """ + install_lib_deps_cmd + install_lualib_deps_cmd + install_webui_cmd + link_modules_dir + install_wasm_deps_cmd + """ + mkdir -p ${BUILD_DESTDIR}/etc/kong + cp kong.conf.default ${BUILD_DESTDIR}/etc/kong/kong.conf.default + # housecleaning if [[ -d ${BUILD_DESTDIR}/kong/lib64 ]]; then cp -r ${BUILD_DESTDIR}/kong/lib64/* ${BUILD_DESTDIR}/kong/lib/. diff --git a/build/openresty/repositories.bzl b/build/openresty/repositories.bzl index 0d992fd3fd99..b6c82da08955 100644 --- a/build/openresty/repositories.bzl +++ b/build/openresty/repositories.bzl @@ -7,6 +7,7 @@ load("@kong_bindings//:variables.bzl", "KONG_VAR") load("//build/openresty/pcre:pcre_repositories.bzl", "pcre_repositories") load("//build/openresty/openssl:openssl_repositories.bzl", "openssl_repositories") load("//build/openresty/atc_router:atc_router_repositories.bzl", "atc_router_repositories") +load("//build/openresty/resty_protobuf:resty_protobuf_repositories.bzl", "resty_protobuf_repositories") load("//build/openresty/wasmx:wasmx_repositories.bzl", "wasmx_repositories") load("//build/openresty/wasmx/filters:repositories.bzl", "wasm_filters_repositories") load("//build/openresty/brotli:brotli_repositories.bzl", "brotli_repositories") @@ -31,6 +32,7 @@ def openresty_repositories(): pcre_repositories() openssl_repositories() atc_router_repositories() + resty_protobuf_repositories() wasmx_repositories() wasm_filters_repositories() brotli_repositories() diff --git a/build/openresty/resty_protobuf/BUILD.bazel b/build/openresty/resty_protobuf/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/openresty/resty_protobuf/resty_protobuf_repositories.bzl b/build/openresty/resty_protobuf/resty_protobuf_repositories.bzl new file mode 100644 index 000000000000..a5b661281bcf --- /dev/null +++ b/build/openresty/resty_protobuf/resty_protobuf_repositories.bzl @@ -0,0 +1,13 @@ +"""A module defining the dependency atc-router""" + +# load("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +# load("@kong_bindings//:variables.bzl", "KONG_VAR") + +def resty_protobuf_repositories(): + maybe( + native.local_repository, + name = "resty_protobuf", + path = "lua-resty-protobuf", + # visibility = ["//visibility:public"], # let this to be referenced by openresty build + ) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 000000000000..4ae23734b76b --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -Eeo pipefail + +if [ "$KONG_ROLE" == "data_plane" ]; then + consumer & +fi + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + # Do not continue if _FILE env is not set + if ! [ "${!fileVar:-}" ]; then + return + elif [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +export KONG_NGINX_DAEMON=${KONG_NGINX_DAEMON:=off} + +if [[ "$1" == "kong" ]]; then + + all_kong_options="/usr/local/share/lua/5.1/kong/templates/kong_defaults.lua" + set +Eeo pipefail + while IFS='' read -r LINE || [ -n "${LINE}" ]; do + opt=$(echo "$LINE" | grep "=" | sed "s/=.*$//" | sed "s/ //" | tr '[:lower:]' '[:upper:]') + file_env "KONG_$opt" + done < $all_kong_options + set -Eeo pipefail + + file_env KONG_PASSWORD + PREFIX=${KONG_PREFIX:=/usr/local/kong} + + if [[ "$2" == "docker-start" ]]; then + kong prepare -p "$PREFIX" "$@" + + # remove all dangling sockets in $PREFIX dir before starting Kong + LOGGED_SOCKET_WARNING=0 + for localfile in "$PREFIX"/*; do + if [ -S "$localfile" ]; then + if (( LOGGED_SOCKET_WARNING == 0 )); then + printf >&2 'WARN: found dangling unix sockets in the prefix directory ' + printf >&2 '(%q) ' "$PREFIX" + printf >&2 'while preparing to start Kong. This may be a sign that Kong ' + printf >&2 'was previously shut down uncleanly or is in an unknown state ' + printf >&2 'and could require further investigation.\n' + LOGGED_SOCKET_WARNING=1 + fi + rm -f "$localfile" + fi + done + + ln -sfn /dev/stdout $PREFIX/logs/access.log + ln -sfn /dev/stdout $PREFIX/logs/admin_access.log + ln -sfn /dev/stderr $PREFIX/logs/error.log + + exec /usr/local/openresty/nginx/sbin/nginx \ + -p "$PREFIX" \ + -c nginx.conf + fi +fi + +exec "$@" \ No newline at end of file diff --git a/kong-3.8.0-0.rockspec b/kong-3.8.0-0.rockspec index bec67ea7cce8..db505cee759d 100644 --- a/kong-3.8.0-0.rockspec +++ b/kong-3.8.0-0.rockspec @@ -652,6 +652,13 @@ build = { ["kong.timing.hooks.redis"] = "kong/timing/hooks/redis.lua", ["kong.timing.hooks.socket"] = "kong/timing/hooks/socket.lua", + ["kong.etrace"] = "kong/etrace/init.lua", + ["kong.etrace.queue"] = "kong/etrace/queue.lua", + ["kong.etrace.hooks"] = "kong/etrace/hooks/init.lua", + ["kong.etrace.hooks.dns"] = "kong/etrace/hooks/dns.lua", + ["kong.etrace.hooks.redis"] = "kong/etrace/hooks/redis.lua", + ["kong.dynamic_hook"] = "kong/dynamic_hook/init.lua", } } + diff --git a/kong/etrace/hooks/dns.lua b/kong/etrace/hooks/dns.lua new file mode 100644 index 000000000000..6f6cf569aabf --- /dev/null +++ b/kong/etrace/hooks/dns.lua @@ -0,0 +1,64 @@ +-- This software is copyright Kong Inc. and its licensors. +-- Use of the software is subject to the agreement between your organization +-- and Kong Inc. If there is no such agreement, use is governed by and +-- subject to the terms of the Kong Master Software License Agreement found +-- at https://konghq.com/enterprisesoftwarelicense/. +-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ] + +local _M = {} + +local etrace +local etrace_enter_span +local etrace_add_string_attribute +local etrace_add_bool_attribute +local etrace_add_int64_attribute +local etrace_add_double_attribute +local etrace_exit_span + +local client = package.loaded["kong.resty.dns.client"] +if not client then + client = require("kong.tools.dns")() +end + + +local function before_toip(qname, _port, _dnsCacheOnly, _try_list) + etrace_enter_span("dns") + etrace_add_string_attribute("qname", qname) +end + + +local function after_toip(ip_addr, res_port) + etrace_add_int64_attribute("record.port", tonumber(res_port)) + + if ip_addr then + etrace_add_string_attribute("record.ip", ip_addr) + end + + etrace_exit_span() -- dns +end + + +function _M.globalpatches(etrace_module) + local req_dyn_hook = require("kong.dynamic_hook") + + --[[ + The `toip()` function can receive <= 4 arguments (including `self`). + Here is the signature of the `toip()` function: + function toip(self, qname, port, dnsCacheOnly, try_list) + --]] + req_dyn_hook.hook_function("etrace", client, "toip", 4, { + befores = { before_toip }, + afters = { after_toip }, + }) + + etrace = etrace_module + etrace_enter_span = etrace.enter_span + etrace_add_string_attribute = etrace.add_string_attribute + etrace_add_bool_attribute = etrace.add_bool_attribute + etrace_add_int64_attribute = etrace.add_int64_attribute + etrace_add_double_attribute = etrace.add_double_attribute + etrace_exit_span = etrace.exit_span +end + + +return _M diff --git a/kong/etrace/hooks/init.lua b/kong/etrace/hooks/init.lua new file mode 100644 index 000000000000..a0cd2f9cb213 --- /dev/null +++ b/kong/etrace/hooks/init.lua @@ -0,0 +1,30 @@ +-- This software is copyright Kong Inc. and its licensors. +-- Use of the software is subject to the agreement between your organization +-- and Kong Inc. If there is no such agreement, use is governed by and +-- subject to the terms of the Kong Master Software License Agreement found +-- at https://konghq.com/enterprisesoftwarelicense/. +-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ] + +local _M = {} + +-- order matters +local HOOKS = { + -- "socket", + "dns", + -- "http", + "redis", + -- XXX EE [[ + -- "rediscluster" + -- XXX EE ]] +} + + +function _M.globalpatches(etrace_module) + for _, hook_name in ipairs(HOOKS) do + local hook_module = require("kong.etrace.hooks." .. hook_name) + hook_module.globalpatches(etrace_module) + end +end + + +return _M diff --git a/kong/etrace/hooks/redis.lua b/kong/etrace/hooks/redis.lua new file mode 100644 index 000000000000..2b5391c769b3 --- /dev/null +++ b/kong/etrace/hooks/redis.lua @@ -0,0 +1,57 @@ +-- This software is copyright Kong Inc. and its licensors. +-- Use of the software is subject to the agreement between your organization +-- and Kong Inc. If there is no such agreement, use is governed by and +-- subject to the terms of the Kong Master Software License Agreement found +-- at https://konghq.com/enterprisesoftwarelicense/. +-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ] + +local _M = {} + +local etrace +local etrace_enter_span +local etrace_add_string_attribute +local etrace_add_bool_attribute +local etrace_add_int64_attribute +local etrace_add_double_attribute +local etrace_exit_span + + +local function before() + etrace_enter_span("redis") +end + + +local function after() + etrace_exit_span() -- redis +end + + +function _M.globalpatches(etrace_module) + local req_dyn_hook = require("kong.dynamic_hook") + + local redis = require("resty.redis") + for method_name, _ in pairs(redis) do + if type(redis[method_name]) ~= "function" then + goto continue + end + + req_dyn_hook.hook_function("etrace", redis, method_name, "varargs", { + befores = { before }, + afters = { after }, + }) + + ::continue:: + end + + etrace = etrace_module + etrace_enter_span = etrace.enter_span + etrace_add_string_attribute = etrace.add_string_attribute + etrace_add_bool_attribute = etrace.add_bool_attribute + etrace_add_int64_attribute = etrace.add_int64_attribute + etrace_add_double_attribute = etrace.add_double_attribute + etrace_exit_span = etrace.exit_span +end + + + +return _M diff --git a/kong/etrace/init.lua b/kong/etrace/init.lua new file mode 100644 index 000000000000..107a2c7a950e --- /dev/null +++ b/kong/etrace/init.lua @@ -0,0 +1,582 @@ +local req_dyn_hook = require("kong.dynamic_hook") +local kproto_trace = require("resty.kong.kproto.trace") +local kproto_metric = require("resty.kong.kproto.metric") +local kproto_log = require("resty.kong.kproto.log") +local errlog = require "ngx.errlog" +local http = require "resty.http" + +local ngx_var = ngx.var +local ngx_timer_at = ngx.timer.at +local ngx_worker_exiting = ngx.worker.exiting +local ngx_sleep = ngx.sleep + +local math_random = math.random + +local ngx_get_phase = ngx.get_phase +local ngx_req_get_method = ngx.req.get_method +local ngx_req_http_version = ngx.req.http_version + +local request_id_get = require ("kong.tracing.request_id").get + +local trace_new = kproto_trace.new +local trace_get_serialized = kproto_trace.get_serialized +local trace_enter_span = kproto_trace.enter_span +local trace_exit_span = kproto_trace.exit_span +local trace_add_string_attribute = kproto_trace.add_string_attribute +local trace_add_bool_attribute = kproto_trace.add_bool_attribute +local trace_add_int64_attribute = kproto_trace.add_int64_attribute +local trace_add_double_attribute = kproto_trace.add_double_attribute + +local metric_new = kproto_metric.new +local metric_get_serialized = kproto_metric.get_serialized +local metric_add_gauge = kproto_metric.add_gauge +local metric_add_sum = kproto_metric.add_sum + +local log_new = kproto_log.new +local log_get_serialized = kproto_log.get_serialized +local log_add_info = kproto_log.add_info +local log_add_warn = kproto_log.add_warn +local log_add_error = kproto_log.add_error +local log_add_fatal = kproto_log.add_fatal + +local trace_otlp_client +local metric_otlp_client +local log_otlp_client + +local trace_queue = require("kong.etrace.queue").new(1024) + +local TRCCE_CONNECT_OPTS = { + scheme = "http", + host = "localhost", + pool = "qiqi-demo-trace", + pool_size = 4096, + port = 5000, +} + +local METRIC_CONNECT_OPTS = { + scheme = "http", + host = "localhost", + pool = "qiqi-demo-metric", + pool_size = 16, + port = 5000, +} + +local LOG_CONNECT_OPTS = { + scheme = "http", + host = "localhost", + pool = "qiqi-demo-log", + pool_size = 16, + port = 5000, +} + +local TRACE_REQ_PARAMS = { + -- version = "1.1", + method = "POST", + path = "/v1/traces", + query = "", + headers = { + ["Content-Type"] = "application/x-protobuf", + ["Content-Length"] = nil, + }, + body = nil +} + +local METRIC_REQ_PARAMS = { + -- version = "1.1", + method = "POST", + path = "/v1/metrics", + query = "", + headers = { + ["Content-Type"] = "application/x-protobuf", + ["Content-Length"] = nil, + }, + body = nil +} + +local LOG_REQ_PARAMS = { + -- version = "1.1", + method = "POST", + path = "/v1/logs", + query = "", + headers = { + ["Content-Type"] = "application/x-protobuf", + ["Content-Length"] = nil, + }, + body = nil +} + + +local sampling_rate + +local _M = {} + +local VALID_PHASES = { + rewrite = true, + balancer = true, + access = true, + header_filter = true, + body_filter = true, + log = true, +} + +local ADD_LOG_FUNC = { + [ngx.DEBUG] = log_add_info, + [ngx.INFO] = log_add_info, + [ngx.NOTICE] = log_add_info, + [ngx.WARN] = log_add_warn, + [ngx.ERR] = log_add_error, + [ngx.CRIT] = log_add_fatal, + [ngx.ALERT] = log_add_fatal, + [ngx.EMERG] = log_add_fatal, +} + + +local function otlp_trace_cb(premature, buf) + if premature then + return + end + + local httpc = http.new() + + assert(httpc:connect(TRCCE_CONNECT_OPTS)) + + local res = assert(httpc:request({ + method = "POST", + path = "/v1/traces", + query = "", + headers = { + ["Content-Type"] = "application/x-protobuf", + ["Content-Length"] = #buf, + }, + body = buf + })) + + if res.has_body then + res:read_body() + res:read_trailers() + end + + if res.status ~= 200 and res.status ~= 201 then + ngx.log(ngx.ERR, "failed to send trace: ", res.status, " ", res.body) + end + + httpc:set_keepalive(9999999) +end + + +local function otlp_metric_cb(premature) + if premature then + return + end + + local httpc = http.new() + assert(httpc:connect(METRIC_CONNECT_OPTS)) + + local metrics = metric_new() + + local nginx_statistics = kong.nginx.get_statistics() + + local total_requests = nginx_statistics['total_requests'] + metric_add_gauge(metrics, "kong_total_requests", total_requests) + + local nginx_connections_active = nginx_statistics['connections_active'] + local nginx_connections_reading = nginx_statistics['connections_reading'] + local nginx_connections_writing = nginx_statistics['connections_writing'] + local nginx_connections_waiting = nginx_statistics['connections_waiting'] + local nginx_connections_accepted = nginx_statistics['connections_accepted'] + local nginx_connections_handled = nginx_statistics['connections_handled'] + + metric_add_gauge(metrics, "nginx_connections_active", nginx_connections_active) + metric_add_gauge(metrics, "nginx_connections_reading", nginx_connections_reading) + metric_add_gauge(metrics, "nginx_connections_writing", nginx_connections_writing) + metric_add_gauge(metrics, "nginx_connections_waiting", nginx_connections_waiting) + metric_add_gauge(metrics, "nginx_connections_accepted", nginx_connections_accepted) + metric_add_gauge(metrics, "nginx_connections_handled", nginx_connections_handled) + + local pending_timers = ngx.timer.pending_count() + metric_add_gauge(metrics, "nginx_pending_timers", pending_timers) + + local running_timers = ngx.timer.running_count() + metric_add_gauge(metrics, "nginx_running_timers", running_timers) + + -- local request_latency = message.latencies.request + -- metric_add_gauge(metrics, "kong_request_latency", request_latency) + + local gcsize = collectgarbage("count") + metric_add_gauge(metrics, "lua_vm_size", gcsize) + + local buf = metric_get_serialized(metrics) + + METRIC_REQ_PARAMS.body = buf + METRIC_REQ_PARAMS.headers["Content-Length"] = #buf + local res = assert(httpc:request(METRIC_REQ_PARAMS)) + METRIC_REQ_PARAMS.body = nil + + if res.has_body then + res:read_body() + res:read_trailers() + end + + if res.status ~= 200 and res.status ~= 201 then + ngx.log(ngx.ERR, "failed to send metrics: ", res.status, " ", res.body) + end + + httpc:set_keepalive(9999999) +end + + +local function otlp_log_cb(premature) + if premature then + return + end + + local httpc = http.new() + assert(httpc:connect(LOG_CONNECT_OPTS)) + + local raw_logs = assert(errlog.get_logs(64)) + local logs = log_new() + + for i = 1, #raw_logs, 3 do + local level = raw_logs[i] + local time = (assert(tonumber(raw_logs[i + 1]))) * 1000000000 + local msg = raw_logs[i + 2] + + ADD_LOG_FUNC[level](logs, time, msg) + end + + local buf = log_get_serialized(logs) + + LOG_REQ_PARAMS.body = buf + LOG_REQ_PARAMS.headers["Content-Length"] = #buf + local res, err = httpc:request(LOG_REQ_PARAMS) + LOG_REQ_PARAMS.body = nil + + if res.has_body then + res:read_body() + res:read_trailers() + end + + if res.status ~= 200 and res.status ~= 201 then + ngx.log(ngx.ERR, "failed to send logs: ", res.status, " ", res.body) + end + + httpc:set_keepalive(9999999) +end + + +function _M.globalpatches() + require("kong.etrace.hooks").globalpatches(_M) + + req_dyn_hook.hook("etrace", "before:rewrite", function(ctx) + if math_random() > sampling_rate then + return + end + + local tr = trace_new() + ctx.tr = tr + + local client = kong.client + local scheme = ctx.scheme or ngx_var.scheme + local host = ngx_var.host + local request_uri = scheme .. "://" .. host .. (ctx.request_uri or ngx_var.request_uri) + + trace_enter_span(tr, "Kong") + trace_add_string_attribute(tr, "http.method", ngx_req_get_method()) + trace_add_string_attribute(tr, "http.url", request_uri) + trace_add_string_attribute(tr, "http.host", host) + trace_add_string_attribute(tr, "http.scheme", scheme) + trace_add_int64_attribute(tr, "http.flavor", ngx_req_http_version()) + trace_add_string_attribute(tr, "http.client_ip", client.get_forwarded_ip()) + trace_add_string_attribute(tr, "net.peer.ip", client.get_ip()) + trace_add_string_attribute(tr, "kong.request.id", request_id_get()) + + trace_enter_span(tr, "rewrite") + end) + + req_dyn_hook.hook("etrace", "after:rewrite", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) -- rewrite + end) + + req_dyn_hook.hook("etrace", "before:balancer", function(ctx) + if not ctx.tr then + return + end + + trace_enter_span(ctx.tr, "balancer") + end) + + req_dyn_hook.hook("etrace", "after:balancer", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + + trace_enter_span(ctx.tr, "Upstream (Time to first byte)") + end) + + req_dyn_hook.hook("etrace", "before:access", function(ctx) + if not ctx.tr then + return + end + + trace_enter_span(ctx.tr, "access") + end) + + req_dyn_hook.hook("etrace", "after:access", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + end) + + req_dyn_hook.hook("etrace", "before:response", function(ctx) + if not ctx.tr then + return + end + + trace_enter_span(ctx.tr, "response") + end) + + req_dyn_hook.hook("etrace", "after:response", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + end) + + req_dyn_hook.hook("etrace", "before:header_filter", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) -- Upstream (Time to first byte) + + trace_enter_span(ctx.tr, "header_filter") + trace_add_string_attribute(ctx.tr, "http.status_code", tostring(ngx.status)) + local r = ngx.ctx.route + trace_add_string_attribute(ctx.tr, "http.route", r and r.paths and r.paths[1] or "") + end) + + req_dyn_hook.hook("etrace", "after:header_filter", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + + trace_enter_span(ctx.tr, "Upstream (Streaming)") + end) + + req_dyn_hook.hook("etrace", "before:body_filter", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) -- Upstream (Streaming) + + trace_enter_span(ctx.tr, "body_filter") + end) + + req_dyn_hook.hook("etrace", "after:body_filter", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + + trace_enter_span(ctx.tr, "Upstream (Streaming)") + end) + + req_dyn_hook.hook("etrace", "before:log", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) -- Upstream (Streaming) + + trace_enter_span(ctx.tr, "log") + end) + + req_dyn_hook.hook("etrace", "after:log", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) -- log + trace_exit_span(ctx.tr) -- Kong + + local serialized_data = assert(trace_get_serialized(ctx.tr)) + -- local fp = assert(io.open("/tmp/etrace.bin", "w")) + -- assert(fp:write(serialized_data)) + -- assert(fp:close()) + + ngx_timer_at(0, otlp_trace_cb, serialized_data) + + -- if trace_queue:length() >= 500000 then + -- ngx.log(ngx.ERR, "trace queue is full") + -- return + -- end + + -- trace_queue:push_right(serialized_data) + end) + + req_dyn_hook.hook("etrace", "before:plugin_iterator", function(ctx) + if not ctx.tr then + return + end + + trace_enter_span(assert(ctx.tr), "plugins") + end) + + req_dyn_hook.hook("etrace", "after:plugin_iterator", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(assert(ctx.tr)) -- plugins + end) + + req_dyn_hook.hook("etrace", "before:a_plugin", function(ctx, plugin_name, plugin_id) + if not ctx.tr then + return + end + + trace_enter_span(assert(ctx.tr), plugin_name) + trace_add_string_attribute(ctx.tr, "plugin_id", plugin_id) + end) + + req_dyn_hook.hook("etrace", "after:a_plugin", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(assert(ctx.tr)) -- plugin_name + end) + + req_dyn_hook.hook("etrace", "before:router", function(ctx) + if not ctx.tr then + return + end + + trace_enter_span(ctx.tr, "router") + end) + + req_dyn_hook.hook("etrace", "after:router", function(ctx) + if not ctx.tr then + return + end + + trace_exit_span(ctx.tr) + end) +end + + +function _M.init(kong_config) + sampling_rate = tonumber(kong_config.tracing_sampling_rate) or 1 + assert(type(sampling_rate) == "number" and sampling_rate >= 0 and sampling_rate <= 1, "invalid tracing_sampling_rate") + ngx.log(ngx.ERR, "etrace init: sampling_rate=", sampling_rate) +end + + +function _M.init_worker() + req_dyn_hook.always_enable("etrace") + ngx.log(ngx.ERR, "etrace init_worker") + + -- ngx.timer.at(0, otlp_trace_cb) + ngx.timer.every(30, otlp_metric_cb) + ngx.timer.every(30, otlp_log_cb) +end + + +function _M.enter_span(name) + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_enter_span(tr, name) +end + + +function _M.add_string_attribute(key, value) + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_add_string_attribute(tr, key, value) +end + + +function _M.add_bool_attribute(key, value) + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_add_bool_attribute(tr, key, value) +end + + +function _M.add_int64_attribute(key, value) + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_add_int64_attribute(tr, key, value) +end + + +function _M.add_double_attribute(key, value) + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_add_double_attribute(tr, key, value) +end + + +function _M.exit_span() + if not VALID_PHASES[ngx_get_phase()] then + return + end + + local tr = ngx.ctx.tr + if not tr then + return + end + + trace_exit_span(tr) +end + + +return _M diff --git a/kong/etrace/queue.lua b/kong/etrace/queue.lua new file mode 100644 index 000000000000..5d6488bf4c09 --- /dev/null +++ b/kong/etrace/queue.lua @@ -0,0 +1,129 @@ +local table_insert = table.insert +local table_remove = table.remove +local table_new = require("table.new") +local table_clear = require("table.clear") + + +local error = error +local setmetatable = setmetatable + +local array_pool = {} + +local _M = {} + +local meta_table = { + __index = _M, +} + + +function _M.next(array, index) + if index == nil then + index = 0 + end + + if array.nelts < index then + return nil + end + + if index + 1 > array.nelts then + return nil + end + + return index + 1, array.elts[index] +end + + +function _M:length() + return self.last - self.first + 1 +end + + + +function _M:is_empty() + return self.first > self.last +end + + +function _M:push_left (value) + local first = self.first - 1 + self.first = first + self.elts[first] = value +end + + +function _M:push_right(value) + local last = self.last + 1 + self.last = last + self.elts[last] = value +end + + +function _M:pop_left() + local first = self.first + + if first > self.last then + error("list is empty") + end + + local value = self.elts[first] + self.elts[first] = nil + self.first = first + 1 + return value +end + + +function _M:pop_right() + local last = self.last + + if self.first > last then + error("list is empty") + end + + local value = self.elts[last] + self.elts[last] = nil + self.last = last - 1 + return value +end + + +function _M.new(n) + if n == nil then + n = 8 + end + + local self = table_remove(array_pool) + + if self then + return self + end + + self = { + elts = table_new(n, 0), + first = 0, + last = -1, + } + + return setmetatable(self, meta_table) +end + + +function _M:release() + table_clear(self.elts) + self.first = 0 + self.last = -1 + table_insert(array_pool, self) +end + + +function _M.merge(dst, src) + if src == nil then + return + end + + while not src:is_empty() do + dst:push_left(src:pop_left()) + end +end + + +return _M \ No newline at end of file diff --git a/kong/globalpatches.lua b/kong/globalpatches.lua index 4c9581f49d0f..52af7775ee03 100644 --- a/kong/globalpatches.lua +++ b/kong/globalpatches.lua @@ -591,7 +591,10 @@ return function(options) if not options.cli and not options.rbusted then local timing = require "kong.timing" + local etrace = require "kong.etrace" + timing.register_hooks() + etrace.globalpatches() end -- STEP 5: load code that should be using the patched versions, if any (because of dependency chain) diff --git a/kong/init.lua b/kong/init.lua index 24a6c6b1071e..d4ec4caae45d 100644 --- a/kong/init.lua +++ b/kong/init.lua @@ -97,6 +97,7 @@ local pl_file = require "pl.file" local req_dyn_hook = require "kong.dynamic_hook" local uuid = require("kong.tools.uuid").uuid local kong_time = require("kong.tools.time") +local etrace = require "kong.etrace" local kong = kong @@ -323,8 +324,11 @@ local function execute_global_plugins_iterator(plugins_iterator, phase, ctx) return end + req_dyn_hook_run_hook("etrace", "before:plugin_iterator", ctx) + local old_ws = ctx.workspace local has_timing = ctx.has_timing + local old_ctx = ctx if has_timing then req_dyn_hook_run_hook("timing", "before:plugin_iterator") @@ -342,12 +346,16 @@ local function execute_global_plugins_iterator(plugins_iterator, phase, ctx) req_dyn_hook_run_hook("timing", "before:plugin", plugin.name, ctx.plugin_id) end + req_dyn_hook_run_hook("etrace", "before:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + plugin.handler[phase](plugin.handler, configuration) if has_timing then req_dyn_hook_run_hook("timing", "after:plugin") end + req_dyn_hook_run_hook("etrace", "after:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + reset_plugin_context(ctx, old_ws) if span then @@ -358,6 +366,8 @@ local function execute_global_plugins_iterator(plugins_iterator, phase, ctx) if has_timing then req_dyn_hook_run_hook("timing", "after:plugin_iterator") end + + req_dyn_hook_run_hook("etrace", "after:plugin_iterator", old_ctx) end @@ -373,8 +383,11 @@ local function execute_collecting_plugins_iterator(plugins_iterator, phase, ctx) ctx.delay_response = true + req_dyn_hook_run_hook("etrace", "before:plugin_iterator", ctx) + local old_ws = ctx.workspace local has_timing = ctx.has_timing + local old_ctx = ctx if has_timing then req_dyn_hook_run_hook("timing", "before:plugin_iterator") @@ -393,6 +406,8 @@ local function execute_collecting_plugins_iterator(plugins_iterator, phase, ctx) req_dyn_hook_run_hook( "timing", "before:plugin", plugin.name, ctx.plugin_id) end + req_dyn_hook_run_hook("etrace", "before:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + local co = coroutine.create(plugin.handler[phase]) local cok, cerr = coroutine.resume(co, plugin.handler, configuration) @@ -400,6 +415,8 @@ local function execute_collecting_plugins_iterator(plugins_iterator, phase, ctx) req_dyn_hook_run_hook("timing", "after:plugin") end + req_dyn_hook_run_hook("etrace", "after:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + if not cok then -- set tracing error if span then @@ -430,6 +447,8 @@ local function execute_collecting_plugins_iterator(plugins_iterator, phase, ctx) req_dyn_hook_run_hook("timing", "after:plugin_iterator") end + req_dyn_hook_run_hook("etrace", "after:plugin_iterator", old_ctx) + ctx.delay_response = nil end @@ -444,8 +463,11 @@ local function execute_collected_plugins_iterator(plugins_iterator, phase, ctx) return end + req_dyn_hook_run_hook("etrace", "before:plugin_iterator", ctx) + local old_ws = ctx.workspace local has_timing = ctx.has_timing + local old_ctx = ctx if has_timing then req_dyn_hook_run_hook("timing", "before:plugin_iterator") @@ -463,12 +485,16 @@ local function execute_collected_plugins_iterator(plugins_iterator, phase, ctx) req_dyn_hook_run_hook("timing", "before:plugin", plugin.name, ctx.plugin_id) end + req_dyn_hook_run_hook("etrace", "before:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + plugin.handler[phase](plugin.handler, configuration) if has_timing then req_dyn_hook_run_hook("timing", "after:plugin") end + req_dyn_hook_run_hook("etrace", "after:a_plugin", old_ctx, plugin.name, ctx.plugin_id) + reset_plugin_context(ctx, old_ws) if span then @@ -479,6 +505,8 @@ local function execute_collected_plugins_iterator(plugins_iterator, phase, ctx) if has_timing then req_dyn_hook_run_hook("timing", "after:plugin_iterator") end + + req_dyn_hook_run_hook("etrace", "after:plugin_iterator", ctx) end @@ -791,6 +819,8 @@ function Kong.init() " token for request debugging: ", kong.request_debug_token) end + + etrace.init(config) end @@ -886,6 +916,7 @@ function Kong.init_worker() kong.timing = kong_global.init_timing() kong.timing.init_worker(kong.configuration.request_debug) + etrace.init_worker() if is_dbless(kong.configuration) then -- databases in LMDB need to be explicitly created, otherwise `get` @@ -1121,6 +1152,8 @@ function Kong.rewrite() local has_timing req_dyn_hook_run_hook("timing:auth", "auth") + req_dyn_hook_run_hook("timing:auth", "auth") + req_dyn_hook_run_hook("etrace", "before:rewrite", ctx) if req_dyn_hook_is_group_enabled("timing") then ctx.has_timing = true @@ -1159,6 +1192,8 @@ function Kong.rewrite() if has_timing then req_dyn_hook_run_hook("timing", "after:rewrite") end + + req_dyn_hook_run_hook("etrace", "after:rewrite", ctx) end @@ -1170,6 +1205,8 @@ function Kong.access() req_dyn_hook_run_hook("timing", "before:access") end + req_dyn_hook_run_hook("etrace", "before:access", ctx) + if not ctx.KONG_ACCESS_START then ctx.KONG_ACCESS_START = get_now_ms() @@ -1196,6 +1233,8 @@ function Kong.access() req_dyn_hook_run_hook("timing", "after:access") end + req_dyn_hook_run_hook("etrace", "after:access", ctx) + return flush_delayed_response(ctx) end @@ -1212,6 +1251,8 @@ function Kong.access() req_dyn_hook_run_hook("timing", "after:access") end + req_dyn_hook_run_hook("etrace", "after:access", ctx) + return kong.response.error(503, "no Service found with those values") end @@ -1233,6 +1274,8 @@ function Kong.access() req_dyn_hook_run_hook("timing", "after:access") end + req_dyn_hook_run_hook("etrace", "after:access", ctx) + return Kong.response() end @@ -1248,6 +1291,8 @@ function Kong.access() if has_timing then req_dyn_hook_run_hook("timing", "after:access") end + + req_dyn_hook_run_hook("etrace", "after:access", ctx) end @@ -1259,6 +1304,8 @@ function Kong.balancer() req_dyn_hook_run_hook("timing", "before:balancer") end + req_dyn_hook_run_hook("etrace", "before:balancer", ctx) + -- This may be called multiple times, and no yielding here! local now_ms = get_now_ms() local now_ns = time_ns() @@ -1341,6 +1388,8 @@ function Kong.balancer() req_dyn_hook_run_hook("timing", "after:balancer") end + req_dyn_hook_run_hook("etrace", "after:balancer", ctx) + return ngx.exit(errcode) end @@ -1353,6 +1402,8 @@ function Kong.balancer() req_dyn_hook_run_hook("timing", "after:balancer") end + req_dyn_hook_run_hook("etrace", "after:balancer", ctx) + return ngx.exit(500) end end @@ -1408,6 +1459,8 @@ function Kong.balancer() req_dyn_hook_run_hook("timing", "after:balancer") end + req_dyn_hook_run_hook("etrace", "after:balancer", ctx) + return ngx.exit(500) end @@ -1447,6 +1500,8 @@ function Kong.balancer() if has_timing then req_dyn_hook_run_hook("timing", "after:balancer") end + + req_dyn_hook_run_hook("etrace", "after:balancer", ctx) end @@ -1477,6 +1532,8 @@ do req_dyn_hook_run_hook("timing", "before:response") end + req_dyn_hook_run_hook("etrace", "before:response", ctx) + local plugins_iterator = runloop.get_plugins_iterator() -- buffered proxying (that also executes the balancer) @@ -1498,6 +1555,8 @@ do req_dyn_hook_run_hook("timing", "after:response") end + req_dyn_hook_run_hook("etrace", "after:response", ctx) + return kong_error_handlers(ctx) end @@ -1552,6 +1611,8 @@ do req_dyn_hook_run_hook("timing", "after:response") end + req_dyn_hook_run_hook("etrace", "after:response", ctx) + -- jump over the balancer to header_filter ngx.exit(status) end @@ -1566,6 +1627,8 @@ function Kong.header_filter() req_dyn_hook_run_hook("timing", "before:header_filter") end + req_dyn_hook_run_hook("etrace", "before:header_filter", ctx) + if not ctx.KONG_PROCESSING_START then ctx.KONG_PROCESSING_START = get_start_time_ms() end @@ -1637,6 +1700,8 @@ function Kong.header_filter() if has_timing then req_dyn_hook_run_hook("timing", "after:header_filter") end + + req_dyn_hook_run_hook("etrace", "after:header_filter", ctx) end @@ -1648,6 +1713,8 @@ function Kong.body_filter() req_dyn_hook_run_hook("timing", "before:body_filter") end + req_dyn_hook_run_hook("etrace", "before:body_filter", ctx) + if not ctx.KONG_BODY_FILTER_START then ctx.KONG_BODY_FILTER_START = get_now_ms() @@ -1707,6 +1774,8 @@ function Kong.body_filter() req_dyn_hook_run_hook("timing", "after:body_filter") end + req_dyn_hook_run_hook("etrace", "after:body_filter", ctx) + return end @@ -1728,6 +1797,8 @@ function Kong.body_filter() if has_timing then req_dyn_hook_run_hook("timing", "after:body_filter") end + + req_dyn_hook_run_hook("etrace", "after:body_filter", ctx) end @@ -1739,6 +1810,8 @@ function Kong.log() req_dyn_hook_run_hook("timing", "before:log") end + req_dyn_hook_run_hook("etrace", "before:log", ctx) + if not ctx.KONG_LOG_START then ctx.KONG_LOG_START = get_now_ms() ctx.KONG_LOG_START_NS = time_ns() @@ -1834,6 +1907,8 @@ function Kong.log() req_dyn_hook_run_hook("timing", "after:log") end + req_dyn_hook_run_hook("etrace", "after:log", ctx) + release_table(CTX_NS, ctx) -- this is not used for now, but perhaps we need it later? diff --git a/kong/runloop/handler.lua b/kong/runloop/handler.lua index 1f3210d2a889..587c7d8d3de6 100644 --- a/kong/runloop/handler.lua +++ b/kong/runloop/handler.lua @@ -1115,6 +1115,8 @@ return { req_dyn_hook_run_hook("timing", "before:router") end + req_dyn_hook_run_hook("etrace", "before:router", ctx) + -- routing request local router = get_updated_router() local match_t = router:exec(ctx) @@ -1123,6 +1125,8 @@ return { req_dyn_hook_run_hook("timing", "after:router") end + req_dyn_hook_run_hook("etrace", "after:router", ctx) + if not match_t then -- tracing if span then diff --git a/kong/templates/nginx.lua b/kong/templates/nginx.lua index 108d268f56be..8bb634c12209 100644 --- a/kong/templates/nginx.lua +++ b/kong/templates/nginx.lua @@ -71,6 +71,7 @@ wasm { > if role == "control_plane" or #proxy_listeners > 0 or #admin_listeners > 0 or #status_listeners > 0 then http { + lua_capture_error_log 32m; include 'nginx-kong.conf'; } > end diff --git a/lua-resty-protobuf/.bazelrc b/lua-resty-protobuf/.bazelrc new file mode 100644 index 000000000000..710c4d81ab58 --- /dev/null +++ b/lua-resty-protobuf/.bazelrc @@ -0,0 +1 @@ +common --noenable_bzlmod diff --git a/lua-resty-protobuf/.bazelversion b/lua-resty-protobuf/.bazelversion new file mode 100644 index 000000000000..dfda3e0b4f01 --- /dev/null +++ b/lua-resty-protobuf/.bazelversion @@ -0,0 +1 @@ +6.1.0 diff --git a/lua-resty-protobuf/.github/workflows/bazel.yml b/lua-resty-protobuf/.github/workflows/bazel.yml new file mode 100644 index 000000000000..ca6eab46fab6 --- /dev/null +++ b/lua-resty-protobuf/.github/workflows/bazel.yml @@ -0,0 +1,40 @@ +name: Bazel build + +on: + pull_request: + paths-ignore: + # ignore markdown files (CHANGELOG.md, README.md, etc.) + - '**/*.md' + push: + paths-ignore: + # ignore markdown files (CHANGELOG.md, README.md, etc.) + - '**/*.md' + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tests: + name: Tests + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Build + run: bazel build :resty_protobuf --verbose_failures + + - name: Ensure `ffi` feature is present (Linux) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: readelf -Ws --dyn-syms bazel-bin/libresty_protobuf.so | grep lua_resty_protobuf_ + + - name: Ensure `ffi` feature is present (macOS) + if: ${{ matrix.os == 'macos-latest' }} + run: nm -gU bazel-bin/libresty_protobuf.dylib | grep lua_resty_protobuf_ diff --git a/lua-resty-protobuf/.gitignore b/lua-resty-protobuf/.gitignore new file mode 100644 index 000000000000..d7cc386381c8 --- /dev/null +++ b/lua-resty-protobuf/.gitignore @@ -0,0 +1,2 @@ +/target +bazel-* diff --git a/lua-resty-protobuf/BUILD b/lua-resty-protobuf/BUILD new file mode 100644 index 000000000000..7578986afbcc --- /dev/null +++ b/lua-resty-protobuf/BUILD @@ -0,0 +1,36 @@ +load("@rules_rust//rust:defs.bzl", "rust_shared_library") +load("@resty_protobuf_crate_index//:defs.bzl", "aliases", "all_crate_deps") + +filegroup( + name = "all_srcs", + srcs = glob( + include = [ + "src/**", + "Cargo.toml", + "Cargo.lock", + ], + exclude = [ + "*.bazel", + ], + ), +) + +rust_shared_library( + name = "resty_protobuf", + srcs = [":all_srcs"], + aliases = aliases(), + rustc_flags = [ + "--codegen=strip=symbols", + ], + crate_features = [ + "default", + "ffi", + ], + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + visibility = ["//visibility:public"], + deps = all_crate_deps( + normal = True, + ), +) diff --git a/lua-resty-protobuf/Cargo.lock b/lua-resty-protobuf/Cargo.lock new file mode 100644 index 000000000000..e750162e4926 --- /dev/null +++ b/lua-resty-protobuf/Cargo.lock @@ -0,0 +1,1151 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "anyhow" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-channel" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "h2" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float", + "percent-encoding", + "rand", + "serde_json", + "thiserror", +] + +[[package]] +name = "ordered-float" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost_test" +version = "0.1.0" +dependencies = [ + "bytes", + "opentelemetry-proto", + "prost", + "rand", + "snap", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "syn" +version = "2.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "thiserror" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" diff --git a/lua-resty-protobuf/Cargo.toml b/lua-resty-protobuf/Cargo.toml new file mode 100644 index 000000000000..a33d2de4b097 --- /dev/null +++ b/lua-resty-protobuf/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "prost_test" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bytes = "1.6.0" +prost = "0.12.3" +rand = "0.8.5" +opentelemetry-proto = { version = "0.5.0", features = ["gen-tonic", "trace", "logs", "metrics"] } +snap = "1.1.1" + +[lib] +crate-type = ["lib", "cdylib", "staticlib"] + +[features] +default = ["ffi"] +ffi = [] diff --git a/lua-resty-protobuf/Makefile b/lua-resty-protobuf/Makefile new file mode 100644 index 000000000000..333fc8b296a0 --- /dev/null +++ b/lua-resty-protobuf/Makefile @@ -0,0 +1,14 @@ +OPENRESTY_PREFIX?= /usr/local/openresty +LUA_LIB_DIR ?= $(OPENRESTY_PREFIX)/lualib + +LIB_NAME = libprost_test + +ifeq ($(shell uname), Darwin) + SUFFIX = dylib +else + SUFFIX = so +endif + +install-lualib: + $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty/kong/kproto/ + $(INSTALL) -m 664 lualib/resty/kong/kproto/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/kong/kproto/ \ No newline at end of file diff --git a/lua-resty-protobuf/WORKSPACE b/lua-resty-protobuf/WORKSPACE new file mode 100644 index 000000000000..4affd7abb841 --- /dev/null +++ b/lua-resty-protobuf/WORKSPACE @@ -0,0 +1,13 @@ +workspace(name = "resty_protobuf") + +load("//build:repos.bzl", "resty_protobuf_repositories") + +resty_protobuf_repositories() + +load("//build:deps.bzl", "resty_protobuf_dependencies") + +resty_protobuf_dependencies(cargo_home_isolated = False) # use system `$CARGO_HOME` to speed up builds + +load("//build:crates.bzl", "resty_protobuf_crates") + +resty_protobuf_crates() diff --git a/lua-resty-protobuf/build/BUILD.bazel b/lua-resty-protobuf/build/BUILD.bazel new file mode 100644 index 000000000000..ffd0fb0cdc5b --- /dev/null +++ b/lua-resty-protobuf/build/BUILD.bazel @@ -0,0 +1 @@ +package(default_visibility = ["//visibility:public"]) diff --git a/lua-resty-protobuf/build/README.md b/lua-resty-protobuf/build/README.md new file mode 100644 index 000000000000..02238a080e3d --- /dev/null +++ b/lua-resty-protobuf/build/README.md @@ -0,0 +1,44 @@ +# Bazel project for atc-router + + +To use in other Bazel projects, add the following to your WORKSPACE file: + +```python + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +git_repository( + name = "resty_protobuf", + branch = "some-tag", + remote = "https://github.com/Kong/atc-router", +) + +load("@resty_protobuf//build:repos.bzl", "resty_protobuf_repositories") + +resty_protobuf_repositories() + +load("@resty_protobuf//build:deps.bzl", "resty_protobuf_dependencies") + +resty_protobuf_dependencies(cargo_home_isolated = False) # use system `$CARGO_HOME` to speed up builds + +load("@resty_protobuf//build:crates.bzl", "resty_protobuf_crates") + +resty_protobuf_crates() + + +``` + +In your rule, add `resty_protobuf` as dependency: + +```python +configure_make( + name = "openresty", + # ... + deps = [ + "@resty_protobuf", + ], +) +``` + +When building this library in Bazel, use the `-c opt` flag to ensure optimal performance. The default fastbuild mode produces a less performant binary. \ No newline at end of file diff --git a/lua-resty-protobuf/build/crates.bzl b/lua-resty-protobuf/build/crates.bzl new file mode 100644 index 000000000000..53d16e11c1bd --- /dev/null +++ b/lua-resty-protobuf/build/crates.bzl @@ -0,0 +1,6 @@ +"""Setup Crates repostories """ + +load("@resty_protobuf_crate_index//:defs.bzl", "crate_repositories") + +def resty_protobuf_crates(): + crate_repositories() diff --git a/lua-resty-protobuf/build/deps.bzl b/lua-resty-protobuf/build/deps.bzl new file mode 100644 index 000000000000..fdc2d10e0d03 --- /dev/null +++ b/lua-resty-protobuf/build/deps.bzl @@ -0,0 +1,39 @@ +"""Setup dependencies after repostories are downloaded.""" + +load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains", "rust_repository_set") +load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") +load("@rules_rust//crate_universe:defs.bzl", "crates_repository") + +def resty_protobuf_dependencies(cargo_home_isolated = True): + """ + resty_protobuf_dependencies setup rust toolchain and cargo dependency repositories. + + Args: + cargo_home_isolated (bool): cargo_home_isolated to False to reuse system CARGO_HOME + for faster builds. cargo_home_isolated is default False and will use isolated + Cargo home, which takes about 2 minutes to bootstrap. + """ + rules_rust_dependencies() + + rust_register_toolchains( + edition = "2021", + extra_target_triples = ["aarch64-unknown-linux-gnu"], + ) + + rust_repository_set( + name = "rust_linux_arm64_linux_tuple", + edition = "2021", + exec_triple = "x86_64-unknown-linux-gnu", + extra_target_triples = ["aarch64-unknown-linux-gnu"], + versions = ["stable"], + ) + + crate_universe_dependencies() + + crates_repository( + name = "resty_protobuf_crate_index", + cargo_lockfile = "@resty_protobuf//:Cargo.lock", + # lockfile = "//:Cargo.Bazel.lock", + manifests = ["@resty_protobuf//:Cargo.toml"], + isolated = cargo_home_isolated, + ) diff --git a/lua-resty-protobuf/build/repos.bzl b/lua-resty-protobuf/build/repos.bzl new file mode 100644 index 000000000000..82caf731ed47 --- /dev/null +++ b/lua-resty-protobuf/build/repos.bzl @@ -0,0 +1,19 @@ +"""Setup repostories.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def resty_protobuf_repositories(): + http_archive( + name = "bazel_skylib", + sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + ], + ) + + http_archive( + name = "rules_rust", + integrity = "sha256-ZQGWDD5NoySV0eEAfe0HaaU0yxlcMN6jaqVPnYo/A2E=", + urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.38.0/rules_rust-v0.38.0.tar.gz"], + ) diff --git a/lua-resty-protobuf/cbindgen.toml b/lua-resty-protobuf/cbindgen.toml new file mode 100644 index 000000000000..36b5ee371e80 --- /dev/null +++ b/lua-resty-protobuf/cbindgen.toml @@ -0,0 +1,5 @@ +language = "C" +header = "/* Generated by cbindgen. Do NOT edit. */" + +[enum] +prefix_with_name = true diff --git a/lua-resty-protobuf/lualib/resty/kong/kproto/cdefs.lua b/lua-resty-protobuf/lualib/resty/kong/kproto/cdefs.lua new file mode 100644 index 000000000000..50e45eb77830 --- /dev/null +++ b/lua-resty-protobuf/lualib/resty/kong/kproto/cdefs.lua @@ -0,0 +1,333 @@ +local ffi = require("ffi") +local base = require("resty.core.base") + +local math_floor = math.floor +local get_string_buf = base.get_string_buf + +-- From: https://github.com/openresty/lua-resty-signal/blob/master/lib/resty/signal.lua +local load_shared_lib +do + local tostring = tostring + local string_gmatch = string.gmatch + local string_match = string.match + local io_open = io.open + local io_close = io.close + local table_new = require("table.new") + + local cpath = package.cpath + + function load_shared_lib(so_name) + local tried_paths = table_new(32, 0) + local i = 1 + + for k, _ in string_gmatch(cpath, "[^;]+") do + local fpath = tostring(string_match(k, "(.*/)")) + fpath = fpath .. so_name + -- Don't get me wrong, the only way to know if a file exist is + -- trying to open it. + local f = io_open(fpath) + if f ~= nil then + io_close(f) + return ffi.load(fpath, true) + end + + tried_paths[i] = fpath + i = i + 1 + end + + return nil, tried_paths + end -- function +end -- do + +local lib_name = ffi.os == "OSX" and "libresty_protobuf.dylib" or "libresty_protobuf.so" + +local clib, tried_paths = load_shared_lib(lib_name) +if not clib then + error(("could not load %s shared library from the following paths:\n"):format(lib_name) .. + table.concat(tried_paths, "\n"), 2) +end + +-- generated by `cbindgen -c cbindgen.toml` +ffi.cdef([[ + /* Generated by cbindgen. Do NOT edit. */ + + typedef struct Logs Logs; + + typedef struct Metrics Metrics; + + typedef struct Traces Traces; + + struct Traces *lua_resty_protobuf_trace_new(void); + + unsigned int lua_resty_protobuf_trace_get_serialized(struct Traces *traces, + unsigned char *buf, + unsigned int buf_len); + + int lua_resty_protobuf_trace_enter_span(struct Traces *traces, + const char *name, + unsigned int name_len, + unsigned char *err_buf, + unsigned int err_buf_len); + + void lua_resty_protobuf_trace_exit_span(struct Traces *traces); + + int lua_resty_protobuf_trace_add_string_attribute(struct Traces *traces, + const char *key, + unsigned int key_len, + const char *value, + unsigned int value_len, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_trace_add_bool_attribute(struct Traces *traces, + const char *key, + unsigned int key_len, + int value, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_trace_add_int64_attribute(struct Traces *traces, + const char *key, + unsigned int key_len, + long long value, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_trace_add_double_attribute(struct Traces *traces, + const char *key, + unsigned int key_len, + double value, + unsigned char *err_buf, + unsigned int err_buf_len); + + struct Metrics *lua_resty_protobuf_metrics_new(void); + + unsigned int lua_resty_protobuf_metrics_get_serialized(struct Metrics *metrics, + unsigned char *buf, + unsigned int buf_len); + + int lua_resty_protobuf_metrics_add_gauge(struct Metrics *metrics, + const char *name, + unsigned int name_len, + long long value, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_metrics_add_sum(struct Metrics *metrics, + const char *name, + unsigned int name_len, + long long value, + unsigned char *err_buf, + unsigned int err_buf_len); + + struct Logs *lua_resty_protobuf_logs_new(void); + + unsigned int lua_resty_protobuf_logs_get_serialized(struct Logs *logs, + unsigned char *buf, + unsigned int buf_len); + + int lua_resty_protobuf_logs_add_info(struct Logs *logs, + unsigned long long time_unix_nano, + const char *message, + unsigned int message_len, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_logs_add_warn(struct Logs *logs, + unsigned long long time_unix_nano, + const char *message, + unsigned int message_len, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_logs_add_error(struct Logs *logs, + unsigned long long time_unix_nano, + const char *message, + unsigned int message_len, + unsigned char *err_buf, + unsigned int err_buf_len); + + int lua_resty_protobuf_logs_add_fatal(struct Logs *logs, + unsigned long long time_unix_nano, + const char *message, + unsigned int message_len, + unsigned char *err_buf, + unsigned int err_buf_len); +]]) + + +local _M = {} + +return { + clib = clib, +} + + + +-- function _M.new() +-- local handle = clib.lua_resty_protobuf_trace_new() + +-- -- avoid tail call +-- return handle +-- end + +-- function _M.free(handle) +-- clib.lua_resty_protobuf_trace_free(handle) +-- end + +-- function _M.enter_span(handle, name) +-- local err_buf = get_string_buf(ERROR_BUF_LEN) +-- local rc = clib.lua_resty_protobuf_trace_enter_span(handle, name, #name, err_buf, ERROR_BUF_LEN) + +-- if rc ~= 0 then +-- return false, ffi.string(err_buf) +-- end + +-- return true +-- end + +-- function _M.add_string_attribute(handle, name, val) +-- local err_buf = get_string_buf(ERROR_BUF_LEN) +-- local rc = clib.lua_resty_protobuf_trace_add_string_attribute(handle, name, #name, val, #val, err_buf, ERROR_BUF_LEN) + +-- if rc ~= 0 then +-- return false, ffi.string(err_buf) +-- end + +-- return true +-- end + +-- function _M.add_bool_attribute(handle, name, val) +-- local err_buf = get_string_buf(ERROR_BUF_LEN) +-- local rc = clib.lua_resty_protobuf_trace_add_bool_attribute(handle, name, #name, val and 1 or 0, err_buf, ERROR_BUF_LEN) + +-- if rc ~= 0 then +-- return false, ffi.string(err_buf) +-- end + +-- return true +-- end + +-- function _M.add_int64_attribute(handle, name, val) +-- local err_buf = get_string_buf(ERROR_BUF_LEN) +-- local rc = clib.lua_resty_protobuf_trace_add_int64_attribute(handle, name, #name, val, err_buf, ERROR_BUF_LEN) + +-- if rc ~= 0 then +-- return false, ffi.string(err_buf) +-- end + +-- return true +-- end + +-- function _M.add_double_attribute(handle, name, val) +-- local err_buf = get_string_buf(ERROR_BUF_LEN) +-- local rc = clib.lua_resty_protobuf_trace_add_double_attribute(handle, name, #name, val, err_buf, ERROR_BUF_LEN) + +-- if rc ~= 0 then +-- return false, ffi.string(err_buf) +-- end + +-- return true +-- end + +-- function _M.exit_span(handle) +-- clib.lua_resty_protobuf_trace_exit_span(handle) +-- end + + +-- local BUF_INIT_LEN = 4096 +-- local BUF_MAX_LEN = 40960 +-- local buf_cur_len = BUF_INIT_LEN +-- local buf = ffi.new("uint8_t[?]", buf_cur_len) + +-- -- we are not going to call this function now +-- -- as we would like to send the data on the C-land, +-- -- cosocket is not fast enough. +-- function _M.get_serialized(handle) +-- local sz = 0 + +-- repeat +-- sz = clib.lua_resty_protobuf_trace_get_serialized(handle, buf, buf_cur_len) + +-- if sz == 0 then +-- buf_cur_len = buf_cur_len * 2 +-- buf = ffi.new("uint8_t[?]", buf_cur_len) +-- end +-- until sz ~= 0 + +-- local serialized = ffi.string(buf, sz) + +-- if buf_cur_len > BUF_MAX_LEN then +-- buf_cur_len = math_floor(buf_cur_len / 2) +-- buf = ffi.new("uint8_t[?]", buf_cur_len) +-- end + +-- return serialized +-- end + +-- function _M.serialize_log(time_unix_nano, observed_time_unix_nano, message) +-- local buf = ffi.new("uint8_t[?]", 10240) +-- local rc = clib.lua_resty_protobuf_serialize_log(time_unix_nano, observed_time_unix_nano, message, #message, buf, 10240) + +-- assert(rc ~= 0, "failed to serialize log") + +-- return ffi.string(buf, rc) +-- end + +-- function _M.serialize_metrics_gauge(name, time_unix_nano, value) +-- local buf = ffi.new("uint8_t[?]", 10240) +-- local rc = clib.lua_resty_protobuf_serialize_metrics_gauge(name, #name, time_unix_nano, value, buf, 10240) + +-- assert(rc ~= 0, "failed to serialize metrics gauge") + +-- return ffi.string(buf, rc) +-- end + +-- function _M.metrics_new() +-- local handle = clib.lua_resty_protobuf_metrics_new() + +-- -- avoid tail call +-- return handle +-- end + +-- function _M.metrics_free(handle) +-- clib.lua_resty_protobuf_metrics_free(handle) +-- end + +-- function _M.metrics_get_serialized(handle) +-- local sz = 0 + +-- repeat +-- sz = clib.lua_resty_protobuf_metrics_get_serialized(handle, buf, buf_cur_len) + +-- if sz == 0 then +-- buf_cur_len = buf_cur_len * 2 +-- buf = ffi.new("uint8_t[?]", buf_cur_len) +-- end +-- until sz ~= 0 + +-- local serialized = ffi.string(buf, sz) + +-- if buf_cur_len > BUF_MAX_LEN then +-- buf_cur_len = math_floor(buf_cur_len / 2) +-- buf = ffi.new("uint8_t[?]", buf_cur_len) +-- end + +-- return serialized +-- end + +-- function _M.metrics_add_gauge(handle, name, value) +-- local rc = clib.lua_resty_protobuf_metrics_add_gauge(handle, name, #name, math.floor(value)) + +-- assert(rc == 0, "failed to add gauge") +-- end + +-- function _M.metrics_add_sum(handle, name, value) +-- local rc = clib.lua_resty_protobuf_metrics_add_sum(handle, name, #name, math.floor(value)) + +-- assert(rc == 0, "failed to add sum") +-- end + + + +-- return _M diff --git a/lua-resty-protobuf/lualib/resty/kong/kproto/log.lua b/lua-resty-protobuf/lualib/resty/kong/kproto/log.lua new file mode 100644 index 000000000000..72cd9cd20fc9 --- /dev/null +++ b/lua-resty-protobuf/lualib/resty/kong/kproto/log.lua @@ -0,0 +1,76 @@ +local ffi = require("ffi") +local base = require("resty.core.base") +local clib = require("resty.kong.kproto.cdefs").clib + +local get_string_buf = base.get_string_buf +local ffi_string = ffi.string + +local err_buf_len = 1024 +local err_buf = ffi.new("uint8_t[?]", err_buf_len) + +local DEFAULT_SERIALIZATION_BUF_SIZE = 1024 * 1024 * 5 -- 5MB + +local _M = {} + +function _M.new() + local hdl = clib.lua_resty_protobuf_logs_new() + return hdl +end + + +function _M.get_serialized(hdl, trace_id) + local buf = get_string_buf(DEFAULT_SERIALIZATION_BUF_SIZE) + local rc = clib.lua_resty_protobuf_logs_get_serialized(hdl, buf, DEFAULT_SERIALIZATION_BUF_SIZE) + + assert(rc > 0, "failed to get serialized log") + + local binary_data = ffi_string(buf, rc) + return binary_data +end + + +function _M.add_info(hdl, time_unix_nano, message) + local rc = clib.lua_resty_protobuf_logs_add_info(hdl, time_unix_nano, message, #message, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_warn(hdl, time_unix_nano, message) + local rc = clib.lua_resty_protobuf_logs_add_warn(hdl, time_unix_nano, message, #message, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_error(hdl, time_unix_nano, message) + local rc = clib.lua_resty_protobuf_logs_add_error(hdl, time_unix_nano, message, #message, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_fatal(hdl, time_unix_nano, message) + local rc = clib.lua_resty_protobuf_logs_add_fatal(hdl, time_unix_nano, message, #message, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +return _M diff --git a/lua-resty-protobuf/lualib/resty/kong/kproto/metric.lua b/lua-resty-protobuf/lualib/resty/kong/kproto/metric.lua new file mode 100644 index 000000000000..8008165ea8f5 --- /dev/null +++ b/lua-resty-protobuf/lualib/resty/kong/kproto/metric.lua @@ -0,0 +1,55 @@ +local ffi = require("ffi") +local base = require("resty.core.base") +local clib = require("resty.kong.kproto.cdefs").clib + +local math_floor = math.floor +local get_string_buf = base.get_string_buf +local ffi_string = ffi.string + +local err_buf_len = 1024 +local err_buf = ffi.new("uint8_t[?]", err_buf_len) + +local DEFAULT_SERIALIZATION_BUF_SIZE = 10240 + +local _M = {} + +function _M.new() + local hdl = clib.lua_resty_protobuf_metrics_new() + return hdl +end + + +function _M.get_serialized(hdl, trace_id) + local buf = get_string_buf(DEFAULT_SERIALIZATION_BUF_SIZE) + local rc = clib.lua_resty_protobuf_metrics_get_serialized(hdl, buf, DEFAULT_SERIALIZATION_BUF_SIZE) + + assert(rc > 0, "failed to get serialized metrics") + + local binary_data = ffi_string(buf, rc) + return binary_data +end + + +function _M.add_gauge(hdl, key, value) + local rc = clib.lua_resty_protobuf_metrics_add_gauge(hdl, key, #key, math_floor(value), err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_sum(hdl, key, value) + local rc = clib.lua_resty_protobuf_metrics_add_sum(hdl, key, #key, math_floor(value), err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +return _M diff --git a/lua-resty-protobuf/lualib/resty/kong/kproto/trace.lua b/lua-resty-protobuf/lualib/resty/kong/kproto/trace.lua new file mode 100644 index 000000000000..4a2efad4eb3e --- /dev/null +++ b/lua-resty-protobuf/lualib/resty/kong/kproto/trace.lua @@ -0,0 +1,92 @@ +local ffi = require("ffi") +local base = require("resty.core.base") +local clib = require("resty.kong.kproto.cdefs").clib + +local get_string_buf = base.get_string_buf +local ffi_string = ffi.string + +local err_buf_len = 1024 +local err_buf = ffi.new("uint8_t[?]", err_buf_len) + +local DEFAULT_SERIALIZATION_BUF_SIZE = 10240 + +local _M = {} + +function _M.new() + local hdl = clib.lua_resty_protobuf_trace_new() + return hdl +end + + +function _M.get_serialized(hdl, trace_id) + local buf = get_string_buf(DEFAULT_SERIALIZATION_BUF_SIZE) + local rc = clib.lua_resty_protobuf_trace_get_serialized(hdl, buf, DEFAULT_SERIALIZATION_BUF_SIZE) + + assert(rc ~= 0, "failed to get serialized trace") + + local binary_data = ffi_string(buf, rc) + return binary_data +end + + +function _M.enter_span(hdl, name) + local rc = clib.lua_resty_protobuf_trace_enter_span(hdl, name, #name, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.exit_span(hdl) + clib.lua_resty_protobuf_trace_exit_span(hdl) +end + + +function _M.add_string_attribute(hdl, key, value) + local rc = clib.lua_resty_protobuf_trace_add_string_attribute(hdl, key, #key, value, #value, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_bool_attribute(hdl, key, value) + local rc = clib.lua_resty_protobuf_trace_add_bool_attribute(hdl, key, #key, value and 1 or 0, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_int64_attribute(hdl, key, value) + local rc = clib.lua_resty_protobuf_trace_add_int64_attribute(hdl, key, #key, value, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +function _M.add_double_attribute(hdl, key, value) + local rc = clib.lua_resty_protobuf_trace_add_double_attribute(hdl, key, #key, value, err_buf, err_buf_len) + + if rc ~= 0 then + return nil, ffi_string(err_buf, rc) + end + + return true +end + + +return _M diff --git a/lua-resty-protobuf/src/ffi/logs.rs b/lua-resty-protobuf/src/ffi/logs.rs new file mode 100644 index 000000000000..dea42dfebe27 --- /dev/null +++ b/lua-resty-protobuf/src/ffi/logs.rs @@ -0,0 +1,141 @@ +use bytes::BytesMut; + +use crate::Logs; +use std::ffi; + +use super::utils::write_error_buffer; + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_new() -> *mut Logs { + Box::into_raw(Box::new(Logs::new())) +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_get_serialized( + logs: *mut Logs, + buf: *mut ffi::c_uchar, + buf_len: ffi::c_uint, +) -> ffi::c_uint { + let logs = unsafe { Box::from_raw(logs) }; + let mut serialized = BytesMut::with_capacity(buf_len as usize); + logs.get_serialized(&mut serialized); + + let expected_len = serialized.len(); + if expected_len > buf_len as usize { + return 0; + } + + unsafe { + std::ptr::copy_nonoverlapping(serialized.as_ptr(), buf, expected_len); + } + + expected_len as ffi::c_uint +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_add_info( + logs: *mut Logs, + time_unix_nano: ffi::c_ulonglong, + message: *const ffi::c_char, + message_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let logs = unsafe { &mut *logs }; + let message = unsafe { std::slice::from_raw_parts(message as *const u8, message_len as usize) }; + let message = std::str::from_utf8(message); + + if let Err(e) = message { + let mut err_msg = String::new(); + err_msg.push_str("arguement `message` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + logs.add_info_log(time_unix_nano, message.unwrap()); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_add_warn( + logs: *mut Logs, + time_unix_nano: ffi::c_ulonglong, + message: *const ffi::c_char, + message_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let logs = unsafe { &mut *logs }; + let message = unsafe { std::slice::from_raw_parts(message as *const u8, message_len as usize) }; + let message = std::str::from_utf8(message); + + if let Err(e) = message { + let mut err_msg = String::new(); + err_msg.push_str("arguement `message` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + logs.add_warning_log(time_unix_nano, message.unwrap()); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_add_error( + logs: *mut Logs, + time_unix_nano: ffi::c_ulonglong, + message: *const ffi::c_char, + message_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let logs = unsafe { &mut *logs }; + let message = unsafe { std::slice::from_raw_parts(message as *const u8, message_len as usize) }; + let message = std::str::from_utf8(message); + + if let Err(e) = message { + let mut err_msg = String::new(); + err_msg.push_str("arguement `message` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + logs.add_error_log(time_unix_nano, message.unwrap()); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_logs_add_fatal( + logs: *mut Logs, + time_unix_nano: ffi::c_ulonglong, + message: *const ffi::c_char, + message_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let logs = unsafe { &mut *logs }; + let message = unsafe { std::slice::from_raw_parts(message as *const u8, message_len as usize) }; + let message = std::str::from_utf8(message); + + if let Err(e) = message { + let mut err_msg = String::new(); + err_msg.push_str("arguement `message` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + logs.add_fatal_log(time_unix_nano, message.unwrap()); + + 0 +} diff --git a/lua-resty-protobuf/src/ffi/metrics.rs b/lua-resty-protobuf/src/ffi/metrics.rs new file mode 100644 index 000000000000..da4e5ee939b1 --- /dev/null +++ b/lua-resty-protobuf/src/ffi/metrics.rs @@ -0,0 +1,87 @@ +use bytes::BytesMut; + +use crate::Metrics; +use std::ffi; + +use super::utils::write_error_buffer; + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_metrics_new() -> *mut Metrics { + Box::into_raw(Box::new(Metrics::new())) +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_metrics_get_serialized( + metrics: *mut Metrics, + buf: *mut ffi::c_uchar, + buf_len: ffi::c_uint, +) -> ffi::c_uint { + let metrics = unsafe { Box::from_raw(metrics) }; + let mut serialized = BytesMut::with_capacity(buf_len as usize); + metrics.get_serialized(&mut serialized); + + let expected_len = serialized.len(); + if expected_len > buf_len as usize { + return 0; + } + + unsafe { + std::ptr::copy_nonoverlapping(serialized.as_ptr(), buf, expected_len); + } + + expected_len as ffi::c_uint +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_metrics_add_gauge( + metrics: *mut Metrics, + name: *const ffi::c_char, + name_len: ffi::c_uint, + value: ffi::c_longlong, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let metrics = unsafe { &mut *metrics }; + let name = unsafe { std::slice::from_raw_parts(name as *const u8, name_len as usize) }; + let name = std::str::from_utf8(name); + + if let Err(e) = name { + let mut err_msg = String::new(); + err_msg.push_str("arguement `name` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + metrics.add_int64_gauge(name.unwrap(), value); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_metrics_add_sum( + metrics: *mut Metrics, + name: *const ffi::c_char, + name_len: ffi::c_uint, + value: ffi::c_longlong, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let metrics = unsafe { &mut *metrics }; + let name = unsafe { std::slice::from_raw_parts(name as *const u8, name_len as usize) }; + let name = std::str::from_utf8(name); + + if let Err(e) = name { + let mut err_msg = String::new(); + err_msg.push_str("arguement `name` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + metrics.add_int64_sum(name.unwrap(), value); + + 0 +} diff --git a/lua-resty-protobuf/src/ffi/mod.rs b/lua-resty-protobuf/src/ffi/mod.rs new file mode 100644 index 000000000000..82e7f2f130c5 --- /dev/null +++ b/lua-resty-protobuf/src/ffi/mod.rs @@ -0,0 +1,4 @@ +mod trace; +mod metrics; +mod logs; +mod utils; \ No newline at end of file diff --git a/lua-resty-protobuf/src/ffi/trace.rs b/lua-resty-protobuf/src/ffi/trace.rs new file mode 100644 index 000000000000..9eb6093bd845 --- /dev/null +++ b/lua-resty-protobuf/src/ffi/trace.rs @@ -0,0 +1,187 @@ +use bytes::BytesMut; + +use crate::Traces; +use std::ffi; + +use super::utils::write_error_buffer; + +#[no_mangle] +pub unsafe extern "C" fn lua_resty_protobuf_trace_new() -> *mut Traces { + Box::into_raw(Box::new(Traces::new())) +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_get_serialized( + traces: *mut Traces, + buf: *mut ffi::c_uchar, + buf_len: ffi::c_uint, +) -> ffi::c_uint { + let traces = unsafe { Box::from_raw(traces) }; + let mut serialized = BytesMut::with_capacity(buf_len as usize); + traces.get_serialized(&mut serialized); + + let expected_len = serialized.len(); + if expected_len > buf_len as usize { + return 0; + } + + unsafe { + std::ptr::copy_nonoverlapping(serialized.as_ptr(), buf, expected_len); + } + + expected_len as ffi::c_uint +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_enter_span( + traces: *mut Traces, + name: *const ffi::c_char, + name_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let traces = unsafe { &mut *traces }; + let name = unsafe { std::slice::from_raw_parts(name as *const u8, name_len as usize) }; + let name = std::str::from_utf8(name); + + if let Err(e) = name { + let mut err_msg = String::new(); + err_msg.push_str("arguement `name` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + traces.enter_span(name.unwrap()); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_exit_span(traces: *mut Traces) { + let traces = unsafe { &mut *traces }; + traces.exit_span(); +} + + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_add_string_attribute( + traces: *mut Traces, + key: *const ffi::c_char, + key_len: ffi::c_uint, + value: *const ffi::c_char, + value_len: ffi::c_uint, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let traces = unsafe { &mut *traces }; + let key = unsafe { std::slice::from_raw_parts(key as *const u8, key_len as usize) }; + let key = std::str::from_utf8(key); + + if let Err(e) = key { + let mut err_msg = String::new(); + err_msg.push_str("arguement `key` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + let value = unsafe { std::slice::from_raw_parts(value as *const u8, value_len as usize) }; + let value = std::str::from_utf8(value); + + if let Err(e) = value { + let mut err_msg = String::new(); + err_msg.push_str("arguement `value` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + traces.add_string_attribute(key.unwrap(), value.unwrap()); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_add_bool_attribute( + traces: *mut Traces, + key: *const ffi::c_char, + key_len: ffi::c_uint, + value: ffi::c_int, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let trace = unsafe { &mut *traces }; + let key = unsafe { std::slice::from_raw_parts(key as *const u8, key_len as usize) }; + let key = std::str::from_utf8(key); + + if let Err(e) = key { + let mut err_msg = String::new(); + err_msg.push_str("arguement `key` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + trace.add_bool_attribute(key.unwrap(), value != 0); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_add_int64_attribute( + traces: *mut Traces, + key: *const ffi::c_char, + key_len: ffi::c_uint, + value: ffi::c_longlong, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let trace = unsafe { &mut *traces }; + let key = unsafe { std::slice::from_raw_parts(key as *const u8, key_len as usize) }; + let key = std::str::from_utf8(key); + + if let Err(e) = key { + let mut err_msg = String::new(); + err_msg.push_str("arguement `key` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + trace.add_int64_attribute(key.unwrap(), value); + + 0 +} + +#[no_mangle] +pub extern "C" fn lua_resty_protobuf_trace_add_double_attribute( + traces: *mut Traces, + key: *const ffi::c_char, + key_len: ffi::c_uint, + value: ffi::c_double, + err_buf: *mut ffi::c_uchar, + err_buf_len: ffi::c_uint, +) -> ffi::c_int { + let trace = unsafe { &mut *traces }; + let key = unsafe { std::slice::from_raw_parts(key as *const u8, key_len as usize) }; + let key = std::str::from_utf8(key); + + if let Err(e) = key { + let mut err_msg = String::new(); + err_msg.push_str("arguement `key` is not a valid utf-8 string: "); + err_msg.push_str(e.to_string().as_str()); + write_error_buffer(err_buf, err_buf_len, &err_msg); + + return 1; + } + + trace.add_double_attribute(key.unwrap(), value); + + 0 +} diff --git a/lua-resty-protobuf/src/ffi/utils.rs b/lua-resty-protobuf/src/ffi/utils.rs new file mode 100644 index 000000000000..e2a1765331cd --- /dev/null +++ b/lua-resty-protobuf/src/ffi/utils.rs @@ -0,0 +1,13 @@ +use std::{cmp::min, ffi}; + +pub fn write_error_buffer(err_buf: *mut ffi::c_uchar, err_buf_len: ffi::c_uint, err_msg: &String) { + if err_buf_len as usize == 0 { + return; + } + + let len = min((err_buf_len as usize) - 1, err_msg.len()); + unsafe { + std::ptr::copy_nonoverlapping(err_msg.as_ptr(), err_buf, len); + *err_buf.add(len) = 0; + } +} diff --git a/lua-resty-protobuf/src/lib.rs b/lua-resty-protobuf/src/lib.rs new file mode 100644 index 000000000000..9af94f9d3f1e --- /dev/null +++ b/lua-resty-protobuf/src/lib.rs @@ -0,0 +1,413 @@ +mod ffi; + +use prost::Message; +use rand::prelude::*; +use std::time; + +use bytes::BytesMut; + +use opentelemetry_proto::tonic::{ + common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}, + logs::v1::{LogRecord, LogsData, ResourceLogs, ScopeLogs, SeverityNumber}, + metrics::v1::{ + metric::Data, number_data_point::Value, Gauge, Metric, MetricsData, NumberDataPoint, + ResourceMetrics, ScopeMetrics, Sum, + }, + resource::v1::Resource, + trace::v1::{ResourceSpans, ScopeSpans, Span, TracesData}, +}; + +const DEFAULT_CONTEXT_CAPACITY: usize = 16; + +fn _get_current_time_unix_nano() -> u64 { + let now = time::SystemTime::now(); + let nano = now.duration_since(time::UNIX_EPOCH).unwrap().as_nanos(); + + let nano_u64 = nano.try_into(); + if let Ok(nano_u64) = nano_u64 { + nano_u64 + } else { + // TODO: error handlingx + panic!("Failed to convert time to u64"); + } +} + +pub struct Traces { + rng: ThreadRng, + context: Vec, + + trace_id: [u8; 16], + + traces_data: TracesData, + resource_spans: ResourceSpans, + scope_spans: ScopeSpans, +} + +impl Traces { + pub fn new() -> Self { + let mut rng = thread_rng(); + let mut trace_id = [0; 16]; + rng.fill_bytes(&mut trace_id); + + let mut scope_spans = ScopeSpans::default(); + scope_spans.spans.reserve(128); + + Traces { + rng, + context: Vec::with_capacity(DEFAULT_CONTEXT_CAPACITY), + + trace_id, + + traces_data: TracesData::default(), + resource_spans: ResourceSpans::default(), + scope_spans, + } + } + + pub fn get_serialized(mut self, buf: &mut BytesMut) { + let mut resource = Resource::default(); + resource.attributes.push(KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "env".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("dev".to_string())), + }), + }); + + let mut instrumentation_scope = InstrumentationScope::default(); + instrumentation_scope.name = "kong-gateway-efficient-tracing-subsystem".to_string(); + self.scope_spans.scope = Some(instrumentation_scope); + + self.resource_spans.resource = Some(resource); + self.resource_spans.scope_spans.push(self.scope_spans); + self.traces_data.resource_spans.push(self.resource_spans); + + self.traces_data.encode(buf).unwrap(); + } + + pub fn enter_span(&mut self, name: &str) { + let mut span = Span::default(); + span.trace_id = self.trace_id.to_vec(); + span.span_id = self._gen_span_id().to_vec(); + span.name = name.to_string(); + span.start_time_unix_nano = _get_current_time_unix_nano(); + + if let Some(parent_span) = self.context.last() { + span.parent_span_id = parent_span.span_id.clone(); + } + + self.context.push(span); + } + + pub fn exit_span(&mut self) { + debug_assert!(self.context.len() > 0, "No span to exit"); + + let mut span = self.context.pop().unwrap(); + span.end_time_unix_nano = _get_current_time_unix_nano(); + + self.scope_spans.spans.push(span); + } + + pub fn add_string_attribute(&mut self, key: &str, value: &str) { + debug_assert!(self.context.len() > 0, "No span to add attribute"); + + let str_value = any_value::Value::StringValue(value.to_string()); + let any_value = AnyValue { + value: Some(str_value), + }; + + let mut attribute = KeyValue::default(); + attribute.key = key.to_string(); + attribute.value = Some(any_value); + + self.context.last_mut().unwrap().attributes.push(attribute); + } + + pub fn add_bool_attribute(&mut self, key: &str, value: bool) { + debug_assert!(self.context.len() > 0, "No span to add attribute"); + + let bool_value = any_value::Value::BoolValue(value); + let any_value = AnyValue { + value: Some(bool_value), + }; + + let mut attribute = KeyValue::default(); + attribute.key = key.to_string(); + attribute.value = Some(any_value); + + self.context.last_mut().unwrap().attributes.push(attribute); + } + + pub fn add_int64_attribute(&mut self, key: &str, value: i64) { + debug_assert!(self.context.len() > 0, "No span to add attribute"); + + let int_value = any_value::Value::IntValue(value); + let any_value = AnyValue { + value: Some(int_value), + }; + + let mut attribute = KeyValue::default(); + attribute.key = key.to_string(); + attribute.value = Some(any_value); + + self.context.last_mut().unwrap().attributes.push(attribute); + } + + pub fn add_double_attribute(&mut self, key: &str, value: f64) { + debug_assert!(self.context.len() > 0, "No span to add attribute"); + + let double_value = any_value::Value::DoubleValue(value); + let any_value = AnyValue { + value: Some(double_value), + }; + + let mut attribute = KeyValue::default(); + attribute.key = key.to_string(); + attribute.value = Some(any_value); + + self.context.last_mut().unwrap().attributes.push(attribute); + } + + fn _gen_span_id(&mut self) -> [u8; 8] { + let mut span_id = [0; 8]; + self.rng.fill_bytes(&mut span_id); + span_id + } +} + +pub struct Metrics { + metrics_data: MetricsData, + resources_metrics: ResourceMetrics, + scope_metrics: ScopeMetrics, +} + +impl Metrics { + pub fn new() -> Self { + let mut scope_metrics = ScopeMetrics::default(); + scope_metrics.metrics.reserve(16); + + Metrics { + metrics_data: MetricsData::default(), + resources_metrics: ResourceMetrics::default(), + scope_metrics, + } + } + + pub fn get_serialized(mut self, buf: &mut BytesMut) { + let mut resource = Resource::default(); + resource.attributes.push(KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "env".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("dev".to_string())), + }), + }); + + self.resources_metrics.resource = Some(resource); + self.resources_metrics + .scope_metrics + .push(self.scope_metrics); + self.metrics_data + .resource_metrics + .push(self.resources_metrics); + + self.metrics_data.encode(buf).unwrap(); + } + + pub fn add_int64_gauge(&mut self, name: &str, value: i64) { + let mut gauge = Gauge::default(); + let mut data_point = NumberDataPoint::default(); + + data_point.time_unix_nano = _get_current_time_unix_nano(); + data_point.value = Some(Value::AsInt(value)); + + gauge.data_points.push(data_point); + + let mut metric = Metric::default(); + metric.name = name.to_string(); + metric.data = Some(Data::Gauge(gauge)); + + self.scope_metrics.metrics.push(metric); + } + + pub fn add_double_gauge(&mut self, name: &str, value: f64) { + let mut gauge = Gauge::default(); + let mut data_point = NumberDataPoint::default(); + + data_point.time_unix_nano = _get_current_time_unix_nano(); + data_point.value = Some(Value::AsDouble(value)); + + gauge.data_points.push(data_point); + + let mut metric = Metric::default(); + metric.name = name.to_string(); + metric.data = Some(Data::Gauge(gauge)); + + self.scope_metrics.metrics.push(metric); + } + + pub fn add_int64_sum(&mut self, name: &str, value: i64) { + let mut sum = Sum::default(); + let mut data_point = NumberDataPoint::default(); + + data_point.time_unix_nano = _get_current_time_unix_nano(); + data_point.value = Some(Value::AsInt(value)); + + sum.data_points.push(data_point); + + let mut metric = Metric::default(); + metric.name = name.to_string(); + metric.data = Some(Data::Sum(sum)); + + self.scope_metrics.metrics.push(metric); + } + + pub fn add_double_sum(&mut self, name: &str, value: f64) { + let mut sum = Sum::default(); + let mut data_point = NumberDataPoint::default(); + + data_point.time_unix_nano = _get_current_time_unix_nano(); + data_point.value = Some(Value::AsDouble(value)); + + sum.data_points.push(data_point); + + let mut metric = Metric::default(); + metric.name = name.to_string(); + metric.data = Some(Data::Sum(sum)); + + self.scope_metrics.metrics.push(metric); + } +} + +pub struct Logs { + logs_data: LogsData, + resources_logs: ResourceLogs, + scope_logs: ScopeLogs, +} + +impl Logs { + pub fn new() -> Self { + let mut scope_logs = ScopeLogs::default(); + scope_logs.log_records.reserve(64); + + Logs { + logs_data: LogsData::default(), + resources_logs: ResourceLogs::default(), + scope_logs, + } + } + + pub fn get_serialized(mut self, buf: &mut BytesMut) { + let mut resource = Resource::default(); + resource.attributes.push(KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "kong-gateway-qiqi-demo".to_string(), + )), + }), + }); + resource.attributes.push(KeyValue { + key: "env".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("dev".to_string())), + }), + }); + + self.resources_logs.resource = Some(resource); + self.resources_logs.scope_logs.push(self.scope_logs); + self.logs_data.resource_logs.push(self.resources_logs); + + self.logs_data.encode(buf).unwrap(); + } + + pub fn add_info_log(&mut self, time_unix_nano: u64, message: &str) { + let mut log_record = LogRecord::default(); + log_record.time_unix_nano = time_unix_nano; + log_record.observed_time_unix_nano = _get_current_time_unix_nano(); + log_record.severity_number = SeverityNumber::Info as i32; + log_record.body = Some(AnyValue { + value: Some(any_value::Value::StringValue(message.to_string())), + }); + + self.scope_logs.log_records.push(log_record); + } + + pub fn add_notice_log(&mut self, time_unix_nano: u64, message: &str) { + self.add_info_log(time_unix_nano, message); + } + + pub fn add_warning_log(&mut self, time_unix_nano: u64, message: &str) { + let mut log_record = LogRecord::default(); + log_record.time_unix_nano = time_unix_nano; + log_record.observed_time_unix_nano = _get_current_time_unix_nano(); + log_record.severity_number = SeverityNumber::Warn as i32; + log_record.body = Some(AnyValue { + value: Some(any_value::Value::StringValue(message.to_string())), + }); + + self.scope_logs.log_records.push(log_record); + } + + pub fn add_error_log(&mut self, time_unix_nano: u64, message: &str) { + let mut log_record = LogRecord::default(); + log_record.time_unix_nano = time_unix_nano; + log_record.observed_time_unix_nano = _get_current_time_unix_nano(); + log_record.severity_number = SeverityNumber::Error as i32; + log_record.body = Some(AnyValue { + value: Some(any_value::Value::StringValue(message.to_string())), + }); + + self.scope_logs.log_records.push(log_record); + } + + pub fn add_fatal_log(&mut self, time_unix_nano: u64, message: &str) { + let mut log_record = LogRecord::default(); + log_record.time_unix_nano = time_unix_nano; + log_record.observed_time_unix_nano = _get_current_time_unix_nano(); + log_record.severity_number = SeverityNumber::Fatal as i32; + log_record.body = Some(AnyValue { + value: Some(any_value::Value::StringValue(message.to_string())), + }); + + self.scope_logs.log_records.push(log_record); + } +} diff --git a/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt index bee32048e1f7..610104ec70be 100644 --- a/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt +++ b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt @@ -147,6 +147,22 @@ Needed : - libc.so.6 +- Path : /usr/local/openresty/lualib/libatc_router.so + Needed : + - libgcc_s.so.1 + - libm.so.6 + - libc.so.6 + - ld-linux-x86-64.so.2 + - libstdc++.so.6 + +- Path : /usr/local/openresty/lualib/libresty_protobuf.so + Needed : + - libgcc_s.so.1 + - libm.so.6 + - libc.so.6 + - ld-linux-x86-64.so.2 + - libstdc++.so.6 + - Path : /usr/local/openresty/lualib/librestysignal.so - Path : /usr/local/openresty/lualib/rds/parser.so