diff --git a/.github/workflows/main_build.yaml b/.github/workflows/main_build.yaml new file mode 100644 index 000000000..19efe759c --- /dev/null +++ b/.github/workflows/main_build.yaml @@ -0,0 +1,179 @@ +# This workflow build the aws-opentelemetry-distro wheel file, upload to staging S3 bucket, and build project docker image then push to staging ECR +name: Python Instrumentation Main Build +on: + push: + branches: + - main + - contract-tests-workflow + - "release/v*" +env: + AWS_DEFAULT_REGION: us-east-1 + STAGING_ECR_REGISTRY: 637423224110.dkr.ecr.us-east-1.amazonaws.com + STAGING_ECR_REPOSITORY: aws-observability/adot-autoinstrumentation-python-staging + S3_INTEGRATION_BUCKET: ${{ secrets.S3_INTEGRATION_BUCKET }} + +concurrency: + group: python-instrumentation-main-build + cancel-in-progress: false + +permissions: + id-token: write + contents: read + +jobs: + build: + env: + py311: "3.11" + RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # ensures the entire test matrix is run, even if one permutation fails + matrix: + python-version: [ py311 ] + package: [ "aws-opentelemetry-distro" ] + os: [ ubuntu-latest ] + outputs: + python_distro_tag: ${{ steps.python_distro_versioning.outputs.STAGING_TAG}} + #staging-image-name: ${{ steps.imageNameOutput.outputs.imageName }} + staging_wheel_file: ${{ steps.staging_wheel_build.outputs.STAGING_WHEEL}} + steps: + - name: Checkout Contrib Repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env[matrix.python-version] }} + + - name: Install tox + run: pip install tox==3.27.1 tox-factor + + - name: Cache tox environment + # Preserves .tox directory between runs for faster installs + uses: actions/cache@v1 + with: + path: | + .tox + ~/.cache/pip + key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }} + +# - name: run tox +# run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-json=${{ env.RUN_MATRIX_COMBINATION }}-benchmark.json + + - name: Install Dependencies and Build Wheel + id: staging_wheel_build + run: | + pip install --upgrade pip setuptools wheel packaging build + rm -rf ./dist/* + cd ./aws-opentelemetry-distro + python -m build --outdir ../dist + cd ../dist + pkg_version=$(grep '__version__' ../aws-opentelemetry-distro/src/amazon/opentelemetry/distro/version.py | awk -F '"' '{print $2}') + echo "ADOT_PYTHON_VERSION=$pkg_version" >> $GITHUB_OUTPUT + shortsha="$(git rev-parse --short HEAD)" + echo "STAGING_WHEEL=aws_opentelemetry_distro-$pkg_version-$shortsha.whl" >> $GITHUB_OUTPUT + cp aws_opentelemetry_distro-$pkg_version-py3-none-any.whl aws_opentelemetry_distro-$pkg_version-$shortsha.whl + + + - name: Run setup script + run: bash contract-tests/set-up-contract-tests.sh + + - name: Run contract tests + run: | + pip install pytest + pytest contract-tests/tests + + - name: Upload to GitHub Actions + uses: actions/upload-artifact@v3 + with: + name: aws_opentelemetry_distro.whl + path: dist/${{ steps.staging_wheel_build.outputs.STAGING_WHEEL}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Upload wheel to S3 + run: aws s3 cp dist/${{ steps.staging_wheel_build.outputs.STAGING_WHEEL}} ${{ env.S3_INTEGRATION_BUCKET }} + +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 +# +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v3 +# +# - name: Log in to AWS ECR +# uses: docker/login-action@v3 +# with: +# registry: ${{ env.STAGING_ECR_REGISTRY }} +# env: +# AWS_REGION: ${{ env.AWS_DEFAULT_REGION }} +# +# - name: Get Python Distro Image Tag +# id: python_distro_versioning +# run: | +# shortsha="$(git rev-parse --short HEAD)" +# python_distro_tag=${{ steps.staging_wheel_build.outputs.ADOT_PYTHON_VERSION }}-$shortsha +# echo "STAGING_TAG=$python_distro_tag" >> $GITHUB_OUTPUT +# +# - name: Build and push staging image +# uses: docker/build-push-action@v5 +# with: +# push: true +# context: . +# file: ./Dockerfile +# platforms: linux/amd64 +# tags: | +# ${{ env.STAGING_ECR_REGISTRY }}/${{ env.STAGING_ECR_REPOSITORY }}:${{ steps.python_distro_versioning.outputs.STAGING_TAG }} +# +# - name: Set image name to output +# id: imageNameOutput +# run: echo "imageName=${{ env.STAGING_ECR_REGISTRY }}/${{ env.STAGING_ECR_REPOSITORY }}:${{ steps.python_distro_versioning.outputs.STAGING_TAG }}" >> "$GITHUB_OUTPUT" + + contract-tests: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" +# +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} +# aws-region: ${{ env.AWS_DEFAULT_REGION }} +# +# - name: Log in to AWS ECR +# uses: docker/login-action@v3 +# with: +# registry: public.ecr.aws +# +# # cache local patch outputs +# - name: Cache local Maven repository +# id: cache-local-maven-repo +# uses: actions/cache@v3 +# with: +# path: | +# ~/.m2/repository/io/opentelemetry/ +# key: ${{ runner.os }}-maven-local-${{ hashFiles('.github/patches/opentelemetry-java*.patch') }} +# +# - name: Pull base image of Contract Tests Sample Apps +# run: docker pull public.ecr.aws/docker/library/python:3.11-slim + + + - name: Whereami + #working-directory: ${{ github.workspace }}/aws-otel-python-instrumentation + run: pwd; ls -R + + - name: Run setup script + run: bash contract-tests/set-up-contract-tests.sh + + - name: Run contract tests + run: pytest contract-tests/tests diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..b50bfea90 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# To build one auto-instrumentation image for Python, please: +# - Ensure the packages are installed in the `/autoinstrumentation` directory. This is required as when instrumenting the pod, +# one init container will be created to copy all the content in `/autoinstrumentation` directory to your app's container. Then +# update the `PYTHONPATH` environment variable accordingly. To achieve this, you can mimic the one in `autoinstrumentation/python/Dockerfile` +# by using multi-stage builds. In the first stage, install all the required packages in one custom directory with `pip install --target`. +# Then in the second stage, copy the directory to `/autoinstrumentation`. +# - Ensure you have `opentelemetry-distro` and `opentelemetry-instrumentation` or your customized alternatives installed. +# Those two packages are essential to Python auto-instrumentation. +# - Grant the necessary access to `/autoinstrumentation` directory. `chmod -R go+r /autoinstrumentation` +# - For auto-instrumentation by container injection, the Linux command cp is +# used and must be available in the image. +FROM python:3.10 AS build + +WORKDIR /operator-build + +ADD aws-opentelemetry-distro/ ./aws-opentelemetry-distro/ + +RUN mkdir workspace && pip install --target workspace ./aws-opentelemetry-distro + +FROM busybox + +COPY --from=build /operator-build/workspace /autoinstrumentation + +RUN chmod -R go+r /autoinstrumentation \ No newline at end of file diff --git a/aws-opentelemetry-distro/pyproject.toml b/aws-opentelemetry-distro/pyproject.toml index 5cc1227f1..739f79f97 100644 --- a/aws-opentelemetry-distro/pyproject.toml +++ b/aws-opentelemetry-distro/pyproject.toml @@ -11,12 +11,61 @@ license = "Apache-2.0" requires-python = ">=3.8" dependencies = [ + "grpcio ~= 1.60.0", "opentelemetry-api ~= 1.12", - "opentelemetry-instrumentation == 0.43b0", + "opentelemetry-instrumentation ~= 0.43b0", "opentelemetry-sdk ~= 1.13", - "opentelemetry-distro == 0.43b0", + "opentelemetry-distro ~= 0.43b0", "opentelemetry-sdk-extension-aws ~= 2.0.1", - "opentelemetry-exporter-otlp-proto-grpc == 1.22.0" + "opentelemetry-exporter-otlp-proto-grpc ~= 1.22.0", + "opentelemetry-exporter-otlp-proto-http ~= 1.22.0", + "opentelemetry-propagator-b3 ~= 1.22.0", + "opentelemetry-propagator-jaeger ~= 1.22.0", + "opentelemetry-propagator-aws-xray ~= 1.0.1", + "opentelemetry-propagator-ot-trace ~= 0.43b0", + "opentelemetry-exporter-otlp-proto-common ~= 1.22.0", + "opentelemetry-instrumentation-aio-pika ~= 0.43b0", + "opentelemetry-instrumentation-aiohttp-client ~= 0.43b0", + "opentelemetry-instrumentation-aiopg ~= 0.43b0", + "opentelemetry-instrumentation-asgi ~= 0.43b0", + "opentelemetry-instrumentation-asyncpg ~= 0.43b0", + "opentelemetry-instrumentation-boto ~= 0.43b0", + "opentelemetry-instrumentation-boto3sqs ~= 0.43b0", + "opentelemetry-instrumentation-botocore ~= 0.43b0", + "opentelemetry-instrumentation-celery ~= 0.43b0", + "opentelemetry-instrumentation-confluent-kafka ~= 0.43b0", + "opentelemetry-instrumentation-dbapi ~= 0.43b0", + "opentelemetry-instrumentation-django ~= 0.43b0", + "opentelemetry-instrumentation-elasticsearch ~= 0.43b0", + "opentelemetry-instrumentation-falcon ~= 0.43b0", + "opentelemetry-instrumentation-fastapi ~= 0.43b0", + "opentelemetry-instrumentation-flask ~= 0.43b0", + "opentelemetry-instrumentation-grpc ~= 0.43b0", + "opentelemetry-instrumentation-httpx ~= 0.43b0", + "opentelemetry-instrumentation-jinja2 ~= 0.43b0", + "opentelemetry-instrumentation-kafka-python ~= 0.43b0", + "opentelemetry-instrumentation-logging ~= 0.43b0", + "opentelemetry-instrumentation-mysql ~= 0.43b0", + "opentelemetry-instrumentation-mysqlclient ~= 0.43b0", + " opentelemetry-instrumentation-pika ~= 0.43b0", + "opentelemetry-instrumentation-psycopg2 ~= 0.43b0", + "opentelemetry-instrumentation-pymemcache ~= 0.43b0", + "opentelemetry-instrumentation-pymongo ~= 0.43b0", + "opentelemetry-instrumentation-pymysql ~= 0.43b0", + "opentelemetry-instrumentation-pyramid ~= 0.43b0", + "opentelemetry-instrumentation-redis ~= 0.43b0", + "opentelemetry-instrumentation-remoulade ~= 0.43b0", + "opentelemetry-instrumentation-requests ~= 0.43b0", + "opentelemetry-instrumentation-sklearn ~= 0.43b0", + "opentelemetry-instrumentation-sqlalchemy ~= 0.43b0", + "opentelemetry-instrumentation-sqlite3 ~= 0.43b0", + "opentelemetry-instrumentation-starlette ~= 0.43b0", + "opentelemetry-instrumentation-system-metrics ~= 0.43b0", + "opentelemetry-instrumentation-tornado ~= 0.43b0", + "opentelemetry-instrumentation-tortoiseorm ~= 0.43b0", + "opentelemetry-instrumentation-urllib ~= 0.43b0", + "opentelemetry-instrumentation-urllib3 ~= 0.43b0", + "opentelemetry-instrumentation-wsgi ~= 0.43b0" ] [project.optional-dependencies] @@ -41,4 +90,4 @@ include = [ ] [tool.hatch.build.targets.wheel] -packages = ["src/amazon"] +packages = ["src/amazon"] \ No newline at end of file diff --git a/contract-tests/README.md b/contract-tests/README.md index 1725a4f1a..f3f087dae 100644 --- a/contract-tests/README.md +++ b/contract-tests/README.md @@ -32,12 +32,11 @@ The steps to add a new test for a library or framework are: Pre-requirements: * Have `docker` installed and running - verify by running the `docker` command. -* Create `aws-otel-python-instrumentation/contract-tests/dist` folder -* Copy the `aws_opentelemetry_distro` wheel file to `aws-otel-python-instrumentation/contract-tests/dist` folder +* Ensure the `aws_opentelemetry_distro` wheel file exists in to `aws-otel-python-instrumentation/dist` folder -From `aws-otel-python-instrumentation/contract-tests` execute: +From `aws-otel-python-instrumentation` dir, execute: ``` -./set-up-contract-tests.sh -pytest tests +./contract-tests/set-up-contract-tests.sh +pytest contract-tests/tests ``` \ No newline at end of file diff --git a/contract-tests/images/applications/requests/Dockerfile b/contract-tests/images/applications/requests/Dockerfile index 32e949c8b..dcb068b84 100644 --- a/contract-tests/images/applications/requests/Dockerfile +++ b/contract-tests/images/applications/requests/Dockerfile @@ -1,10 +1,10 @@ # Meant to be run from aws-otel-python-instrumentation/contract-tests. # Assumes existence of dist/aws_opentelemetry_distro--py3-none-any.whl. # Assumes filename of aws_opentelemetry_distro--py3-none-any.whl is passed in as "DISTRO" arg. -FROM python:3.9-slim +FROM public.ecr.aws/docker/library/python:3.11-slim WORKDIR /requests -COPY ./dist /requests -COPY ./images/applications/requests /requests +COPY ./dist/$DISTRO /requests +COPY ./contract-tests/images/applications/requests /requests ARG DISTRO RUN pip install -r requirements.txt && pip install ${DISTRO} --force-reinstall diff --git a/contract-tests/images/mock-collector/Dockerfile b/contract-tests/images/mock-collector/Dockerfile index 7c9fa4c0b..6c0c96cc2 100644 --- a/contract-tests/images/mock-collector/Dockerfile +++ b/contract-tests/images/mock-collector/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM public.ecr.aws/docker/library/python:3.11-slim WORKDIR /mock-collector COPY . /mock-collector diff --git a/contract-tests/set-up-contract-tests.sh b/contract-tests/set-up-contract-tests.sh index 375a8e830..f859756ef 100755 --- a/contract-tests/set-up-contract-tests.sh +++ b/contract-tests/set-up-contract-tests.sh @@ -2,8 +2,8 @@ # Check script is running in contract-tests current_path=`pwd` current_dir="${current_path##*/}" -if [ "$current_dir" != "contract-tests" ]; then - echo "Please run from contract-tests dir" +if [ "$current_dir" != "aws-otel-python-instrumentation" ]; then + echo "Please run from aws-otel-python-instrumentation dir" exit fi @@ -12,38 +12,36 @@ rm -rf dist/mock_collector* rm -rf dist/contract_tests* # Create mock-collector image -cd images/mock-collector +cd contract-tests/images/mock-collector docker build . -t aws-appsignals-mock-collector-python -cd ../.. + # Find and store aws_opentelemetry_distro whl file -cd dist -DISTRO=(aws_opentelemetry_distro*.whl) -if [ "$DISTRO" = "aws_opentelemetry_distro*.whl" ]; then +cd ../../../dist +DISTRO=(aws_opentelemetry_distro-*-py3-none-any.whl) +if [ "$DISTRO" = "aws_opentelemetry_distro-*-py3-none-any.whl" ]; then echo "Could not find aws_opentelemetry_distro whl file in dist dir." exit 1 fi -cd .. # Create application images -for dir in images/applications/* +cd .. +for dir in contract-tests/images/applications/* do application="${dir##*/}" docker build . -t aws-appsignals-tests-${application}-app -f ${dir}/Dockerfile --build-arg="DISTRO=${DISTRO}" done # Build and install mock-collector -cd images/mock-collector -python3 -m build --outdir ../../dist --no-isolation -cd ../../dist +cd contract-tests/images/mock-collector +python3 -m build --outdir ../../../dist +cd ../../../dist pip install mock_collector-1.0.0-py3-none-any.whl --force-reinstall # Build and install contract-tests -cd ../tests -python3 -m build --outdir ../dist --no-isolation -cd ../dist +cd ../contract-tests/tests +python3 -m build --outdir ../../dist +cd ../../dist # --force-reinstall causes `ERROR: No matching distribution found for mock-collector==1.0.0`, but uninstalling and reinstalling works pretty reliably. pip uninstall contract-tests -y -pip install contract_tests-1.0.0-py3-none-any.whl - -cd .. \ No newline at end of file +pip install contract_tests-1.0.0-py3-none-any.whl \ No newline at end of file