diff --git a/.ci/perf_tests/dockerfiles/perf_tests.Dockerfile b/.ci/perf_tests/dockerfiles/perf_tests.Dockerfile index a134aadd..6adf9f1d 100644 --- a/.ci/perf_tests/dockerfiles/perf_tests.Dockerfile +++ b/.ci/perf_tests/dockerfiles/perf_tests.Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.8 +ARG PYTHON_VERSION=3.11 FROM mcr.microsoft.com/azure-functions/python:4-python$PYTHON_VERSION @@ -15,7 +15,8 @@ ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED=1 RUN apt-get --quiet update && \ - apt-get install --quiet -y git && \ + apt-get install --quiet -y git procps && \ + # Procps is required for displaying worker and profiling processes info cd /home && \ git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \ mkdir -p /home/site/wwwroot/ && \ diff --git a/.ci/perf_tests/dockerfiles/perf_tests_3.6.Dockerfile b/.ci/perf_tests/dockerfiles/perf_tests_3.6.Dockerfile deleted file mode 100644 index f566f70b..00000000 --- a/.ci/perf_tests/dockerfiles/perf_tests_3.6.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM mcr.microsoft.com/azure-functions/python:3.0.14492-python3.6 - -# Mounting local machines azure-functions-python-worker and azure-functions-python-library onto it -RUN rm -rf /azure-functions-host/workers/python/3.6/LINUX/X64/azure_functions_worker - -# Use the following command to run the docker image with customizible worker and library -VOLUME ["/azure-functions-host/workers/python/3.6/LINUX/X64/azure_functions_worker"] - -ENV FUNCTIONS_WORKER_RUNTIME_VERSION=3.6 \ - AzureWebJobsScriptRoot=/home/site/wwwroot \ - AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ - FUNCTIONS_WORKER_PROCESS_COUNT=1 \ - AZURE_FUNCTIONS_ENVIRONMENT=Development - -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 && \ - echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y git k6 && \ - cd /home && \ - git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \ - mkdir -p /home/site/wwwroot/ && \ - cp -r AzFunctionsPythonPerformance/* /home/site/wwwroot/ && \ - pip install -r /home/site/wwwroot/requirements.txt - -CMD [ "/azure-functions-host/Microsoft.Azure.WebJobs.Script.WebHost" ] diff --git a/.ci/perf_tests/dockerfiles/perf_tests_3.7.Dockerfile b/.ci/perf_tests/dockerfiles/perf_tests_3.7.Dockerfile deleted file mode 100644 index 40190a44..00000000 --- a/.ci/perf_tests/dockerfiles/perf_tests_3.7.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM mcr.microsoft.com/azure-functions/python:3.0.14492-python3.7 - -# Mounting local machines azure-functions-python-worker and azure-functions-python-library onto it -RUN rm -rf /azure-functions-host/workers/python/3.7/LINUX/X64/azure_functions_worker - -# Use the following command to run the docker image with customizible worker and library -VOLUME ["/azure-functions-host/workers/python/3.7/LINUX/X64/azure_functions_worker"] - -ENV FUNCTIONS_WORKER_RUNTIME_VERSION=3.7 \ - AzureWebJobsScriptRoot=/home/site/wwwroot \ - AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ - FUNCTIONS_WORKER_PROCESS_COUNT=1 \ - AZURE_FUNCTIONS_ENVIRONMENT=Development - -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 && \ - echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y git k6 && \ - cd /home && \ - git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \ - mkdir -p /home/site/wwwroot/ && \ - cp -r AzFunctionsPythonPerformance/* /home/site/wwwroot/ && \ - pip install -r /home/site/wwwroot/requirements.txt - -CMD [ "/azure-functions-host/Microsoft.Azure.WebJobs.Script.WebHost" ] diff --git a/.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile b/.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile index b8c967ae..39596e15 100644 --- a/.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile +++ b/.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile @@ -194,7 +194,8 @@ VOLUME ["/azure-functions-host/workers/python/3.8/LINUX/X64/azure_functions_work RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 && \ echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list && \ apt-get update && \ - apt-get install -y git k6 && \ + apt-get install -y git k6 procps && \ + # Procps is required for displaying worker and profiling processes info cd /home && \ git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \ mkdir -p /home/site/wwwroot/ && \ diff --git a/.ci/perf_tests/k6scripts/SyncHttpTriggerHelloWorld.js b/.ci/perf_tests/k6scripts/SyncHelloWorld.js similarity index 98% rename from .ci/perf_tests/k6scripts/SyncHttpTriggerHelloWorld.js rename to .ci/perf_tests/k6scripts/SyncHelloWorld.js index 50694761..acb28f56 100644 --- a/.ci/perf_tests/k6scripts/SyncHttpTriggerHelloWorld.js +++ b/.ci/perf_tests/k6scripts/SyncHelloWorld.js @@ -35,7 +35,7 @@ export let options = { // Main function export default function () { - let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHttpTriggerHelloWorld`); + let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHelloWorld`); // check() returns false if any of the specified conditions fail let checkRes = check(response, { @@ -45,4 +45,4 @@ export default function () { // We reverse the check() result since we want to count the failures failureRate.add(!checkRes); -} \ No newline at end of file +} diff --git a/.github/workflows/perf-testing-setup.yml b/.github/workflows/perf-testing-setup.yml index c78e2f17..19c7453f 100644 --- a/.github/workflows/perf-testing-setup.yml +++ b/.github/workflows/perf-testing-setup.yml @@ -2,35 +2,37 @@ name: Throughput testing workflow on: workflow_dispatch: - pull_request: - branches: [ dev ] - + inputs: + profiling_sampling_rate: + description: 'Profiling sampling rate (tps)' + required: false + default: '500' + test_to_run: + description: 'List of perf tests to run' + required: false + default: SyncHelloWorld + issue_comment: + types: [created] env: TESTS_DIR_PATH: ".ci/perf_tests/k6scripts/" - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.10" + PYTHON_FUNCTION_PROFILING_STORAGE_ACCT: "azpyfuncpipelinestorage" PORT: 8000 jobs: build: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.issue.pull_request && contains(github.event.comment.body, '/profile') }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - test_to_run: [ SyncHttpTriggerHelloWorld ] + test_to_run: ['${{ github.event.inputs.test_to_run }}'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Set up Dotnet 2.x - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '3.1.405' - - name: Set up Dotnet 6.x - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '6.x' - name: Setup k6 for throughput testing run: | cd $GITHUB_WORKSPACE @@ -47,13 +49,76 @@ jobs: python setup.py extension - name: Build and Run the Docker image run: | + echo "Building Docker image with Python version ${{ env.PYTHON_VERSION }}" docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --file .ci/perf_tests/dockerfiles/perf_tests.Dockerfile --tag perfimage:latest . - docker run -d --shm-size="2g" --env FUNCTIONS_WORKER_RUNTIME_VERSION=${{ env.PYTHON_VERSION }} -p ${PORT}:80 -v $GITHUB_WORKSPACE/azure_functions_worker:/azure-functions-host/workers/python/${{ env.PYTHON_VERSION }}/LINUX/X64/azure_functions_worker perfimage:latest + + echo "Running Docker container..." + container_id=$(docker run -d --privileged --env FUNCTIONS_WORKER_RUNTIME_VERSION=${{ env.PYTHON_VERSION }} -p ${PORT}:80 -v $GITHUB_WORKSPACE/azure_functions_worker:/azure-functions-host/workers/python/${{ env.PYTHON_VERSION }}/LINUX/X64/azure_functions_worker perfimage:latest) sleep 10 # host needs some time to start. + echo "Container ID is $container_id" + + echo "Fetching Docker container logs..." + docker logs $container_id + worker_pid=$(docker exec $container_id sh -c "ps aux | grep '[p]ython'" | awk '{print $2}') + echo "Python worker process id is $worker_pid" + + echo "container_id=$container_id" >> $GITHUB_ENV + echo "worker_pid=$worker_pid" >> $GITHUB_ENV + - name: Validate if the functions are now running run: | curl --get http://localhost:${PORT}/api/${{ matrix.test_to_run }} + + - name: Start py-spy in the background + run: | + docker exec $container_id sh -c "pip install py-spy" + docker exec $container_id sh -c "mkdir /home/profiling_reports" + profiling_sampling_rate=${{ github.event.inputs.profiling_sampling_rate }} + # report_name="${GITHUB_REF#refs/heads/}_${{ github.run_id }}.svg" + report_name="${{ github.run_id }}.svg" + docker exec -d $container_id sh -c "RUST_BACKTRACE=1 py-spy record -p $worker_pid -o /home/profiling_reports/$report_name -f flamegraph --idle --nonblocking --rate $profiling_sampling_rate > /home/site/wwwroot/py-spy.log 2>&1 &" + sleep 2 # Give it a moment to start + py_spy_id=$(docker exec $container_id sh -c "ps aux | grep '[p]y-spy record'" | awk '{print $2}') + + echo "py_spy_id=$py_spy_id" >> $GITHUB_ENV + echo "report_name=$report_name" >> $GITHUB_ENV + - name: Run Throughput tests run: | chmod 755 .ci/perf_tests/run-perftests.sh .ci/perf_tests/run-perftests.sh localhost $PORT ${{ env.TESTS_DIR_PATH }} ${{ matrix.test_to_run }} + + - name: Stop profiling and generate report + run: | + echo "Tests completed, terminating py-spy..." + docker exec $container_id cat /home/site/wwwroot/py-spy.log + docker exec $container_id sh -c "kill -2 $py_spy_id" + sleep 2 + + mkdir profiling_reports + chmod 777 profiling_reports + docker cp $container_id:/home/profiling_reports/$report_name profiling_reports + + - name: Upload SVG to Azure Blob Storage + uses: bacongobbler/azure-blob-storage-upload@v3.0.0 + with: + source_dir: 'profiling_reports' # Directory containing the $report_name file + container_name: 'profiling' + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + sync: 'false' + + - name: Output Blob URL + run: | + blob_url="https://${{ env.PYTHON_FUNCTION_PROFILING_STORAGE_ACCT }}.blob.core.windows.net/profiling/${{ env.report_name }}" + echo "You can view the Blob at: $blob_url" + + - name: Upload profiling result to artifact + uses: actions/upload-artifact@v2 + with: + name: py-spy-output + path: 'profiling_reports/${{ env.report_name }}' + + - name: Create Artifact Link + run: | + echo "You can download the SVG artifact from the Actions run page." + echo "Link to the Actions run page: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"