From 74c77b115081e3dae0cef344bcd7f25017ea55fd Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 10 Oct 2024 10:29:36 -0500 Subject: [PATCH 1/6] pin to extension versions when testing --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cdba09cc..657ddc62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,12 +72,12 @@ dev = [ "invoke" ] test-http-v2 = [ - "azurefunctions-extensions-http-fastapi", + "azurefunctions-extensions-http-fastapi==1.0.0b1", "ujson", "orjson" ] test-deferred-bindings = [ - "azurefunctions-extensions-bindings-blob" + "azurefunctions-extensions-bindings-blob==1.0.0b2" ] [build-system] From 54bc3e0245aecd362ef58d82c6e40344fa845c1e Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 10 Oct 2024 13:46:23 -0500 Subject: [PATCH 2/6] add eg sourced blob functions --- .../blob_functions_stein/function_app.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py index 2621a336..ea872bde 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py @@ -391,3 +391,55 @@ def put_get_multiple_blobs_as_bytes_return_http_response( mimetype="application/json", status_code=200 ) + + +@app.function_name(name="blob_trigger_default_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.LOGS_AND_CONTAINER_SCAN) +def blob_trigger_default_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.EVENT_GRID) +def blob_trigger_eventgrid_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_default_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="LogsAndContainerScan") +def blob_trigger_default_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="EventGrid") +def blob_trigger_eventgrid_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) From 7ba94222782bca087c8fda349ccdbab47a195c8c Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 15 Oct 2024 10:07:24 -0500 Subject: [PATCH 3/6] skipping dependency isolation test when using sdk artifact --- eng/templates/official/jobs/ci-e2e-tests.yml | 20 ++++++++++++++++++- .../test_dependency_isolation_functions.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index b69d9090..bd1e6254 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -115,6 +115,24 @@ jobs: eng/scripts/test-setup.sh displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - powershell: | + $pipelineVarSet = "$(USETESTPYTHONSDK)" + Write-Host "pipelineVarSet: $pipelineVarSet" + $branch = "$(Build.SourceBranch)" + Write-Host "Branch: $branch" + if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true") + { + Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]false" + } + displayName: 'Set SkipDependencyIsolationTests variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - powershell: | + Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" + displayName: 'Display SkipDependencyIsolationTests variable' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: @@ -125,5 +143,5 @@ jobs: AzureWebJobsSqlConnectionString: $(SQL_CONNECTION) AzureWebJobsEventGridTopicUri: $(EVENTGRID_URI) AzureWebJobsEventGridConnectionKey: $(EVENTGRID_CONNECTION) - USETESTPYTHONSDK: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + skipDependencyIsolationTests: $(skipDependencyIsolationTests) displayName: "Running $(PYTHON_VERSION) Python E2E Tests" diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index ec1cc251..42efe551 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('USETESTPYTHONSDK'), + @skipIf(is_envvar_true('skipDependencyIsolationTests'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the From 3608af15484c21b7dc443bfe7a070a36ceb5ecfe Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 15 Oct 2024 10:33:05 -0500 Subject: [PATCH 4/6] reference before assignment --- eng/templates/official/jobs/ci-e2e-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index bd1e6254..481efd3d 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -133,6 +133,7 @@ jobs: - powershell: | Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" displayName: 'Display SkipDependencyIsolationTests variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: From 9c01d26c9a5488a7fe8e9b35e30a7382eca36b06 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 21 Nov 2024 11:44:40 -0600 Subject: [PATCH 5/6] run vulnerability scan --- eng/templates/jobs/build.yml | 6 +++++- pack/templates/macos_64_env_gen.yml | 4 ++++ pack/templates/nix_env_gen.yml | 4 ++++ pack/templates/win_env_gen.yml | 4 ++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml index 3b0500df..dd422f4f 100644 --- a/eng/templates/jobs/build.yml +++ b/eng/templates/jobs/build.yml @@ -21,4 +21,8 @@ jobs: python -m pip install . displayName: 'Build python worker' # Skip the build stage for SDK and Extensions release branches. This stage will fail because pyproject.toml contains the updated (and unreleased) library version - condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) \ No newline at end of file + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' \ No newline at end of file diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 8fb49f73..87fb2d20 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/mac_arm64_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index 7c2b6870..9ef378d8 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/nix_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index 0ae4f70e..e07d8410 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -12,6 +12,10 @@ steps: - task: PowerShell@2 inputs: filePath: 'pack\scripts\win_deps.ps1' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | From e42618dfef0a183cf23a16f859cbeaf3b63668d9 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 3 Dec 2024 16:44:19 -0600 Subject: [PATCH 6/6] feedback --- README.md | 33 +++++++++---------- eng/templates/official/jobs/ci-e2e-tests.yml | 12 +++---- pack/templates/macos_64_env_gen.yml | 1 + pack/templates/nix_env_gen.yml | 1 + pack/templates/win_env_gen.yml | 1 + requirements.txt | 3 +- .../test_dependency_isolation_functions.py | 2 +- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a889aef3..08baeb6d 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,34 @@ # Functions Header Image - Lightning Logo Azure Functions Python Worker -| Branch | Status | CodeCov | Unittests | E2E tests | -|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| -| main | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=main)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=main) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/main/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=main) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=main) | -| dev | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev) | +| Branch | Build Status | CodeCov | Test Status | +|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| dev | [![Build Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | [![Test Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | -Python support for Azure Functions is based on Python 3.6, 3.7, 3.8, 3.9, and 3.10 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. +Python support for Azure Functions is based on Python 3.8, 3.9, 3.10, 3.11, and 3.12 serverless hosting on Linux and the Functions 4.0 runtime. Here is the current status of Python in Azure Functions: What are the supported Python versions? -| Azure Functions Runtime | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | -|----------------------------------|------------|------------|------------|------------|-------------|-------------| -| Azure Functions 2.0 (deprecated) | ✔ | ✔ | - | - | - | - | -| Azure Functions 3.0 (deprecated) | ✔ | ✔ | ✔ | ✔ | - | - | -| Azure Functions 4.0 | - | - | ✔ | ✔ | ✔ | ✔ | +| Azure Functions Runtime | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | Python 3.12 | +|----------------------------------|------------|------------|-------------|-------------|-------------| +| Azure Functions 3.0 (deprecated) | ✔ | ✔ | - | - | - | +| Azure Functions 4.0 | ✔ | ✔ | ✔ | ✔ | ✔ | For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page. ### What's available? -- Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code -- Deploy Python Function project onto consumption, dedicated, or elastic premium plan. -- Deploy Python Function project in a custom docker image onto dedicated, or elastic premium plan. -- Triggers / Bindings : HTTP, Blob, Queue, Timer, Cosmos DB, Event Grid, Event Hubs and Service Bus +- Build, test, debug, and publish using Azure Functions Core Tools (CLI) or Visual Studio Code +- Deploy Python Function project onto consumption, dedicated, elastic premium, or flex consumption plan. +- Deploy Python Function project in a custom docker image onto dedicated or elastic premium plan. +- Triggers / Bindings : Blob, Cosmos DB, Event Grid, Event Hub, HTTP, Kafka, MySQL, Queue, ServiceBus, SQL, Timer, and Warmup - Triggers / Bindings : Custom binding support -What's coming? +### What's new? -- [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python) +- [SDK Type Bindings for Blob](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-sdk-type-bindings-for-azure-blob-storage-with/ba-p/4146744) +- [HTTP Streaming](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-support-for-http-streams-in-python-is-now-in/ba-p/4146697) ### Get Started @@ -72,4 +71,4 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 481efd3d..b3ff4c57 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -122,17 +122,17 @@ jobs: Write-Host "Branch: $branch" if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true") { - Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]true" + Write-Host "##vso[task.setvariable variable=skipTest;]true" } else { - Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]false" + Write-Host "##vso[task.setvariable variable=skipTest;]false" } - displayName: 'Set SkipDependencyIsolationTests variable' + displayName: 'Set skipTest variable' condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - powershell: | - Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" - displayName: 'Display SkipDependencyIsolationTests variable' + Write-Host "skipTest: $(skipTest)" + displayName: 'Display skipTest variable' condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests @@ -144,5 +144,5 @@ jobs: AzureWebJobsSqlConnectionString: $(SQL_CONNECTION) AzureWebJobsEventGridTopicUri: $(EVENTGRID_URI) AzureWebJobsEventGridConnectionKey: $(EVENTGRID_CONNECTION) - skipDependencyIsolationTests: $(skipDependencyIsolationTests) + skipTest: $(skipTest) displayName: "Running $(PYTHON_VERSION) Python E2E Tests" diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 87fb2d20..9bf2027a 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index 9ef378d8..b89d4813 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index e07d8410..8e9b0321 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources\** !*.dist-info\** + !werkzeug\debug\shared\debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/requirements.txt b/requirements.txt index 6b5f87b7..3fdb69c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -# Please list runtime dependencies in setup.py 'install_requires' and -# 'extras_require'. +# Required dependencies listed in pyproject.toml . diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index 42efe551..7ada0ae6 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('skipDependencyIsolationTests'), + @skipIf(is_envvar_true('skipTest'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the