Docker tests updates (#1457)

* Docker tests updates * Fixed consumption custom image if empty * Updated extension csproj to use .net8 * Update extensions to include System.Drawing.Common * Update extensions to include System.Text.Json * Added table extension * table & eh, unit, lint * eh test * lint --------- Co-authored-by: hallvictoria <[email protected]>
Azure · Apr 2, 2024 · 30789ed · 30789ed
1 parent 4c8fb7b
commit 30789ed
Show file tree

Hide file tree

Showing 24 changed files with 113 additions and 173 deletions.
diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml
@@ -5,10 +5,14 @@ name: CI Consumption E2E tests
 
 on:
   workflow_dispatch:
+    inputs:
+      custom_image:
+        description: "Use a custom image to run consumption tests"
+        required: false
   push:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
   pull_request:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
 
 jobs:
   build:
@@ -35,24 +39,29 @@ jobs:
         if: matrix.python-version == 3.7
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv  --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.8 Tests
         if: matrix.python-version == 3.8
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv  --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.9 Tests
         if: matrix.python-version == 3.9
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.10 Tests
         if: matrix.python-version == 3.10
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.11 Tests
         if: matrix.python-version == 3.11
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
diff --git a/.github/workflows/ci_docker_con_workflow.yml b/.github/workflows/ci_docker_con_workflow.yml
@@ -5,9 +5,8 @@ name: CI Docker Consumption tests
 on:
   workflow_dispatch:
   schedule:
-    # Monday to Thursday 2 AM PDT build
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 9 * * *"
+    # Run everyday at 5 AM CST
+    - cron: "0 10 * * *"
 
 jobs:
   build:
@@ -30,7 +29,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
           python setup.py build
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7

diff --git a/.github/workflows/ci_docker_custom_workflow.yml b/.github/workflows/ci_docker_custom_workflow.yml
@@ -49,7 +49,7 @@ jobs:
           python-version: ${{ env.python_version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
+          python setup.py build
       - name: Running 3.7 Tests
         if: env.python_version == 3.7
         env:

diff --git a/.github/workflows/ci_docker_ded_workflow.yml b/.github/workflows/ci_docker_ded_workflow.yml
@@ -5,9 +5,8 @@ name: CI Docker Dedicated tests
 on:
   workflow_dispatch:
   schedule:
-    # Monday to Thursday 1 AM PDT build
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 8 * * *"
+    # Run everyday at 4 AM CST
+    - cron: "0 9 * * *"
 
 jobs:
   build:
@@ -30,7 +29,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
           python setup.py build
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7

diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml
@@ -11,11 +11,11 @@ on:
         required: false
         default: "false"
   push:
-    branches: [dev, master, main, release/*]
+    branches: [dev, main, release/*]
   pull_request:
-    branches: [dev, master, main, release/*]
+    branches: [dev, main, release/*]
   schedule:
-    # Monday to Thursday 1 AM PDT build
+    # Monday to Thursday 3 AM CST build
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * 1,2,3,4"
 
@@ -35,10 +35,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Set up Dotnet 6.x
-        uses: actions/setup-dotnet@v4
-        with:
-          dotnet-version: "6.x"
       - name: Set up Dotnet 8.0.x
         uses: actions/setup-dotnet@v4
         with:

diff --git a/.github/workflows/ci_ut_workflow.yml b/.github/workflows/ci_ut_workflow.yml
@@ -11,12 +11,12 @@ on:
         required: false
         default: "false"
   schedule:
-    # Monday to Thursday 1 AM PDT build
+    # Monday to Thursday 3 AM CST build
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * 1,2,3,4"
   push:
   pull_request:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
 
 jobs:
   build:
@@ -63,6 +63,7 @@ jobs:
         # Retry a couple times to avoid certificate issue
         retry 5 python setup.py build
         retry 5 python setup.py webhost --branch-name=dev
+        retry 5 python setup.py extension
         mkdir logs
     - name: Test with pytest
       env:

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ What are the supported Python versions?
 |----------------------------------|------------|------------|------------|------------|-------------|-------------|
 | Azure Functions 2.0 (deprecated) | ✔          | ✔          | -          | -          | -           | -           |
 | Azure Functions 3.0 (deprecated)             | ✔          | ✔          | ✔          | ✔          | -           | -           |
-| Azure Functions 4.0              | -          | ✔          | ✔          | ✔          | ✔           | coming soon |
+| Azure Functions 4.0              | -          | -          | ✔          | ✔          | ✔           | ✔           |     
 
 For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
 

diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py b/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py
@@ -18,10 +18,10 @@
     connection="AzureWebJobsEventHubConnectionString",
     data_type="string",
     cardinality="many")
-@app.table_output(arg_name="$return",
-                  connection="AzureWebJobsStorage",
-                  table_name="EventHubBatchTest")
-def eventhub_multiple(events):
+@app.blob_output(arg_name="$return",
+                 path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                 connection="AzureWebJobsStorage")
+def eventhub_multiple(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()
@@ -46,13 +46,12 @@ def eventhub_output_batch(req: func.HttpRequest, out: func.Out[str]) -> str:
 
 # Retrieve the event data from storage blob and return it as Http response
 @app.function_name(name="get_eventhub_batch_triggered")
-@app.route(route="get_eventhub_batch_triggered/{id}")
-@app.table_input(arg_name="testEntities",
-                 connection="AzureWebJobsStorage",
-                 table_name="EventHubBatchTest",
-                 partition_key="{id}")
-def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+@app.route(route="get_eventhub_batch_triggered")
+@app.blob_input(arg_name="testEntities",
+                path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                connection="AzureWebJobsStorage")
+def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))
 
 
 # Retrieve the event data from storage blob and return it as Http response

diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py b/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py
@@ -5,7 +5,7 @@
 
 # This is an actual EventHub trigger which handles Eventhub events in batches.
 # It serializes multiple event data into a json and store it into a blob.
-def main(events):
+def main(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()

diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json b/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json
@@ -13,9 +13,9 @@
     },
     {
       "direction": "out",
-      "type": "table",
+      "type": "blob",
       "name": "$return",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     }
   ]

diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py b/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py
@@ -4,5 +4,5 @@
 
 
 # Retrieve the event data from storage blob and return it as Http response
-def main(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+def main(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))
diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json b/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json
@@ -12,10 +12,9 @@
     },
     {
       "direction": "in",
-      "type": "table",
+      "type": "blob",
       "name": "testEntities",
-      "partitionKey": "WillBePopulated",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     },
     {

diff --git a/tests/endtoend/table_functions/table_functions_stein/function_app.py b/tests/endtoend/table_functions/table_functions_stein/function_app.py
@@ -15,8 +15,7 @@
                  row_key='{id}',
                  partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -28,6 +27,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json
diff --git a/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py b/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py
@@ -19,8 +19,7 @@
     row_key="{id}",
     partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -36,6 +35,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json
diff --git a/tests/endtoend/table_functions/table_in_binding/__init__.py b/tests/endtoend/table_functions/table_in_binding/__init__.py
@@ -1,10 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-import json
-
 import azure.functions as func
 
 
 def main(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
diff --git a/tests/endtoend/table_functions/table_out_binding/__init__.py b/tests/endtoend/table_functions/table_out_binding/__init__.py
@@ -10,6 +10,5 @@ def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json