Merge branch 'dev' into jviau/targets

.artifactignore

@@ -0,0 +1,7 @@
+_manifest\**
+bcde-output\**
+Experiment_PipReport_**
+GovCompDisc_Log_**
+GovCompDisc_Manifest_**
+GovCompDisc_Metadata_**
+ScanTelemetry_**

.ci/e2e_integration_test/start-e2e.ps1

@@ -89,8 +89,9 @@ Write-Host "Preparing E2E integration tests..." -ForegroundColor Green
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
 python -m pip install -U pip
 python -m pip install -U -e .[dev]
-python setup.py build
-python setup.py extension
+cd tests
+python -m invoke -c test_setup build-protos
+python -m invoke -c test_setup extensions
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green

.devcontainer/devcontainer.json

@@ -46,7 +46,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost",
+	"postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && cd tests && sudo python -m invoke -c test_setup webhost",
 
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",

.github/linters/tox.ini

@@ -32,7 +32,7 @@ exclude =
         venv*,
         scripts
 
-max-line-length = 80
+max-line-length = 88
 
 per-file-ignores = 
     # import not used

azure_functions_worker/_thirdparty/typing_inspect.py

@@ -11,11 +11,9 @@
 
 # NOTE: This module must support Python 2.7 in addition to Python 3.x
 
-import sys
 import collections.abc
-from typing import (
-    Generic, Callable, Union, TypeVar, ClassVar, Tuple, _GenericAlias
-)
+import sys
+from typing import Callable, ClassVar, Generic, Tuple, TypeVar, Union, _GenericAlias
 
 NEW_39_TYPING = sys.version_info[:3] >= (3, 9, 0)  # PEP 560
 if NEW_39_TYPING:

azure_functions_worker/bindings/__init__.py

@@ -1,18 +1,22 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-from .tracecontext import TraceContext
-from .retrycontext import RetryContext
+from .retrycontext import RetryContext  # isort: skip
+from .tracecontext import TraceContext  # isort: skip
 from .context import Context
-from .meta import check_input_type_annotation
-from .meta import check_output_type_annotation
-from .meta import has_implicit_output
-from .meta import is_trigger_binding, load_binding_registry
-from .meta import from_incoming_proto, to_outgoing_proto, \
-    to_outgoing_param_binding, check_deferred_bindings_enabled, \
-    get_deferred_raw_bindings
+from .meta import (
+    check_deferred_bindings_enabled,
+    check_input_type_annotation,
+    check_output_type_annotation,
+    from_incoming_proto,
+    get_deferred_raw_bindings,
+    has_implicit_output,
+    is_trigger_binding,
+    load_binding_registry,
+    to_outgoing_param_binding,
+    to_outgoing_proto,
+)
 from .out import Out
 
-
 __all__ = (
     'Out', 'Context',
     'is_trigger_binding',

azure_functions_worker/bindings/context.py

@@ -3,8 +3,7 @@
 import threading
 from typing import Type
 
-from . import TraceContext
-from . import RetryContext
+from . import RetryContext, TraceContext
 
 
 class Context:

azure_functions_worker/bindings/datumdef.py

@@ -1,19 +1,26 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-import logging
-from typing import Any, Optional
 import json
+import logging
+from typing import Any, List, Optional
+
 from .. import protos
 from ..logging import logger
-from typing import List
+
 try:
     from http.cookies import SimpleCookie
 except ImportError:
     from Cookie import SimpleCookie
+
 from dateutil import parser
 from dateutil.parser import ParserError
-from .nullable_converters import to_nullable_bool, to_nullable_string, \
-    to_nullable_double, to_nullable_timestamp
+
+from .nullable_converters import (
+    to_nullable_bool,
+    to_nullable_double,
+    to_nullable_string,
+    to_nullable_timestamp,
+)
 
 
 class Datum:

azure_functions_worker/bindings/generic.py

@@ -1,9 +1,9 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 import typing
+from typing import Any, Optional
 
 from . import datumdef
-from typing import Any, Optional
 
 
 class GenericBinding:

azure_functions_worker/bindings/meta.py

@@ -4,17 +4,17 @@
 import sys
 import typing
 
-
 from .. import protos
-from . import datumdef
-from . import generic
-
-from .shared_memory_data_transfer import SharedMemoryManager
+from ..constants import (
+    BASE_EXT_SUPPORTED_PY_MINOR_VERSION,
+    CUSTOMER_PACKAGES_PATH,
+    HTTP,
+    HTTP_TRIGGER,
+)
 from ..http_v2 import HttpV2Registry
-from ..constants import CUSTOMER_PACKAGES_PATH, HTTP, HTTP_TRIGGER, \
-    BASE_EXT_SUPPORTED_PY_MINOR_VERSION
 from ..logging import logger
-
+from . import datumdef, generic
+from .shared_memory_data_transfer import SharedMemoryManager
 
 PB_TYPE = 'rpc_data'
 PB_TYPE_DATA = 'data'
@@ -156,6 +156,7 @@ def from_incoming_proto(
         pytype: typing.Optional[type],
         trigger_metadata: typing.Optional[typing.Dict[str, protos.TypedData]],
         shmem_mgr: SharedMemoryManager,
+        function_name: str,
         is_deferred_binding: typing.Optional[bool] = False) -> typing.Any:
     binding = get_binding(binding, is_deferred_binding)
     if trigger_metadata:
@@ -184,7 +185,8 @@ def from_incoming_proto(
                                             pb=pb,
                                             pytype=pytype,
                                             datum=datum,
-                                            metadata=metadata)
+                                            metadata=metadata,
+                                            function_name=function_name)
         return binding.decode(datum, trigger_metadata=metadata)
     except NotImplementedError:
         # Binding does not support the data.
@@ -281,29 +283,40 @@ def deferred_bindings_decode(binding: typing.Any,
                              pb: protos.ParameterBinding, *,
                              pytype: typing.Optional[type],
                              datum: typing.Any,
-                             metadata: typing.Any):
+                             metadata: typing.Any,
+                             function_name: str):
     """
     This cache holds deferred binding types (ie. BlobClient, ContainerClient)
     That have already been created, so that the worker can reuse the
     Previously created type without creating a new one.
 
+    For async types, the function_name is needed as a key to differentiate.
+    This prevents a known SDK issue where reusing a client across functions
+    can lose the session context and cause an error.
+
+    The cache key is based on: param name, type, resource, function_name
+
     If cache is empty or key doesn't exist, deferred_binding_type is None
     """
     global deferred_bindings_cache
 
     if deferred_bindings_cache.get((pb.name,
                                     pytype,
-                                    datum.value.content), None) is not None:
+                                    datum.value.content,
+                                    function_name), None) is not None:
         return deferred_bindings_cache.get((pb.name,
                                             pytype,
-                                            datum.value.content))
+                                            datum.value.content,
+                                            function_name))
     else:
         deferred_binding_type = binding.decode(datum,
                                                trigger_metadata=metadata,
                                                pytype=pytype)
+
         deferred_bindings_cache[(pb.name,
                                  pytype,
-                                 datum.value.content)] = deferred_binding_type
+                                 datum.value.content,
+                                 function_name)] = deferred_binding_type
         return deferred_binding_type
 
 

azure_functions_worker/bindings/shared_memory_data_transfer/__init__.py

@@ -12,12 +12,12 @@
 https://github.com/Azure/azure-functions-host/issues/6791
 """
 
-from .file_accessor_factory import FileAccessorFactory
 from .file_accessor import FileAccessor
+from .file_accessor_factory import FileAccessorFactory
 from .shared_memory_constants import SharedMemoryConstants
 from .shared_memory_exception import SharedMemoryException
-from .shared_memory_map import SharedMemoryMap
 from .shared_memory_manager import SharedMemoryManager
+from .shared_memory_map import SharedMemoryMap
 
 __all__ = (
     'FileAccessorFactory', 'FileAccessor', 'SharedMemoryConstants',

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor.py

@@ -4,6 +4,7 @@
 import mmap
 from abc import ABCMeta, abstractmethod
 from typing import Optional
+
 from .shared_memory_constants import SharedMemoryConstants as consts
 
 

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py

@@ -4,11 +4,11 @@
 import os
 import sys
 
+from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED
+from ...utils.common import is_envvar_true
 from .file_accessor import DummyFileAccessor
 from .file_accessor_unix import FileAccessorUnix
 from .file_accessor_windows import FileAccessorWindows
-from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED
-from ...utils.common import is_envvar_true
 
 
 class FileAccessorFactory:

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py

@@ -1,16 +1,18 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-from azure_functions_worker import constants
-import os
 import mmap
-from typing import Optional, List
+import os
 from io import BufferedRandom
+from typing import List, Optional
+
+from azure_functions_worker import constants
+
+from ...logging import logger
+from ...utils.common import get_app_setting
+from .file_accessor import FileAccessor
 from .shared_memory_constants import SharedMemoryConstants as consts
 from .shared_memory_exception import SharedMemoryException
-from .file_accessor import FileAccessor
-from ...utils.common import get_app_setting
-from ...logging import logger
 
 
 class FileAccessorUnix(FileAccessor):

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_windows.py

@@ -3,9 +3,10 @@
 
 import mmap
 from typing import Optional
-from .shared_memory_exception import SharedMemoryException
-from .file_accessor import FileAccessor
+
 from ...logging import logger
+from .file_accessor import FileAccessor
+from .shared_memory_exception import SharedMemoryException
 
 
 class FileAccessorWindows(FileAccessor):

azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py

@@ -3,14 +3,15 @@
 
 import uuid
 from typing import Dict, Optional
-from .shared_memory_constants import SharedMemoryConstants as consts
-from .file_accessor_factory import FileAccessorFactory
-from .shared_memory_metadata import SharedMemoryMetadata
-from .shared_memory_map import SharedMemoryMap
-from ..datumdef import Datum
+
+from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED
 from ...logging import logger
 from ...utils.common import is_envvar_true
-from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED
+from ..datumdef import Datum
+from .file_accessor_factory import FileAccessorFactory
+from .shared_memory_constants import SharedMemoryConstants as consts
+from .shared_memory_map import SharedMemoryMap
+from .shared_memory_metadata import SharedMemoryMetadata
 
 
 class SharedMemoryManager:

azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_map.py

@@ -6,10 +6,11 @@
 import struct
 import sys
 from typing import Optional
+
+from ...logging import logger
+from .file_accessor import FileAccessor
 from .shared_memory_constants import SharedMemoryConstants as consts
 from .shared_memory_exception import SharedMemoryException
-from .file_accessor import FileAccessor
-from ...logging import logger
 
 
 class SharedMemoryMap:

azure_functions_worker/constants.py

@@ -11,7 +11,7 @@
 SHARED_MEMORY_DATA_TRANSFER = "SharedMemoryDataTransfer"
 FUNCTION_DATA_CACHE = "FunctionDataCache"
 HTTP_URI = "HttpUri"
-
+REQUIRES_ROUTE_PARAMETERS = "RequiresRouteParameters"
 # When this capability is enabled, logs are not piped back to the
 # host from the worker. Logs will directly go to where the user has
 # configured them to go. This is to ensure that the logs are not
@@ -81,5 +81,15 @@
 # Base extension supported Python minor version
 BASE_EXT_SUPPORTED_PY_MINOR_VERSION = 8
 
+# Appsetting to turn on OpenTelemetry support/features
+# Includes turning on Azure monitor distro to send telemetry to AppInsights
 PYTHON_ENABLE_OPENTELEMETRY = "PYTHON_ENABLE_OPENTELEMETRY"
-PYTHON_ENABLE_OPENTELEMETRY_DEFAULT = True
+PYTHON_ENABLE_OPENTELEMETRY_DEFAULT = False
+
+# Appsetting to specify root logger name of logger to collect telemetry for
+# Used by Azure monitor distro
+PYTHON_AZURE_MONITOR_LOGGER_NAME = "PYTHON_AZURE_MONITOR_LOGGER_NAME"
+PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT = ""
+
+# Appsetting to specify AppInsights connection string
+APPLICATIONINSIGHTS_CONNECTION_STRING = "APPLICATIONINSIGHTS_CONNECTION_STRING"