[WIP] Add microbenchmarks and improve efficiency of core functions

.devcontainer/Dockerfile

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.9"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN sudo git clone --depth=1 https://github.com/P403n1x87/austin.git && cd austin \
+    sudo autoreconf --install && sudo ./configure && sudo make && sudo make install
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,58 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.9",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"python.defaultInterpreterPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+		"python.testing.pytestArgs": [
+			"tests/unittests"
+		],
+		"python.testing.unittestEnabled": false,
+		"python.testing.pytestEnabled": true
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+
+	"features": {
+		"dotnet": "latest"
+	},
+	"runArgs": ["--cap-add", "SYS_PTRACE"]
+}

.gitignore

@@ -131,3 +131,4 @@ prof/
 tests/**/host.json
 tests/**/bin
 tests/**/extensions.csproj
+.benchmarks

azure_functions_worker/functions.py

@@ -7,7 +7,9 @@
 
 from azure.functions import DataType, Function
 
-from . import bindings as bindings_utils
+from .bindings import (has_implicit_output,
+                       check_output_type_annotation,
+                       check_input_type_annotation)
 from . import protos
 from ._thirdparty import typing_inspect
 from .protos import BindingInfo
@@ -60,7 +62,7 @@ def get_explicit_and_implicit_return(binding_name: str,
             typing.Tuple[bool, bool]:
         if binding_name == '$return':
             explicit_return = True
-        elif bindings_utils.has_implicit_output(
+        elif has_implicit_output(
                 binding.type):
             implicit_return = True
             bound_params[binding_name] = binding
@@ -75,7 +77,7 @@ def get_return_binding(binding_name: str,
         if binding_name == "$return":
             return_binding_name = binding_type
             assert return_binding_name is not None
-        elif bindings_utils.has_implicit_output(binding_type):
+        elif has_implicit_output(binding_type):
             return_binding_name = binding_type
 
         return return_binding_name
@@ -202,17 +204,17 @@ def validate_function_params(params: dict, bound_params: dict,
                     'is azure.functions.Out in Python')
 
             if param_has_anno and param_py_type in (str, bytes) and (
-                    not bindings_utils.has_implicit_output(binding.type)):
+                    not has_implicit_output(binding.type)):
                 param_bind_type = 'generic'
             else:
                 param_bind_type = binding.type
 
             if param_has_anno:
                 if is_param_out:
-                    checks_out = bindings_utils.check_output_type_annotation(
+                    checks_out = check_output_type_annotation(
                         param_bind_type, param_py_type)
                 else:
-                    checks_out = bindings_utils.check_input_type_annotation(
+                    checks_out = check_input_type_annotation(
                         param_bind_type, param_py_type)
 
                 if not checks_out:
@@ -263,7 +265,7 @@ def get_function_return_type(annotations: dict, has_explicit_return: bool,
             if return_pytype is (str, bytes):
                 binding_name = 'generic'
 
-            if not bindings_utils.check_output_type_annotation(
+            if not check_output_type_annotation(
                     binding_name, return_pytype):
                 raise FunctionLoadError(
                     func_name,

azure_functions_worker/loader.py

@@ -8,13 +8,14 @@
 import os.path
 import pathlib
 import sys
-import uuid
+from uuid import uuid4
 from os import PathLike, fspath
 from typing import List, Optional, Dict
 
 from azure.functions import Function, FunctionApp
 
-from . import protos, functions
+from . import functions
+from .protos import RpcFunctionMetadata, BindingInfo
 from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \
     PYTHON_LANGUAGE_RUNTIME
 from .utils.wrappers import attach_message_to_exception
@@ -48,28 +49,27 @@ def uninstall() -> None:
 
 
 def build_binding_protos(indexed_function: List[Function]) -> Dict:
-    binding_protos = {}
-    for binding in indexed_function.get_bindings():
-        binding_protos[binding.name] = protos.BindingInfo(
+    return {
+        binding.name: BindingInfo(
             type=binding.type,
             data_type=binding.data_type,
             direction=binding.direction)
-
-    return binding_protos
+        for binding in indexed_function.get_bindings()
+    }
 
 
 def process_indexed_function(functions_registry: functions.Registry,
                              indexed_functions: List[Function]):
     fx_metadata_results = []
     for indexed_function in indexed_functions:
-        function_id = str(uuid.uuid4())
+        function_id = str(uuid4())
         function_info = functions_registry.add_indexed_function(
             function_id,
             function=indexed_function)
 
         binding_protos = build_binding_protos(indexed_function)
 
-        function_metadata = protos.RpcFunctionMetadata(
+        function_metadata = RpcFunctionMetadata(
             name=function_info.name,
             function_id=function_id,
             managed_dependency_enabled=False,  # only enabled for PowerShell

setup.py

@@ -127,6 +127,9 @@
         "pytest-randomly",
         "pytest-instafail",
         "pytest-rerunfailures",
+        "pytest-benchmark",
+        "pytest-asyncio",
+        "pyinstrument",
         "ptvsd"
     ]
 }

tests/.gitignore

@@ -1,3 +1,4 @@
 *_functions/bin/
 *_functions/host.json
 *_functions/ping/
+benchmarks/.profiles

tests/benchmarks/README.md

@@ -0,0 +1,26 @@
+This folder contains benchmarks written using `pytest` and profiled using `pyinstrument`.
+
+# Running the benchmarks
+
+Open this repository in VSCode and wait for the devcontainer to start up.
+
+Then you can run all the benchmarks with this command:
+
+`python -m pytest tests/benchmarks`
+
+Or use the keyword argument to run just one benchmark test:
+
+`python -m pytest tests/benchmarks -k test_process_indexed_function`
+
+When you run that test, a profile will also be saved in the `tests/benchmarks/.profiles` folder, in a file named after the profiled function. Open the file in a browser to see the profile.
+
+If you run a benchmark test multiple times (either on same code or different versions of the code), you probably want to save it.
+
+Either pass in `--benchmark-autosave` to save to an auto-generated filename or pass in `--benchmark-save=YOURNAME` to save with your specified name in the filename. All benchmark files will always start with a counter, beginning with 0001.
+
+Once saved, compare using the `pytest-benchmark` command and the counter numbers:
+
+`pytest-benchmark compare 0004 0005`
+
+You can sort the comparison using `--sort`, save it to a CSV using `--csv`, or save it to a histogram with `--histogram`.
+More details available in the [pytest-benchmark reference](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#comparison-cli).

tests/benchmarks/conftest.py

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import asyncio
+import pytest_asyncio
+
+
+@pytest_asyncio.fixture
+async def aio_benchmark(benchmark, event_loop):
+    def _wrapper(func, *args, **kwargs):
+        if asyncio.iscoroutinefunction(func):
+            @benchmark
+            def _():
+                return event_loop.run_until_complete(func(*args, **kwargs))
+        else:
+            benchmark(func, *args, **kwargs)
+
+    return _wrapper

tests/benchmarks/dummy/__init__.py

@@ -0,0 +1,26 @@
+import azure.functions as func
+
+def foo():
+    pass
+
+app = func.FunctionApp()
+
+@app.route(route="func1")
+def func1(req: func.HttpRequest) -> func.HttpResponse:
+    ...
+
+
+@app.route(route="func1")
+def func2(req: func.HttpRequest, arg1) -> func.HttpResponse:
+    ...
+
+
+@app.route(route="func1")
+def func3(req: func.HttpRequest, arg1, arg2) -> func.HttpResponse:
+    ...
+
+
+@app.route(route="func1")
+def func4(req: func.HttpRequest, arg1, arg2, arg3) -> func.HttpResponse:
+    ...
+

tests/benchmarks/test_dispatcher_benchmark.py

@@ -0,0 +1,103 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import asyncio
+import pathlib
+import typing
+import sys
+
+from azure_functions_worker import protos, testutils, dispatcher
+
+
+class _MockWebHostWithWorkerController:
+
+    def __init__(self, scripts_dir: pathlib.PurePath, event_loop):
+        self._event_loop = event_loop
+        self._host: typing.Optional[testutils._MockWebHost] = None
+        self._scripts_dir: pathlib.PurePath = scripts_dir
+        self._worker: typing.Optional[dispatcher.Dispatcher] = None
+
+    async def __aenter__(self) -> typing.Tuple[testutils._MockWebHost, dispatcher.Dispatcher]:
+        loop = self._event_loop
+        self._host = testutils._MockWebHost(loop, self._scripts_dir)
+
+        await self._host.start()
+
+        self._worker = await dispatcher.\
+            Dispatcher.connect(testutils.LOCALHOST, self._host._port,
+                               self._host.worker_id, self._host.request_id,
+                               connect_timeout=5.0)
+
+        self._worker_task = loop.create_task(self._worker.dispatch_forever())
+
+        done, pending = await asyncio. \
+            wait([self._host._connected_fut, self._worker_task],
+                 return_when=asyncio.FIRST_COMPLETED)
+
+        # noinspection PyBroadException
+        try:
+            if self._worker_task in done:
+                self._worker_task.result()
+
+            if self._host._connected_fut not in done:
+                raise RuntimeError('could not start a worker thread')
+        except Exception:
+            try:
+                await self._host.close()
+                self._worker.stop()
+            finally:
+                raise
+
+        return self._host, self._worker
+
+    async def __aexit__(self, *exc):
+        if not self._worker_task.done():
+            self._worker_task.cancel()
+            try:
+                await self._worker_task
+            except asyncio.CancelledError:
+                pass
+
+        self._worker_task = None
+        self._worker = None
+
+        await self._host.close()
+        self._host = None
+
+def start_mockhost_with_worker(event_loop, script_root=testutils.FUNCS_PATH):
+    scripts_dir = testutils.TESTS_ROOT / script_root
+    if not (scripts_dir.exists() and scripts_dir.is_dir()):
+        raise RuntimeError(
+            f'invalid script_root argument: '
+            f'{scripts_dir} directory does not exist')
+
+    sys.path.append(str(scripts_dir))
+
+    return _MockWebHostWithWorkerController(scripts_dir, event_loop)
+
+def test_invoke_function_benchmark(aio_benchmark, event_loop):
+    async def invoke_function():
+        wc = start_mockhost_with_worker(event_loop)
+        async with wc as (host, worker):
+            await host.load_function('return_http')
+
+            func = host._available_functions['return_http']
+            invocation_id = host.make_id()
+            input_data = [protos.ParameterBinding(
+                            name='req',
+                            data=protos.TypedData(
+                                http=protos.RpcHttp(
+                                    method='GET')))]
+            message = protos.StreamingMessage(
+                        invocation_request=protos.InvocationRequest(
+                            invocation_id=invocation_id,
+                            function_id=func.id,
+                            input_data=input_data,
+                            trigger_metadata={},
+                        )
+                    )
+            for _ in range(1000):
+                event_loop.create_task(worker._handle__invocation_request(message))
+
+    aio_benchmark(invoke_function)
+