feat(azure-functions): support Azure Functions programming model v4

See https://learn.microsoft.com/en-ca/azure/azure-functions/functions-node-upgrade-v4
This builds on the work in #4178
by @qzxlkj (type that 5 times fast), which provide most of the runtime code fix.

The rest of this is adding testing and updated examples and docs.

Refs: #4178
Closes: #3185

.github/dependabot.yml

@@ -48,7 +48,15 @@ updates:
         - "eslint*"
 
   - package-ecosystem: "npm"
-    directory: "/test/instrumentation/azure-functions/fixtures/AJsAzureFnApp"
+    directory: "/test/instrumentation/azure-functions/fixtures/azfunc3"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    reviewers:
+      - "elastic/apm-agent-node-js"
+
+  - package-ecosystem: "npm"
+    directory: "/test/instrumentation/azure-functions/fixtures/azfunc4"
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 5

lib/instrumentation/azure-functions.js

@@ -355,6 +355,8 @@ function instrument(agent) {
     }
 
     const context = hookCtx.invocationContext;
+    // console.log('XXX context:');
+    // console.dir(context, { depth: 50 }); // XXX
     const invocationId = context.invocationId;
     log.trace({ invocationId }, 'azure-functions: preInvocation');
 
@@ -363,30 +365,66 @@ function instrument(agent) {
       isFirstRun = false;
     }
 
+    // XXX https://github.com/elastic/apm-agent-nodejs/pull/4178/files for azfunc v4
+    let bindingDefinitions = context.bindingDefinitions;
+    if (!bindingDefinitions) {
+      // cater for possibly v4 version of the function app
+      bindingDefinitions = [];
+      // push input bindings
+      bindingDefinitions.push({
+        name: context?.options?.trigger?.name,
+        type: context?.options?.trigger?.type,
+        direction: context?.options?.trigger?.direction,
+      });
+      // push the output bindings
+      bindingDefinitions.push(context?.options?.return);
+    }
+    let executionContext = context.executionContext;
+    if (!executionContext) {
+      executionContext = {
+        functionDirectory: '',
+        functionName: context.functionName,
+      };
+    }
+
     const funcInfo = (hookCtx.hookData.funcInfo = new FunctionInfo(
-      context.bindingDefinitions,
-      context.executionContext,
+      bindingDefinitions,
+      executionContext,
       log,
     ));
     const triggerType = funcInfo.triggerType;
 
     // Handle trace-context.
-    // Note: We ignore the `context.traceContext`. By default it is W3C
-    // trace-context that continues the given traceparent in headers. However,
-    // we do not injest that span, so would get a broken distributed trace if
-    // we included it.
+    // XXX bring back distributed tracing language.
+    // XXX also with v4 we don't ahve access to the request headers (context.req.headers),
+    //     so we cannot optionally use a traceparent header there. That means
+    //     distributed traces going through HTTP-triggered Azure Functions
+    //     won't be connected in our waterfall because of the broken trace limitation.
+    // XXX Possible concerns:
+    // - Was `traceContext` appropriate/accurate with Azure Functions v3?
+    //   I.e. for TRIGGER_HTTP, did it reflect an incoming traceparent header?
+    // - When incoming requests do *not* provide trace context, Azure Functions
+    //   provides a traceparent. At least when running locally it always has
+    //   the sampling flag set *false*! If this happens in a deployed function
+    //   then that looks like no tracing at all. Do we want to call out
+    //   `traceContinuationStrategy`?
     let traceparent;
     let tracestate;
     if (triggerType === TRIGGER_HTTP && context.req && context.req.headers) {
       traceparent =
         context.req.headers.traceparent ||
-        context.req.headers['elastic-apm-traceparent'];
-      tracestate = context.req.headers.tracestate;
+        context.req.headers['elastic-apm-traceparent'] ||
+        context.traceContext.traceParent;
+      tracestate =
+        context.req.headers.tracestate || context.traceContext.traceState;
+    } else if (context.traceContext) {
+      traceparent = context.traceContext.traceParent;
+      tracestate = context.traceContext.traceState;
     }
 
     const trans = (hookCtx.hookData.trans = ins.startTransaction(
       // This is the default name. Trigger-specific values are added below.
-      context.executionContext.functionName,
+      executionContext.functionName,
       TRANS_TYPE_FROM_TRIGGER_TYPE[triggerType],
       {
         childOf: traceparent,
@@ -399,7 +437,7 @@ function instrument(agent) {
     const accountId = getAzureAccountId();
     const resourceGroup = process.env.WEBSITE_RESOURCE_GROUP;
     const fnAppName = process.env.WEBSITE_SITE_NAME;
-    const fnName = context.executionContext.functionName;
+    const fnName = executionContext.functionName;
     const faasData = {
       trigger: {
         type: FAAS_TRIGGER_TYPE_FROM_TRIGGER_TYPE[triggerType],

lib/instrumentation/transaction.js

@@ -91,6 +91,7 @@ function Transaction(agent, name, ...args) {
     sampled: this.sampled,
     name: this.name,
     type: this.type,
+    traceparent: this.traceparent,
   });
 
   this._defaultName = name || '';

test/instrumentation/azure-functions/azure-functions.test.js → test/instrumentation/azure-functions/azure-functions-v3.test.js

@@ -6,6 +6,8 @@
 
 'use strict';
 
+// Test Azure Functions programming model v3.
+
 const assert = require('assert');
 const { exec, spawn } = require('child_process');
 const fs = require('fs');
@@ -20,12 +22,14 @@ const treekill = require('tree-kill');
 const { MockAPMServer } = require('../../_mock_apm_server');
 const { formatForTComment } = require('../../_utils');
 
-if (!semver.satisfies(process.version, '>=14.1.0 <19')) {
-  // The "14.1.0" version is selected to skip testing on Node.js v14.0.0
-  // because of the issue described here:
-  // https://github.com/elastic/apm-agent-nodejs/issues/3279#issuecomment-1532084620
+// Azure Functions programming model v3 supports node 14.x-20.x:
+// https://learn.microsoft.com/en-ca/azure/azure-functions/functions-reference-node?tabs=javascript%2Cwindows%2Cazure-cli&pivots=nodejs-model-v4#supported-versions
+// However, let's only test with node 18.x for now. The testing involves
+// installing the ridiculously large "azure-functions-core-tools" dep, so it
+// isn't worth testing all versions.
+if (!semver.satisfies(process.version, '^18.0.1')) {
   console.log(
-    `# SKIP Azure Functions runtime ~4 does not support node ${process.version} (https://aka.ms/functions-node-versions)`,
+    '# SKIP Azure Functions v3 tests, only testing with Node.js v18.latest',
   );
   process.exit();
 } else if (os.platform() === 'win32') {
@@ -168,7 +172,7 @@ function checkExpectedApmEvents(t, apmEvents) {
   if (apmEvents.length > 0) {
     const metadata = apmEvents.shift().metadata;
     t.ok(metadata, 'metadata is first event');
-    t.equal(metadata.service.name, 'AJsAzureFnApp', 'metadata.service.name');
+    t.equal(metadata.service.name, 'azfunc3', 'metadata.service.name');
     t.equal(
       metadata.service.framework.name,
       'Azure Functions',
@@ -196,7 +200,7 @@ function checkExpectedApmEvents(t, apmEvents) {
     );
     t.equal(
       metadata.cloud.instance.name,
-      'AJsAzureFnApp',
+      'azfunc3',
       'metadata.cloud.instance.name',
     );
     t.equal(
@@ -256,7 +260,7 @@ function checkExpectedApmEvents(t, apmEvents) {
 const UUID_RE =
   /^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i;
 
-const fnAppDir = path.join(__dirname, 'fixtures', 'AJsAzureFnApp');
+const fnAppDir = path.join(__dirname, 'fixtures', 'azfunc3');
 const funcExe =
   path.resolve(fnAppDir, 'node_modules/.bin/func') +
   (os.platform() === 'win32' ? '.cmd' : '');
@@ -277,14 +281,10 @@ var TEST_REQUESTS = [
       t.equal(trans.type, 'request', 'transaction.type');
       t.equal(trans.outcome, 'success', 'transaction.outcome');
       t.equal(trans.result, 'HTTP 2xx', 'transaction.result');
-      t.equal(
-        trans.faas.name,
-        'AJsAzureFnApp/HttpFn1',
-        'transaction.faas.name',
-      );
+      t.equal(trans.faas.name, 'azfunc3/HttpFn1', 'transaction.faas.name');
       t.equal(
         trans.faas.id,
-        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/AJsAzureFnApp/functions/HttpFn1',
+        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/azfunc3/functions/HttpFn1',
         'transaction.faas.id',
       );
       t.equal(trans.faas.trigger.type, 'http', 'transaction.faas.trigger.type');
@@ -332,11 +332,7 @@ var TEST_REQUESTS = [
       t.equal(trans.name, 'GET /api/HttpFnError', 'transaction.name');
       t.equal(trans.outcome, 'failure', 'transaction.outcome');
       t.equal(trans.result, 'HTTP 5xx', 'transaction.result');
-      t.equal(
-        trans.faas.name,
-        'AJsAzureFnApp/HttpFnError',
-        'transaction.faas.name',
-      );
+      t.equal(trans.faas.name, 'azfunc3/HttpFnError', 'transaction.faas.name');
       t.equal(trans.faas.coldstart, false, 'transaction.faas.coldstart');
       t.equal(
         trans.context.request.method,
@@ -538,14 +534,10 @@ var TEST_REQUESTS = [
       const trans = apmEventsForReq[0].transaction;
       t.equal(trans.name, 'GET /api/HttpFn1', 'transaction.name');
       t.equal(trans.result, 'HTTP 2xx', 'transaction.result');
-      t.equal(
-        trans.faas.name,
-        'AJsAzureFnApp/HttpFn1',
-        'transaction.faas.name',
-      );
+      t.equal(trans.faas.name, 'azfunc3/HttpFn1', 'transaction.faas.name');
       t.equal(
         trans.faas.id,
-        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/AJsAzureFnApp/functions/HttpFn1',
+        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/azfunc3/functions/HttpFn1',
         'transaction.faas.id',
       );
       t.equal(
@@ -574,12 +566,12 @@ var TEST_REQUESTS = [
       t.equal(trans.result, 'HTTP 2xx', 'transaction.result');
       t.equal(
         trans.faas.name,
-        'AJsAzureFnApp/HttpFnRouteTemplate',
+        'azfunc3/HttpFnRouteTemplate',
         'transaction.faas.name',
       );
       t.equal(
         trans.faas.id,
-        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/AJsAzureFnApp/functions/HttpFnRouteTemplate',
+        '/subscriptions/2491fc8e-f7c1-4020-b9c6-78509919fd16/resourceGroups/my-resource-group/providers/Microsoft.Web/sites/azfunc3/functions/HttpFnRouteTemplate',
         'transaction.faas.id',
       );
       t.equal(
@@ -605,7 +597,7 @@ var TEST_REQUESTS = [
       t.equal(apmEventsForReq.length, 4);
       const t1 = apmEventsForReq[0].transaction;
       t.equal(t1.name, 'GET /api/HttpFnDistTraceA', 't1.name');
-      t.equal(t1.faas.name, 'AJsAzureFnApp/HttpFnDistTraceA', 't1.faas.name');
+      t.equal(t1.faas.name, 'azfunc3/HttpFnDistTraceA', 't1.faas.name');
       const s1 = apmEventsForReq[1].span;
       t.equal(s1.name, 'spanA', 's1.name');
       t.equal(s1.parent_id, t1.id, 's1 is a child of t1');
@@ -615,7 +607,7 @@ var TEST_REQUESTS = [
       t.equal(s2.parent_id, s1.id, 's2 is a child of s1');
       const t2 = apmEventsForReq[3].transaction;
       t.equal(t2.name, 'GET /api/HttpFnDistTraceB', 't2.name');
-      t.equal(t2.faas.name, 'AJsAzureFnApp/HttpFnDistTraceB', 't2.faas.name');
+      t.equal(t2.faas.name, 'azfunc3/HttpFnDistTraceB', 't2.faas.name');
       t.equal(t2.parent_id, s2.id, 't2 is a child of s2');
       t.equal(
         t2.context.request.headers.traceparent,
@@ -659,7 +651,7 @@ tape.test(
   },
 );
 
-tape.test('azure functions', function (suite) {
+tape.test('azure functions v3', function (suite) {
   let apmServer;
   let apmServerUrl;
 
@@ -673,7 +665,7 @@ tape.test('azure functions', function (suite) {
   });
 
   let fnAppProc;
-  suite.test('setup: "func start" for AJsAzureFnApp fixture', (t) => {
+  suite.test('setup: "func start" for azfunc3 fixture', (t) => {
     fnAppProc = spawn(funcExe, ['start'], {
       cwd: fnAppDir,
       env: Object.assign({}, process.env, {
@@ -698,11 +690,11 @@ tape.test('azure functions', function (suite) {
       // binaries to "$fnAppDir/node_modules/azure-functions-core-tools/...",
       // which means a local test run on macOS followed by an attempted test run
       // in Docker will result in a crash:
-      //    node_tests_1  | # ["func start" stderr] /app/test/instrumentation/azure-functions/fixtures/AJsAzureFnApp/node_modules/azure-functions-core-tools/bin/func: 1: Syntax error: "(" unexpected
+      //    node_tests_1  | # ["func start" stderr] /app/test/instrumentation/azure-functions/fixtures/azfunc3/node_modules/azure-functions-core-tools/bin/func: 1: Syntax error: "(" unexpected
       //    node_tests_1  | not ok 2 "func start" failed early: code=2
       // For now the workaround is to manually clean that tree before running
       // tests on a separate OS:
-      //    rm -rf test/instrumentation/azure-functions/fixtures/AJsAzureFnApp/node_modules
+      //    rm -rf test/instrumentation/azure-functions/fixtures/azfunc3/node_modules
       t.fail(`"func start" failed early: code=${code}`);
       fnAppProc = null;
       clearTimeout(earlyCloseTimer);

test/instrumentation/azure-functions/fixtures/azfunc3/README.md

@@ -0,0 +1,3 @@
+A Node.js Azure Functions app, using **version 3** of the Node.js
+programming model. See:
+https://learn.microsoft.com/en-ca/azure/azure-functions/functions-node-upgrade-v4

test/instrumentation/azure-functions/fixtures/AJsAzureFnApp/local.settings.json → test/instrumentation/azure-functions/fixtures/azfunc3/local.settings.json

@@ -7,7 +7,7 @@
     "WEBSITE_NODE_DEFAULT_VERSION": "~16",
     "FUNCTIONS_EXTENSION_VERSION": "~4",
 
-    "WEBSITE_SITE_NAME": "AJsAzureFnApp",
+    "WEBSITE_SITE_NAME": "azfunc3",
     "WEBSITE_OWNER_NAME": "2491fc8e-f7c1-4020-b9c6-78509919fd16+my-resource-group-ARegionShortNamewebspace",
     "WEBSITE_RESOURCE_GROUP": "my-resource-group",
     "WEBSITE_INSTANCE_ID": "test-website-instance-id",

test/instrumentation/azure-functions/fixtures/azfunc4/.funcignore

@@ -0,0 +1,10 @@
+*.js.map
+*.ts
+.git*
+.vscode
+local.settings.json
+test
+getting_started.md
+node_modules/@types/
+node_modules/azure-functions-core-tools/
+node_modules/typescript/

test/instrumentation/azure-functions/fixtures/azfunc4/.npmrc

@@ -0,0 +1 @@
+lockfile-version=3

test/instrumentation/azure-functions/fixtures/azfunc4/README.md

@@ -0,0 +1,3 @@
+A Node.js Azure Functions app, using **version 4** of the Node.js
+programming model. See:
+https://learn.microsoft.com/en-ca/azure/azure-functions/functions-node-upgrade-v4

test/instrumentation/azure-functions/fixtures/azfunc4/host.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}

test/instrumentation/azure-functions/fixtures/azfunc4/initapm.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and other contributors where applicable.
+ * Licensed under the BSD 2-Clause License; you may not use this file except in
+ * compliance with the BSD 2-Clause License.
+ */
+
+// For the normal use case an "initapm.js" would look like:
+//    module.exports = require('elastic-apm-node').start(/* { ... } */)
+
+console.log('initapm: XXX start');
+module.exports = require('../../../../../').start({
+  // XXX remove all this config before merging
+  // XXX
+  logLevel: 'trace',
+  apiRequestTime: '5s',
+  // XXX traceContext Qs
+  traceContinuationStrategy: 'restart_external',
+  // XXX not fancy yet
+  cloudProvider: 'none',
+  centralConfig: false,
+  metricsInterval: '0s',
+});
+console.log('initapm: XXX done');

test/instrumentation/azure-functions/fixtures/azfunc4/local.settings.json

@@ -0,0 +1,14 @@
+{
+  "IsEncrypted": false,
+  "Values": {
+    "FUNCTIONS_WORKER_RUNTIME": "node",
+    "AzureWebJobsStorage": "",
+
+    "FUNCTIONS_EXTENSION_VERSION": "~4",
+    "WEBSITE_SITE_NAME": "azfunc4",
+    "WEBSITE_OWNER_NAME": "2491fc8e-f7c1-4020-b9c6-78509919fd16+my-resource-group-ARegionShortNamewebspace",
+    "WEBSITE_RESOURCE_GROUP": "my-resource-group",
+    "WEBSITE_INSTANCE_ID": "test-website-instance-id",
+    "REGION_NAME": "test-region-name"
+  }
+}