Merge branch 'main' into aws-lite

.gitignore

@@ -5,6 +5,7 @@ __pycache__
 *-vendor.js
 bin/*.json
 bin/sandbox-binary*
+chonky.txt
 coverage/
 node_modules/
 package-lock.json

changelog.md

@@ -2,6 +2,31 @@
 
 ---
 
+## [5.9.0 - 5.9.1] 2023-10-24
+
+### Added
+
+- Add coldstart simulator via `@sandbox coldstart true` setting in `prefs.arc`
+ - Note: Windows users must install [`du`](https://learn.microsoft.com/en-us/sysinternals/downloads/du)
+
+---
+
+## [5.8.5] 2023-10-24
+
+### Fixed
+
+- Fixed issue where qemu/emulator port conflicts were not detected with our open port tester; fixes 1441
+
+---
+
+## [5.8.4] 2023-10-23
+
+### Added
+
+- Added support for hydration of platform-specific binary deps (namely: Python); fixes #1457
+
+---
+
 ## [5.8.3] 2023-10-17
 
 ### Added

package.json

@@ -1,12 +1,12 @@
 {
  "name": "@architect/sandbox",
- "version": "5.8.3",
+ "version": "5.9.1",
  "description": "Architect dev server: run full Architect projects locally & offline",
  "main": "src/index.js",
  "scripts": {
  "test": "npm run lint && npm run test:integration && npm run coverage",
- "test:unit": "cross-env tape 'test/unit/**/*-test.js' | tap-spec",
- "test:integration": "cross-env tape 'test/integration/**/*-test.js' | tap-spec",
+ "test:unit": "cross-env tape 'test/unit/**/*-test.js' | tap-arc",
+ "test:integration": "cross-env tape 'test/integration/**/*-test.js' | tap-arc",
  "coverage": "nyc --reporter=lcov --reporter=text npm run test:unit",
  "lint": "eslint . --fix",
  "rc": "npm version prerelease --preid RC",
@@ -30,8 +30,8 @@
  "dependencies": {
  "@architect/asap": "~6.0.3",
  "@architect/create": "~4.2.3",
- "@architect/hydrate": "~3.3.0",
- "@architect/inventory": "~3.6.1",
+ "@architect/hydrate": "~3.4.1",
+ "@architect/inventory": "~3.6.2",
  "@architect/utils": "~4.0.0-RC.0",
  "@aws-lite/client": "~0.11.1",
  "@aws-lite/dynamodb": "~0.2.2",
@@ -41,9 +41,9 @@
  "depstatus": "~1.1.1",
  "dynalite": "~3.2.2",
  "finalhandler": "~1.2.0",
- "glob": "~10.3.3",
+ "glob": "~10.3.10",
  "http-proxy": "~1.18.1",
- "lambda-runtimes": "~1.1.4",
+ "lambda-runtimes": "~1.1.5",
  "minimist": "~1.2.8",
  "router": "~1.3.8",
  "run-parallel": "~1.2.0",
@@ -53,22 +53,22 @@
  "tmp": "~0.2.1",
  "tree-kill": "~1.2.2",
  "update-notifier-cjs": "~5.1.6",
- "ws": "~8.13.0"
+ "ws": "~8.14.2"
  },
  "devDependencies": {
- "@architect/eslint-config": "~2.1.1",
+ "@architect/eslint-config": "~2.1.2",
  "@architect/functions": "~7.0.0",
  "@architect/req-res-fixtures": "git+https://github.com/architect/req-res-fixtures.git",
  "@aws-lite/apigatewaymanagementapi": "~0.0.2",
  "@aws-lite/ssm": "~0.2.0",
  "cross-env": "~7.0.3",
- "eslint": "~8.47.0",
+ "eslint": "~8.52.0",
  "fs-extra": "~11.1.1",
  "nyc": "~15.1.0",
  "pkg": "~5.8.1",
  "proxyquire": "~2.1.3",
- "tap-spec": "~5.0.0",
- "tape": "~5.6.6",
+ "tap-arc": "~1.1.0",
+ "tape": "~5.7.2",
  "tiny-json-http": "~7.5.1"
  },
  "eslintConfig": {

src/invoke-lambda/exec/index.js

@@ -1,9 +1,15 @@
 let _asap = require('@architect/asap')
 let load = require('./loader')
 let spawn = require('./spawn')
-let { runtimeEval } = require('../../lib')
+let { getFolderSize, runtimeEval } = require('../../lib')
 let { invocations } = require('../../arc/_runtime-api')
 
+let tenMB = 1024 * 1024 * 10
+function coldstartMs (bytes) {
+ // Very rough coldstart estimate: ~10MB = 100ms
+ return Math.floor(bytes / (tenMB / 100))
+}
+
 module.exports = function exec (lambda, params, callback) {
  // ASAP is a special case that doesn't spawn
  if (lambda.arcStaticAssetProxy) {
@@ -34,7 +40,20 @@ module.exports = function exec (lambda, params, callback) {
  let bootstrap = load()[run]
  var { command, args } = runtimeEval[run](bootstrap)
  }
- spawn({ command, args, ...params, lambda }, callback)
+ if (params.coldstart) {
+ getFolderSize(lambda.src, (err, folderSize) => {
+ if (err) callback(err)
+ else {
+ let { requestID } = params
+ let coldstart = coldstartMs(folderSize)
+ params.update.verbose.status(`[${requestID}] Coldstart simulator: ${coldstart}ms latency added to ${folderSize}b Lambda`)
+ spawn({ command, args, ...params, coldstart, lambda }, callback)
+ }
+ })
+ }
+ else {
+ spawn({ command, args, ...params, lambda }, callback)
+ }
  }
 }
 

src/invoke-lambda/exec/runtimes/deno.mjs

@@ -55,9 +55,10 @@ const headers = { 'content-type': 'application/json; charset=utf-8' };
  }
  catch (err) {
  (async function initError () {
- console.log('Lambda init error:', err);
+ const unknown = 'Unknown init error';
+ console.log('Lambda init error:', err || unknown);
  const initErrorEndpoint = url('init/error');
- const errorMessage = err.message || 'Unknown init error';
+ const errorMessage = err.message || unknown;
  const errorType = err.name || 'Unknown init error type';
  const stackTrace = err.stack ? err.stack.split('\n') : undefined;
  const body = JSON.stringify({ errorMessage, errorType, stackTrace });

src/invoke-lambda/exec/runtimes/node-esm.mjs

@@ -101,9 +101,10 @@ function client (method, params) {
  }
  catch (err) {
  (async function initError () {
- console.log('Lambda init error:', err.body || err.message);
+ let unknown = 'Unknown init error';
+ console.log('Lambda init error:', err.body || err.message || unknown);
  let initErrorEndpoint = url('init/error');
- let errorMessage = err.message || 'Unknown init error';
+ let errorMessage = err.message || unknown;
  let errorType = err.name || 'Unknown init error type';
  let stackTrace = err.stack ? err.stack.split('\n') : undefined;
  let body = { errorMessage, errorType, stackTrace };

src/invoke-lambda/exec/runtimes/node.js

@@ -185,9 +185,10 @@ function client (method, params) {
  catch (err) {
  /* eslint-disable-next-line */
  (async function initError () {
- console.log('Lambda init error:', err.body || err.message);
+ let unknown = 'Unknown init error';
+ console.log('Lambda init error:', err.body || err.message || unknown);
  let initErrorEndpoint = url('init/error');
- let errorMessage = err.message || 'Unknown init error';
+ let errorMessage = err.message || unknown;
  let errorType = err.name || 'Unknown init error type';
  let stackTrace = err.stack ? err.stack.split('\n') : undefined;
  let body = { errorMessage, errorType, stackTrace };

src/invoke-lambda/exec/spawn.js

@@ -5,46 +5,50 @@ let errors = require('../../lib/errors')
 let { invocations } = require('../../arc/_runtime-api')
 
 module.exports = function spawnChild (params, callback) {
- let { args, context, command, lambda, options, requestID, timeout } = params
+ let { args, coldstart, context, command, lambda, options, requestID, timeout } = params
  let { apiType, update } = context
  let isInLambda = process.env.AWS_LAMBDA_FUNCTION_NAME
  let timedOut = false
 
  // Let's go!
- let child = spawn(command, args, options)
- let pid = child.pid
- let error
- let closed
-
- child.stdout.on('data', data => process.stdout.write('\n' + data))
- child.stderr.on('data', data => process.stderr.write('\n' + data))
- child.on('error', err => {
- error = err
- // Seen some non-string oob errors come via binary compilation
- if (err.code) shutdown('error')
- })
- child.on('close', (code, signal) => {
- update.debug.status(`[${requestID}] Emitted 'close' (pid ${pid}, code '${code}', signal '${signal}')`)
- shutdown('child process closure')
- })
-
- // Set an execution timeout
- let to = setTimeout(function () {
- timedOut = true
- let duration = `${timeout / 1000}s`
- update.warn(`[${requestID}] Timed out after hitting its ${duration} timeout!`)
- shutdown(`${duration} timeout`)
- }, timeout)
-
- // Terminate once we find a result from the runtime API
- // 25ms is arbitrary, but hopefully it should be solid enough
- let check = setInterval(function () {
- if (invocations[requestID].response ||
- invocations[requestID].initError ||
- invocations[requestID].error) {
- shutdown('runtime API completion check')
- }
- }, 25)
+ let pid = 'init'
+ let child, error, closed, to, check
+ function start () {
+ child = spawn(command, args, options)
+ pid = child.pid
+
+ child.stdout.on('data', data => process.stdout.write('\n' + data))
+ child.stderr.on('data', data => process.stderr.write('\n' + data))
+ child.on('error', err => {
+ error = err
+ // Seen some non-string oob errors come via binary compilation
+ if (err.code) shutdown('error')
+ })
+ child.on('close', (code, signal) => {
+ update.debug.status(`[${requestID}] Emitted 'close' (pid ${pid}, code '${code}', signal '${signal}')`)
+ shutdown('child process closure')
+ })
+
+ // Set an execution timeout
+ to = setTimeout(function () {
+ timedOut = true
+ let duration = `${timeout / 1000}s`
+ update.warn(`[${requestID}] Timed out after hitting its ${duration} timeout!`)
+ shutdown(`${duration} timeout`)
+ }, timeout)
+
+ // Terminate once we find a result from the runtime API
+ // 25ms is arbitrary, but hopefully it should be solid enough
+ check = setInterval(function () {
+ if (invocations[requestID].response ||
+ invocations[requestID].initError ||
+ invocations[requestID].error) {
+ shutdown('runtime API completion check')
+ }
+ }, 25)
+ }
+ if (coldstart) setTimeout(start, coldstart)
+ else start()
 
  // Ensure we don't have dangling processes due to open connections, etc. before we wrap up
  function shutdown (event) {
@@ -63,7 +67,7 @@ module.exports = function spawnChild (params, callback) {
  let isRunning = true
  try {
  // Signal 0 is a special node construct, see: https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_kill_pid_signal
- isRunning = process.kill(pid, 0)
+ isRunning = pid === 'init' ? false : process.kill(pid, 0)
  }
  catch (err) {
  isRunning = err.code === 'EPERM'

src/invoke-lambda/index.js

@@ -42,6 +42,8 @@ module.exports = function invokeLambda (params, callback) {
  update.debug.raw(output + '...')
  if (chonky) update.debug.status(`[${requestID}] Truncated event payload log at 10KB`)
 
+ let coldstart = inventory.inv._project?.preferences?.sandbox?.coldstart || false
+
  invocations[requestID] = {
  request: event,
  lambda,
@@ -58,6 +60,7 @@ module.exports = function invokeLambda (params, callback) {
  },
  requestID,
  timeout: config.timeout * 1000,
+ coldstart,
  update,
  }, function done (err) {
  update.debug.status(`[${requestID}] Final invocation state`)

src/lib/get-folder-size.js

@@ -0,0 +1,37 @@
+let { exec } = require('child_process')
+
+// Adapted with gratitude from https://github.com/simoneb/fast-folder-size under the AWISC license
+let commands = {
+ win32: `du${process.arch === 'x64' ? '64' : ''}.exe -nobanner -accepteula -q -c .`,
+ darwin: `du -sk .`,
+ linux: `du -sb .`,
+}
+let processOutput = {
+ win32 (stdout) {
+ // query stats indexes from the end since path can contain commas as well
+ const stats = stdout.split('\n')[1].split(',')
+ const bytes = +stats.slice(-2)[0]
+ return bytes
+ },
+ darwin (stdout) {
+ const match = /^(\d+)/.exec(stdout)
+ const bytes = Number(match[1]) * 1024
+ return bytes
+ },
+ linux (stdout) {
+ const match = /^(\d+)/.exec(stdout)
+ const bytes = Number(match[1])
+ return bytes
+ },
+}
+
+module.exports = function getFolderSize (cwd, callback) {
+ let sys = process.platform
+ if (!Object.keys(commands).includes(sys)) {
+ return callback(Error('Coldstart testing only supported on Linux, Mac, and Windows'))
+ }
+ exec(commands[sys], { cwd }, (err, stdout) => {
+ if (err) callback(err)
+ else callback(null, processOutput[sys](stdout))
+ })
+}

src/lib/index.js

@@ -1,9 +1,11 @@
+let getFolderSize = require('./get-folder-size')
 let makeRequestId = require('./request-id')
 let runtimeEval = require('./runtime-eval')
 let template = require('./template')
 let userEnvVars = require('./user-env-vars')
 
 module.exports = {
+ getFolderSize,
  makeRequestId,
  runtimeEval,
  template,