feat: allow bundling lambdas with cdk-assets

packages/cdk-assets/README.md

@@ -27,6 +27,7 @@ the manifest.
 Currently the following asset types are supported:
 
 * Files and archives, uploaded to S3
+    * Can be either pre-bundled or bundled when `cdk-assets` is called
 * Docker Images, uploaded to ECR
 * Files, archives, and Docker images built by external utilities
 
@@ -90,6 +91,28 @@ An asset manifest looks like this:
         }
       }
     },
+    "7aac5b80b050e7e4e168f84feffa5894": {
+      "source": {
+        "path": "some_directory",
+        "packaging": "zip",
+        "bundling": {
+            "image": "public.ecr.aws/sam/build-nodejs16.x",
+            "command": [
+              "bash", "-c",
+              "npm i -g esbuild && esbuild --bundle \"/asset-input/app.ts\" --target=node16 --platform=node --outfile=\"/asset-output/index.js\" --external:aws-sdk --loader:.png=dataurl"
+            ],
+            "workingDirectory": "/"
+          }
+      },
+      "destinations": {
+        "us-east-1": {
+          "region": "us-east-1",
+          "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account",
+          "bucketName": "MyBucket",
+          "objectKey": "7aac5b80b050e7e4e168f84feffa5894.zip"
+        }
+      }
+    },
     "3dfe2b80b050e7e4e168f84feff678d4": {
       "source": {
         "executable": ["myzip"]

packages/cdk-assets/lib/private/docker.ts

@@ -1,6 +1,8 @@
+import { spawnSync, SpawnSyncOptions } from 'child_process';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
+import { BundlingOptions, DockerVolumeConsistency } from '@aws-cdk/cloud-assembly-schema';
 import { cdkCredentialsConfig, obtainEcrCredentials } from './docker-credentials';
 import { Logger, shell, ShellOptions } from './shell';
 import { createCriticalSection } from './util';
@@ -215,3 +217,155 @@ function getDockerCmd(): string {
 function flatten(x: string[][]) {
   return Array.prototype.concat([], ...x);
 }
+
+export class ContainerBunder {
+
+  /**
+   * The directory inside the bundling container into which the asset sources will be mounted.
+   */
+  public static readonly BUNDLING_INPUT_DIR = '/asset-input';
+
+  /**
+   * The directory inside the bundling container into which the bundled output should be written.
+   */
+  public static readonly BUNDLING_OUTPUT_DIR = '/asset-output';
+
+  constructor(
+    private readonly bundlingOption: BundlingOptions,
+    private readonly sourceDir: string,
+  ) {}
+  /**
+   * Bundle asset using a container
+   */
+  public bundle() {
+    // Always mount input and output dir
+    const bundleDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'cdk-docker-bundle-'));
+    const volumes = [
+      {
+        hostPath: this.sourceDir,
+        containerPath: ContainerBunder.BUNDLING_INPUT_DIR,
+      },
+      {
+        hostPath: bundleDir,
+        containerPath: ContainerBunder.BUNDLING_OUTPUT_DIR,
+      },
+      ...this.bundlingOption.volumes ?? [],
+    ];
+    const environment = this.bundlingOption.environment || {};
+    const entrypoint = this.bundlingOption.entrypoint?.[0] || null;
+    const command = [
+      ...this.bundlingOption.entrypoint?.[1]
+        ? [...this.bundlingOption.entrypoint.slice(1)]
+        : [],
+      ...this.bundlingOption.command
+        ? [...this.bundlingOption.command]
+        : [],
+    ];
+
+    const dockerArgs: string[] = [
+      'run', '--rm',
+      ...this.bundlingOption.securityOpt
+        ? ['--security-opt', this.bundlingOption.securityOpt]
+        : [],
+      ...this.bundlingOption.network
+        ? ['--network', this.bundlingOption.network]
+        : [],
+      ...this.bundlingOption.user
+        ? ['-u', this.bundlingOption.user]
+        : [],
+      ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}:${isSeLinux() ? 'z,' : ''}${DockerVolumeConsistency.DELEGATED}`])),
+      ...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])),
+      ...this.bundlingOption.workingDirectory
+        ? ['-w', this.bundlingOption.workingDirectory]
+        : [],
+      ...entrypoint
+        ? ['--entrypoint', entrypoint]
+        : [],
+      this.bundlingOption.image,
+      ...command,
+    ];
+
+    dockerExec(dockerArgs);
+    return bundleDir;
+  }
+
+  /**
+     * Copies a file or directory out of the Docker image to the local filesystem.
+     *
+     * If `outputPath` is omitted the destination path is a temporary directory.
+     *
+     * @param imagePath the path in the Docker image
+     * @param outputPath the destination path for the copy operation
+     * @returns the destination path
+     */
+  public cp(imagePath: string, outputPath?: string): string {
+    const { stdout } = dockerExec(['create', this.bundlingOption.image], {}); // Empty options to avoid stdout redirect here
+
+    const match = stdout.toString().match(/([0-9a-f]{16,})/);
+    if (!match) {
+      throw new Error('Failed to extract container ID from Docker create output');
+    }
+
+    const containerId = match[1];
+    const containerPath = `${containerId}:${imagePath}`;
+    const destPath = outputPath ?? fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'cdk-docker-cp-'));
+    try {
+      dockerExec(['cp', containerPath, destPath]);
+      return destPath;
+    } catch (err) {
+      throw new Error(`Failed to copy files from ${containerPath} to ${destPath}: ${err}`);
+    } finally {
+      dockerExec(['rm', '-v', containerId]);
+    }
+  }
+}
+
+function dockerExec(args: string[], options?: SpawnSyncOptions) {
+  const prog = process.env.CDK_DOCKER ?? 'docker';
+  const proc = spawnSync(prog, args, options ?? {
+    stdio: [ // show Docker output
+      'ignore', // ignore stdio
+      process.stderr, // redirect stdout to stderr
+      'inherit', // inherit stderr
+    ],
+  });
+
+  if (proc.error) {
+    throw proc.error;
+  }
+
+  if (proc.status !== 0) {
+    if (proc.stdout || proc.stderr) {
+      throw new Error(`[Status ${proc.status}] stdout: ${proc.stdout?.toString().trim()}\n\n\nstderr: ${proc.stderr?.toString().trim()}`);
+    }
+    throw new Error(`${prog} exited with status ${proc.status}`);
+  }
+
+  return proc;
+}
+
+function isSeLinux() : boolean {
+  if (process.platform != 'linux') {
+    return false;
+  }
+  const prog = 'selinuxenabled';
+  const proc = spawnSync(prog, [], {
+    stdio: [ // show selinux status output
+      'pipe', // get value of stdio
+      process.stderr, // redirect stdout to stderr
+      'inherit', // inherit stderr
+    ],
+  });
+
+  if (proc.error) {
+    // selinuxenabled not a valid command, therefore not enabled
+    return false;
+  }
+  if (proc.status == 0) {
+    // selinux enabled
+    return true;
+  } else {
+    // selinux not enabled
+    return false;
+  }
+}

packages/cdk-assets/lib/private/handlers/bundlable-files.ts

@@ -0,0 +1,26 @@
+import * as path from 'path';
+import { ContainerBunder } from '../docker';
+import { FileAssetHandler } from './files';
+
+export class BundlableFileAssetHandler extends FileAssetHandler {
+
+  public async build(): Promise<void> {
+    if (!this.asset.source.bundling) {
+      throw new Error('Tried to do bundling without BundlingOptions');
+    }
+
+    if (!this.asset.source.path) {
+      throw new Error('Source path is mandatory when bundling inside container');
+    }
+
+    const bundler = new ContainerBunder(
+      this.asset.source.bundling,
+      path.resolve(this.workDir, this.asset.source.path),
+    );
+
+    const bundledPath = await bundler.bundle();
+
+    // Hack to get things tested
+    (this.asset.source.path as any) = bundledPath;
+  }
+}

packages/cdk-assets/lib/private/handlers/files.ts

@@ -21,9 +21,9 @@ export class FileAssetHandler implements IAssetHandler {
   private readonly fileCacheRoot: string;
 
   constructor(
-    private readonly workDir: string,
-    private readonly asset: FileManifestEntry,
-    private readonly host: IHandlerHost) {
+    protected readonly workDir: string,
+    protected readonly asset: FileManifestEntry,
+    protected readonly host: IHandlerHost) {
     this.fileCacheRoot = path.join(workDir, '.cache');
   }
 
@@ -80,6 +80,7 @@ export class FileAssetHandler implements IAssetHandler {
     }
 
     if (this.host.aborted) { return; }
+
     const publishFile = this.asset.source.executable ?
       await this.externalPackageFile(this.asset.source.executable) : await this.packageFile(this.asset.source);
 
@@ -130,7 +131,6 @@ export class FileAssetHandler implements IAssetHandler {
     }
 
     const fullPath = path.resolve(this.workDir, source.path);
-
     if (source.packaging === FileAssetPackaging.ZIP_DIRECTORY) {
       const contentType = 'application/zip';
 

packages/cdk-assets/lib/private/handlers/index.ts

@@ -1,11 +1,16 @@
 import { AssetManifest, DockerImageManifestEntry, FileManifestEntry, IManifestEntry } from '../../asset-manifest';
 import { IAssetHandler, IHandlerHost } from '../asset-handler';
+import { BundlableFileAssetHandler } from './bundlable-files';
 import { ContainerImageAssetHandler } from './container-images';
 import { FileAssetHandler } from './files';
 
 export function makeAssetHandler(manifest: AssetManifest, asset: IManifestEntry, host: IHandlerHost): IAssetHandler {
   if (asset instanceof FileManifestEntry) {
-    return new FileAssetHandler(manifest.directory, asset, host);
+    if (asset.source.bundling) {
+      return new BundlableFileAssetHandler(manifest.directory, asset, host);
+    } else {
+      return new FileAssetHandler(manifest.directory, asset, host);
+    }
   }
   if (asset instanceof DockerImageManifestEntry) {
     return new ContainerImageAssetHandler(manifest.directory, asset, host);

packages/cdk-assets/test/bundlable-files.test.ts

@@ -0,0 +1,97 @@
+jest.mock('child_process');
+
+import * as child_process from 'child_process';
+import { Manifest } from '@aws-cdk/cloud-assembly-schema';
+import * as mockfs from 'mock-fs';
+import { AssetPublishing, AssetManifest } from '../lib';
+import { mockAws, mockedApiResult, mockUpload } from './mock-aws';
+
+const DEFAULT_DESTINATION = {
+  region: 'us-north-50',
+  assumeRoleArn: 'arn:aws:role',
+  bucketName: 'some_bucket',
+  objectKey: 'some_key',
+};
+
+let aws: ReturnType<typeof mockAws>;
+beforeEach(() => {
+  jest.resetAllMocks();
+
+  mockfs({
+    '/bundlable/cdk.out/assets.json': JSON.stringify({
+      version: Manifest.version(),
+      files: {
+        theAsset: {
+          source: {
+            path: 'some_file',
+            packaging: 'zip',
+            bundling: {
+              image: 'node:14',
+            },
+          },
+          destinations: { theDestination: DEFAULT_DESTINATION },
+        },
+      },
+    }),
+  });
+
+  aws = mockAws();
+});
+
+afterEach(() => {
+  mockfs.restore();
+});
+
+function getSpawnSyncReturn(status: number, output=['mock output']): child_process.SpawnSyncReturns<string | Buffer> {
+  return {
+    status,
+    stderr: Buffer.from('stderr'),
+    stdout: Buffer.from('stdout'),
+    pid: 123,
+    output,
+    signal: null,
+  };
+}
+
+describe('bundlable assets', () => {
+  test('bundle correctly within container', async () => {
+    const pub = new AssetPublishing(AssetManifest.fromPath('/bundlable/cdk.out'), { aws });
+    aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined });
+    aws.mockS3.upload = mockUpload();
+    jest.spyOn(child_process, 'spawnSync').mockImplementation((command: string, args?: readonly string[], _options?: child_process.SpawnSyncOptions) => {
+      if (command === 'selinuxenabled') {
+        return getSpawnSyncReturn(0, ['selinuxenabled output', 'stderr']);
+      } else if (command === 'docker' && args) {
+        if (args[0] === 'run') {
+          // Creation of asset by running the image
+          return getSpawnSyncReturn(0, ['Bundling started', 'Bundling done']);
+        }
+      }
+      return getSpawnSyncReturn(127, ['FAIL']);
+    });
+
+    await pub.publish();
+
+    expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({
+      Bucket: 'some_bucket',
+      Key: 'some_key',
+    }));
+  });
+
+  test('fails if container run returns an error', async () => {
+    const pub = new AssetPublishing(AssetManifest.fromPath('/bundlable/cdk.out'), { aws });
+    jest.spyOn(child_process, 'spawnSync').mockImplementation((command: string, args?: readonly string[], _options?: child_process.SpawnSyncOptions) => {
+      if (command === 'selinuxenabled') {
+        return getSpawnSyncReturn(0, ['selinuxenabled output', 'stderr']);
+      } else if (command === 'docker' && args) {
+        if (args[0] === 'run') {
+          // Creation of asset by running the image
+          return getSpawnSyncReturn(127, ['Bundling started', 'Bundling failed in container']);
+        }
+      }
+      return getSpawnSyncReturn(127, ['FAIL']);
+    });
+
+    await expect(pub.publish()).rejects.toThrow('Error building and publishing: [Status 127] stdout: stdout');
+  });
+});