From f6c84566bb6a698709dc3474726000f07b94f3db Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Tue, 7 May 2024 17:41:01 -0700 Subject: [PATCH 1/4] feat(ses): Add XS variant of shim --- .../eslint-plugin/lib/configs/internal.js | 2 +- packages/ses/.gitignore | 1 + packages/ses/NEWS.md | 12 +- packages/ses/package.json | 13 +- packages/ses/scripts/generate-test-xs.js | 56 ++++ packages/ses/src-xs/commons.js | 29 ++ packages/ses/src-xs/compartment-shim.js | 134 ++++++++ packages/ses/src-xs/compartment.js | 315 ++++++++++++++++++ packages/ses/src-xs/index.js | 27 ++ packages/ses/src-xs/lockdown-shim.js | 29 ++ packages/ses/src/commons.js | 28 +- packages/ses/src/compartment-shim.js | 3 + packages/ses/src/compartment.js | 56 +++- packages/ses/src/global-object.js | 5 +- packages/ses/src/make-safe-evaluator.js | 14 +- packages/ses/src/transforms.js | 3 +- packages/ses/test/_meaning.js | 1 + packages/ses/test/_xs.js | 226 +++++++++++++ typedoc.json | 2 + 19 files changed, 927 insertions(+), 29 deletions(-) create mode 100644 packages/ses/.gitignore create mode 100644 packages/ses/scripts/generate-test-xs.js create mode 100644 packages/ses/src-xs/commons.js create mode 100644 packages/ses/src-xs/compartment-shim.js create mode 100644 packages/ses/src-xs/compartment.js create mode 100644 packages/ses/src-xs/index.js create mode 100644 packages/ses/src-xs/lockdown-shim.js create mode 100644 packages/ses/test/_meaning.js create mode 100644 packages/ses/test/_xs.js diff --git a/packages/eslint-plugin/lib/configs/internal.js b/packages/eslint-plugin/lib/configs/internal.js index 9c5f180845..17cc652059 100644 --- a/packages/eslint-plugin/lib/configs/internal.js +++ b/packages/eslint-plugin/lib/configs/internal.js @@ -24,7 +24,7 @@ const rules = { dynamicConfig.overrides.push({ extends: ['plugin:@endo/recommended-requiring-type-checking'], files: fileGlobs, - excludedFiles: ['**/src/**/exports.js'], + excludedFiles: ['**/src*/**/exports.js'], parserOptions, rules, }); diff --git a/packages/ses/.gitignore b/packages/ses/.gitignore new file mode 100644 index 0000000000..a9a5aecf42 --- /dev/null +++ b/packages/ses/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/packages/ses/NEWS.md b/packages/ses/NEWS.md index d1b8131e5a..921175873d 100644 --- a/packages/ses/NEWS.md +++ b/packages/ses/NEWS.md @@ -1,6 +1,6 @@ User-visible changes in `ses`: -# Next version +# Next release - Adds support for dynamic `import` in conjunction with an update to `@endo/module-source`. @@ -8,6 +8,16 @@ User-visible changes in `ses`: - Specifying the long-discontinued `mathTaming` or `dateTaming` options logs a warning. +Incubating: Please do not rely on these features as they are under development +and subject to breaking changes that will not be signaled by semver. + +- Adds support for an XS-specific variant of the SES shim that is triggered + with the `xs` package export condition. + This version of SES preserves all the features of `Compartment` provided + uniquely by the SES shim, but with the `__native__` constructor option, + loses support for importing precompiled module records and gains support + for native `ModuleSource`. + # v1.10.0 (2024-11-13) - Permit [Promise.try](https://github.com/tc39/proposal-promise-try), diff --git a/packages/ses/package.json b/packages/ses/package.json index c6e77c27af..e552bda739 100644 --- a/packages/ses/package.json +++ b/packages/ses/package.json @@ -38,6 +38,7 @@ ".": { "import": { "types": "./types.d.ts", + "xs": "./src-xs/index.js", "default": "./index.js" }, "require": { @@ -57,8 +58,14 @@ }, "./tools.js": "./tools.js", "./assert-shim.js": "./assert-shim.js", - "./lockdown-shim.js": "./lockdown-shim.js", - "./compartment-shim.js": "./compartment-shim.js", + "./lockdown-shim.js": { + "xs": "./src-xs/lockdown-shim.js", + "default": "./lockdown-shim.js" + }, + "./compartment-shim.js": { + "xs": "./src-xs/compartment-shim.js", + "default": "./compartment-shim.js" + }, "./console-shim.js": "./console-shim.js", "./package.json": "./package.json" }, @@ -74,7 +81,7 @@ "prepare": "npm run clean && npm run build", "qt": "ava", "test": "tsd && ava", - "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js", + "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js && node scripts/generate-test-xs.js && xst tmp/test-xs.js && rm -rf tmp", "postpack": "git clean -f '*.d.ts*' '*.tsbuildinfo'" }, "dependencies": { diff --git a/packages/ses/scripts/generate-test-xs.js b/packages/ses/scripts/generate-test-xs.js new file mode 100644 index 0000000000..79588a9fcb --- /dev/null +++ b/packages/ses/scripts/generate-test-xs.js @@ -0,0 +1,56 @@ +/* eslint-env node */ +/* glimport/no-extraneous-dependenciesobal process */ +import '../index.js'; +import { promises as fs } from 'fs'; +// Lerna does not like dependency cycles. +// With an explicit devDependency from module-source to compartment-mapper, +// the build script stalls before running every package's build script. +// yarn lerna run build +// Omitting the dependency from package.json solves the problem and works +// by dint of shared workspace node_modules. +// eslint-disable-next-line import/no-extraneous-dependencies +import { makeBundle } from '@endo/compartment-mapper/bundle.js'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { ModuleSource } from '@endo/module-source'; +import { fileURLToPath } from 'url'; + +const read = async location => { + const path = fileURLToPath(location); + return fs.readFile(path); +}; +const write = async (location, content) => { + const path = fileURLToPath(location); + await fs.writeFile(path, content); +}; + +const main = async () => { + await fs.mkdir(fileURLToPath(new URL('../tmp', import.meta.url)), { + recursive: true, + }); + + const meaningText = await fs.readFile( + fileURLToPath(new URL('../test/_meaning.js', import.meta.url)), + 'utf8', + ); + const meaningModuleSource = new ModuleSource(meaningText); + + await fs.writeFile( + fileURLToPath(new URL('../tmp/_meaning.pre-mjs.json', import.meta.url)), + JSON.stringify(meaningModuleSource), + ); + + const xsPrelude = await makeBundle( + read, + new URL('../test/_xs.js', import.meta.url).href, + { + tags: new Set(['xs']), + }, + ); + + await write(new URL('../tmp/test-xs.js', import.meta.url).href, xsPrelude); +}; + +main().catch(err => { + console.error('Error running main:', err); + process.exitCode = 1; +}); diff --git a/packages/ses/src-xs/commons.js b/packages/ses/src-xs/commons.js new file mode 100644 index 0000000000..353cbc3fde --- /dev/null +++ b/packages/ses/src-xs/commons.js @@ -0,0 +1,29 @@ +/** + * @module In the spirit of ../src/commons.js, this module captures native + * functions specific to the XS engine during initialization, so vetted shims + * are free to modify any intrinsic without risking the integrity of SES. + */ + +/// + +import { + getOwnPropertyDescriptor, + globalThis, + uncurryThis, +} from '../src/commons.js'; + +/** @type {typeof Compartment} */ +export const NativeStartCompartment = /** @type {any} */ (globalThis) + .Compartment; +export const nativeCompartmentPrototype = NativeStartCompartment.prototype; +export const nativeImport = uncurryThis(nativeCompartmentPrototype.import); +export const nativeImportNow = uncurryThis( + nativeCompartmentPrototype.importNow, +); +/** @type {(compartment: any, source: string) => unknown} */ +export const nativeEvaluate = uncurryThis(nativeCompartmentPrototype.evaluate); +/** @type {(compartment: typeof Compartment) => typeof globalThis} */ +export const nativeGetGlobalThis = uncurryThis( + // @ts-expect-error we know it is there on XS + getOwnPropertyDescriptor(nativeCompartmentPrototype, 'globalThis').get, +); diff --git a/packages/ses/src-xs/compartment-shim.js b/packages/ses/src-xs/compartment-shim.js new file mode 100644 index 0000000000..878d5a2b4a --- /dev/null +++ b/packages/ses/src-xs/compartment-shim.js @@ -0,0 +1,134 @@ +/** + * @module Provides a XS-specific variation on the behavior of + * ../compartment-shim.js, completing the story that begins in + * ./compartment.js, adding a Compartment constructor adapter to the global + * scope and transforming all of the methods of a native compartment into + * thunks that will alternately delegate to its native or shim behaviors + * depending on the __native__ Compartment constructor option. + */ + +/// + +import { defineProperty, globalThis, weakmapGet } from '../src/commons.js'; +import { + NativeStartCompartment, + nativeCompartmentPrototype, + nativeImport, + nativeImportNow, + nativeEvaluate, + nativeGetGlobalThis, +} from './commons.js'; +import { + ShimStartCompartment, + adaptCompartmentConstructors, + privateFields, + shimEvaluate, + shimGetGlobalThis, + shimImport, + shimImportNow, +} from './compartment.js'; + +const adapterFunctions = { + evaluate(source, options) { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeEvaluate(this, source); + } + const { delegateNative } = fields; + if (delegateNative) { + const { transforms, nativeEval } = fields; + for (let i = 0; i < transforms.length; i += 1) { + const transform = transforms[i]; + source = transform(source); + } + return nativeEval(source); + } else { + return shimEvaluate(this, source, options); + } + }, + + async import(specifier) { + await null; + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeImport(this, specifier); + } + const { noNamespaceBox, delegateNative } = fields; + const delegateImport = delegateNative ? nativeImport : shimImport; + const namespace = delegateImport(this, specifier); + return noNamespaceBox ? namespace : { namespace: await namespace }; + }, + + importNow(specifier) { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeImportNow(this, specifier); + } + const { delegateNative } = fields; + const delegateImportNow = delegateNative ? nativeImportNow : shimImportNow; + return delegateImportNow(this, specifier); + }, +}; + +defineProperty(nativeCompartmentPrototype, 'evaluate', { + value: adapterFunctions.evaluate, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'import', { + value: adapterFunctions.import, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'importNow', { + value: adapterFunctions.importNow, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'globalThis', { + get() { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeGetGlobalThis(this); + } + const { delegateNative } = fields; + const delegateGetGlobalThis = delegateNative + ? nativeGetGlobalThis + : shimGetGlobalThis; + return delegateGetGlobalThis(this); + }, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'name', { + get() { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return undefined; + } + const { name } = fields; + return name; + }, + configurable: true, + enumerable: false, +}); + +// Adapt the start compartment's native Compartment to the SES-compatibility +// adapter. +// Before Lockdown, the Compartment constructor in transitive child +// Compartments is not (and cannot be) hardened. +const noHarden = object => object; +// @ts-expect-error TypeScript is not inferring from the types above that +// Compartment is on globalThis. +globalThis.Compartment = adaptCompartmentConstructors( + NativeStartCompartment, + ShimStartCompartment, + noHarden, +); diff --git a/packages/ses/src-xs/compartment.js b/packages/ses/src-xs/compartment.js new file mode 100644 index 0000000000..c5ae79218f --- /dev/null +++ b/packages/ses/src-xs/compartment.js @@ -0,0 +1,315 @@ +/* eslint-disable @endo/no-nullish-coalescing */ + +/** @import {ModuleDescriptor} from '../types.js' */ + +import { + Map, + Object, + SyntaxError, + TypeError, + WeakMap, + arrayMap, + create, + defineProperty, + entries, + fromEntries, + getOwnPropertyDescriptor, + globalThis, + mapDelete, + mapGet, + mapSet, + uncurryThis, + weakmapGet, + weakmapSet, +} from '../src/commons.js'; +import { nativeGetGlobalThis } from './commons.js'; +import { + makeCompartmentConstructor, + compartmentOptions, +} from '../src/compartment.js'; +import { getGlobalIntrinsics } from '../src/intrinsics.js'; +import { tameFunctionToString } from '../src/tame-function-tostring.js'; +import { chooseReporter } from '../src/reporting.js'; +import { makeError } from '../src/error/assert.js'; + +const muteReporter = chooseReporter('none'); + +export const ShimStartCompartment = makeCompartmentConstructor( + makeCompartmentConstructor, + getGlobalIntrinsics(globalThis, muteReporter), + tameFunctionToString(), +); + +export const shimCompartmentPrototype = ShimStartCompartment.prototype; + +export const shimEvaluate = uncurryThis(shimCompartmentPrototype.evaluate); +export const shimImport = uncurryThis(shimCompartmentPrototype.import); +export const shimImportNow = uncurryThis(shimCompartmentPrototype.importNow); + +/** @type {(compartment: typeof Compartment) => typeof globalThis} */ +export const shimGetGlobalThis = uncurryThis( + // @ts-expect-error The descriptor will never be undefined. + getOwnPropertyDescriptor(ShimStartCompartment.prototype, 'globalThis').get, +); + +/** + * @typedef {{ + * name: string, + * transforms: Array<(source: string) => string>, + * delegateNative: boolean, + * noNamespaceBox: boolean, + * nativeEval: typeof eval, + * descriptors: Map, + * }} PrivateFields + */ + +/** @type {WeakMap} */ +export const privateFields = new WeakMap(); + +const adaptVirtualModuleSource = ({ + execute, + imports = [], + exports = [], + reexports = [], +}) => { + const resolutions = create(null); + let i = 0; + return { + execute(environment) { + const fakeCompartment = { + importNow(specifier) { + return environment[specifier]; + }, + }; + execute(environment, fakeCompartment, resolutions); + }, + bindings: [ + ...arrayMap(imports, specifier => { + const resolved = `_${i}`; + i += 1; + resolutions[specifier] = resolved; + return { importAllFrom: specifier, as: resolved }; + }), + ...arrayMap(reexports, specifier => ({ exportAllFrom: specifier })), + ...arrayMap(exports, name => ({ export: name })), + ], + }; +}; + +const adaptModuleSource = source => { + if (source.execute) { + return adaptVirtualModuleSource(source); + } + // eslint-disable-next-line no-underscore-dangle + if (source.__syncModuleProgram__) { + throw makeError( + 'XS native compartments do not support precompiled module sources', + SyntaxError, + ); + } + return source; +}; + +const adaptModuleDescriptor = ( + descriptor, + specifier, + compartment = undefined, +) => { + if (Object(descriptor) !== descriptor) { + throw makeError('Module descriptor must be an object', TypeError); + } + if (descriptor.namespace !== undefined) { + return descriptor; + } + if (descriptor.source !== undefined) { + return { + source: adaptModuleSource(descriptor.source), + importMeta: descriptor.importMeta, + specifier: descriptor.specifier, + }; + } + // Legacy support for record descriptors. + if (descriptor.record !== undefined) { + if ( + descriptor.specifier === specifier || + descriptor.specifier === undefined + ) { + return { + source: adaptModuleSource(descriptor.record), + specifier, + importMeta: descriptor.importMeta, + }; + } else { + if (compartment === undefined) { + throw makeError( + 'Cannot construct forward reference module descriptor in module map', + TypeError, + ); + } + const compartmentPrivateFields = weakmapGet(privateFields, compartment); + if (compartmentPrivateFields === undefined) { + throw makeError( + 'Module descriptor compartment is not a recognizable compartment', + TypeError, + ); + } + const { descriptors } = compartmentPrivateFields; + mapSet(descriptors, descriptor.specifier, { + compartment, + namespace: specifier, + }); + return { + source: adaptModuleSource(descriptor.record), + specifier: descriptor.specifier, + importMeta: descriptor.importMeta, + }; + } + } + if (descriptor.specifier !== undefined) { + return { + namespace: descriptor.specifier, + compartment: descriptor.compartment, + }; + } + // Legacy support for a source in the place of a descriptor. + return { source: adaptModuleSource(descriptor) }; +}; + +export const adaptCompartmentConstructors = ( + NativeCompartment, + ShimCompartment, + maybeHarden, +) => { + function Compartment(...args) { + const options = compartmentOptions(...args); + + const { + name = undefined, + globals = {}, + transforms = [], + resolveHook = () => { + throw makeError('Compartment requires a resolveHook', TypeError); + }, + loadHook = undefined, + loadNowHook = undefined, + importHook = loadHook, + importNowHook = loadNowHook, + moduleMapHook = () => {}, + __native__: delegateNative = false, + __noNamespaceBox__: noNamespaceBox = false, + } = options; + + const modules = delegateNative + ? fromEntries( + arrayMap( + entries(options.modules ?? {}), + // Uses desctructuring to avoid invoking iterator protocol and deny + // vetted shims the opportunity to interfere. + ({ 0: specifier, 1: descriptor }) => [ + specifier, + adaptModuleDescriptor(descriptor, specifier, undefined), + ], + ), + ) + : {}; + + // side table for references from one descriptor to another + const descriptors = new Map(); + + let nativeOptions = { globals, modules }; + + if (importHook) { + /** @param {string} specifier */ + const nativeImportHook = async specifier => { + await null; + let descriptor = + mapGet(descriptors, specifier) ?? + moduleMapHook(specifier) ?? + (await importHook(specifier)); + mapDelete(descriptors, specifier); + // eslint-disable-next-line no-use-before-define + descriptor = adaptModuleDescriptor(descriptor, specifier, compartment); + return descriptor; + }; + nativeOptions = { + ...nativeOptions, + resolveHook, + importHook: nativeImportHook, + loadHook: nativeImportHook, + }; + } + + if (importNowHook) { + /** @param {string} specifier */ + const nativeImportNowHook = specifier => { + let descriptor = + mapGet(descriptors, specifier) ?? + moduleMapHook(specifier) ?? + importNowHook(specifier); + mapDelete(descriptors, specifier); + // eslint-disable-next-line no-use-before-define + descriptor = adaptModuleDescriptor(descriptor, specifier, compartment); + return descriptor; + }; + nativeOptions = { + ...nativeOptions, + resolveHook, + importNowHook: nativeImportNowHook, + loadNowHook: nativeImportNowHook, + }; + } + + const compartment = new NativeCompartment(nativeOptions); + + const nativeGlobalThis = nativeGetGlobalThis(compartment); + + const nativeEval = nativeGlobalThis.eval; + + weakmapSet(privateFields, compartment, { + name, + transforms, + delegateNative, + noNamespaceBox, + nativeEval, + descriptors, + }); + + const shimOptions = { + ...options, + __noNamespaceBox__: true, + __options__: true, + }; + + uncurryThis(ShimCompartment)(compartment, shimOptions); + + const shimGlobalThis = shimGetGlobalThis(compartment); + + const ChildCompartment = adaptCompartmentConstructors( + // @ts-expect-error incomplete type information for XS + nativeGlobalThis.Compartment, + // @ts-expect-error incomplete type information for XS + shimGlobalThis.Compartment, + maybeHarden, + ); + + defineProperty(nativeGlobalThis, 'Compartment', { + value: ChildCompartment, + writable: true, + configurable: true, + enumerable: false, + }); + + defineProperty(shimGlobalThis, 'Compartment', { + value: ChildCompartment, + writable: true, + configurable: true, + enumerable: false, + }); + + maybeHarden(compartment); + return compartment; + } + + maybeHarden(Compartment); + return Compartment; +}; diff --git a/packages/ses/src-xs/index.js b/packages/ses/src-xs/index.js new file mode 100644 index 0000000000..a762a52782 --- /dev/null +++ b/packages/ses/src-xs/index.js @@ -0,0 +1,27 @@ +/** + * @module This is an alternate implementation of ../index.js to provide + * access to the native implementation of Hardened JavaScript on the XS + * engine, but adapted for backward compatibility with SES. + * This module can only be reached in the presence of the package export/import + * condition "xs", and should only be used to bundle an XS-specific version of + * SES. + */ +// @ts-nocheck +/// + +import { Object, freeze } from '../src/commons.js'; + +// These are the constituent shims in an arbitrary order, but matched +// to ../index.js to remove doubt. +import './lockdown-shim.js'; +import './compartment-shim.js'; +import '../src/assert-shim.js'; +import '../src/console-shim.js'; + +// XS Object.freeze takes a second argument to apply freeze transitively, but +// with slightly different effects than `harden`. +// We disable this behavior to encourage use of `harden` for portable Hardened +// JavaScript. +// The pattern of creating and extracting preserves Object.freeze.name. +/** @param {object} object */ +Object.freeze = { freeze: object => freeze(object) }.freeze; diff --git a/packages/ses/src-xs/lockdown-shim.js b/packages/ses/src-xs/lockdown-shim.js new file mode 100644 index 0000000000..7cc6cca734 --- /dev/null +++ b/packages/ses/src-xs/lockdown-shim.js @@ -0,0 +1,29 @@ +/** + * @module Alters the XS implementation of Lockdown to be backward compatible + * with SES, providing Compartment constructors in every Compartment that can + * be used with either native ModuleSources or module sources pre-compiled for + * the SES Compartment, depending on the __native__ Compartment constructor + * option. + */ +import { globalThis } from '../src/commons.js'; +import { NativeStartCompartment } from './commons.js'; +import { repairIntrinsics } from '../src/lockdown.js'; +import { + ShimStartCompartment, + adaptCompartmentConstructors, +} from './compartment.js'; + +const lockdown = options => { + const hardenIntrinsics = repairIntrinsics(options); + hardenIntrinsics(); + // Replace global Compartment with a version that is hardened and hardens + // transitive child Compartment. + // @ts-expect-error Incomplete global type on XS. + globalThis.Compartment = adaptCompartmentConstructors( + NativeStartCompartment, + ShimStartCompartment, + harden, + ); +}; + +globalThis.lockdown = lockdown; diff --git a/packages/ses/src/commons.js b/packages/ses/src/commons.js index b4f8f6f391..5925c17eae 100644 --- a/packages/ses/src/commons.js +++ b/packages/ses/src/commons.js @@ -1,17 +1,19 @@ -/* global globalThis */ -/* eslint-disable no-restricted-globals */ - /** - * commons.js - * Declare shorthand functions. Sharing these declarations across modules - * improves on consistency and minification. Unused declarations are - * dropped by the tree shaking process. + * @module Captures native intrinsics during initialization, so vetted shims + * (running between initialization of SES and calling lockdown) are free to + * modify the environment without compromising the integrity of SES. For + * example, a vetted shim can modify Object.assign because we capture and + * export Object and assign here, then never again consult Object to get its + * assign property. * - * We capture these, not just for brevity, but for security. If any code - * modifies Object to change what 'assign' points to, the Compartment shim - * would be corrupted. + * This pattern of use is enforced by eslint rules no-restricted-globals and + * no-polymorphic-call. + * We maintain the list of restricted globals in ../package.json. */ +/* global globalThis */ +/* eslint-disable no-restricted-globals */ + // We cannot use globalThis as the local name since it would capture the // lexical name. const universalThis = globalThis; @@ -305,6 +307,12 @@ export const isObject = value => Object(value) === value; */ export const isError = value => value instanceof FERAL_ERROR; +/** + * @template T + * @param {T} x + */ +export const identity = x => x; + // The original unsafe untamed eval function, which must not escape. // Sample at module initialization time, which is before lockdown can // repair it. Use it only to build powerless abstractions. diff --git a/packages/ses/src/compartment-shim.js b/packages/ses/src/compartment-shim.js index ee3ddface4..5708c6f3dc 100644 --- a/packages/ses/src/compartment-shim.js +++ b/packages/ses/src/compartment-shim.js @@ -16,4 +16,7 @@ globalThis.Compartment = makeCompartmentConstructor( // See https://github.com/endojs/endo/pull/2624#discussion_r1840979770 getGlobalIntrinsics(globalThis, muteReporter), markVirtualizedNativeFunction, + { + enforceNew: true, + }, ); diff --git a/packages/ses/src/compartment.js b/packages/ses/src/compartment.js index 9860f935e0..c72bc6dde9 100644 --- a/packages/ses/src/compartment.js +++ b/packages/ses/src/compartment.js @@ -1,3 +1,40 @@ +/** + * @module Provides the mechanism to create a Compartment constructor that + * can provide either shim-specific or native XS features depending on + * the __native__ constructor option. + * This is necessary because a native Compartment can handle native ModuleSource + * but cannot handle shim-specific pre-compiled ModuleSources like the JSON + * representation of a module that Compartment Mapper can put in bundles. + * Pre-compiling ModuleSource during bundling helps avoid paying the cost + * of importing Babel and transforming ESM syntax to a form that can be + * confined by the shim, which is prohibitively expensive for a web runtime + * and for XS _without this adapter_. + * + * Since any invocation of the Compartment constructor may occur standing + * on a native-flavor or shim-flavor compartment, we create parallel compartment + * constructor trees for compartments created with the Compartment constructor + * of a specific compartment. + * + * A compartment's importHook, importNowHook, moduleMapHook, and the modules + * map itself may provide module descriptors that address another compartment, + * using a compartment instance as a token indicating the compartment the + * module should be loaded or initialized in. + * Consequently, the compartment instance must be a suitable token for the + * underlying native-flavor or shim-flavor compartment. + * We are not in a position to fidddle with the native compartments behavior, + * so adapted compartments use the identity of the native compartment. + * We replace all of the methods of the native compartment prototype with + * thunks that choose behavior based on whether the compartment was + * constructed with the __native__ option. + * The SES shim associates a compartment with its private fields using a weak + * map exported by ../src/compartment.js and held closely by ses by the + * enforcement of explicit exports in package.json, since Node.js 12.11.0. + * + * Evaluating ./compartment.js does not have global side-effects. + * We defer modification of the global environment until the evaluation + * of ./compartment-shim.js. + */ + // @ts-check /* eslint-disable no-underscore-dangle */ /// @@ -6,8 +43,10 @@ import { Map, TypeError, WeakMap, + arrayFlatMap, assign, defineProperties, + identity, promiseThen, toStringTagSymbol, weakmapGet, @@ -171,7 +210,9 @@ defineProperties(InertCompartment, { * @param {MakeCompartmentConstructor} targetMakeCompartmentConstructor * @param {Record} intrinsics * @param {(object: object) => void} markVirtualizedNativeFunction - * @param {Compartment} [parentCompartment] + * @param {object} [options] + * @param {Compartment} [options.parentCompartment] + * @param {boolean} [options.enforceNew] * @returns {Compartment['constructor']} */ @@ -184,7 +225,7 @@ defineProperties(InertCompartment, { // positional arguments, this function detects the temporary sigil __options__ // on the first argument and coerces compartments arguments into a single // compartments object. -const compartmentOptions = (...args) => { +export const compartmentOptions = (...args) => { if (args.length === 0) { return {}; } @@ -229,10 +270,10 @@ export const makeCompartmentConstructor = ( targetMakeCompartmentConstructor, intrinsics, markVirtualizedNativeFunction, - parentCompartment = undefined, + { parentCompartment = undefined, enforceNew = false } = {}, ) => { function Compartment(...args) { - if (new.target === undefined) { + if (enforceNew && new.target === undefined) { throw TypeError( "Class constructor Compartment cannot be invoked without 'new'", ); @@ -252,7 +293,10 @@ export const makeCompartmentConstructor = ( importMetaHook, __noNamespaceBox__: noNamespaceBox = false, } = compartmentOptions(...args); - const globalTransforms = [...transforms, ...__shimTransforms__]; + const globalTransforms = arrayFlatMap( + [transforms, __shimTransforms__], + identity, + ); const endowments = { __proto__: null, ...endowmentsOption }; const moduleMap = { __proto__: null, ...moduleMapOption }; @@ -319,7 +363,7 @@ export const makeCompartmentConstructor = ( */ const compartmentImport = async fullSpecifier => { if (typeof resolveHook !== 'function') { - throw new TypeError( + throw TypeError( `Compartment does not support dynamic import: no configured resolveHook for compartment ${q(name)}`, ); } diff --git a/packages/ses/src/global-object.js b/packages/ses/src/global-object.js index 007399fa5c..1b2c3c038d 100644 --- a/packages/ses/src/global-object.js +++ b/packages/ses/src/global-object.js @@ -117,7 +117,10 @@ export const setGlobalObjectMutableProperties = ( makeCompartmentConstructor, intrinsics, markVirtualizedNativeFunction, - parentCompartment, + { + parentCompartment, + enforceNew: true, + }, ), ); diff --git a/packages/ses/src/make-safe-evaluator.js b/packages/ses/src/make-safe-evaluator.js index 78ce9e1a94..9f7085849b 100644 --- a/packages/ses/src/make-safe-evaluator.js +++ b/packages/ses/src/make-safe-evaluator.js @@ -1,7 +1,7 @@ // Portions adapted from V8 - Copyright 2016 the V8 project authors. // https://github.com/v8/v8/blob/master/src/builtins/builtins-function.cc -import { apply, freeze } from './commons.js'; +import { apply, arrayFlatMap, freeze, identity } from './commons.js'; import { strictScopeTerminator } from './strict-scope-terminator.js'; import { createSloppyGlobalsScopeTerminator } from './sloppy-globals-scope-terminator.js'; import { makeEvalScopeKit } from './eval-scope.js'; @@ -62,11 +62,13 @@ export const makeSafeEvaluator = ({ // Execute the mandatory transforms last to ensure that any rewritten code // meets those mandatory requirements. - source = applyTransforms(source, [ - ...localTransforms, - ...globalTransforms, - mandatoryTransforms, - ]); + source = applyTransforms( + source, + arrayFlatMap( + [localTransforms, globalTransforms, [mandatoryTransforms]], + identity, + ), + ); let err; try { diff --git a/packages/ses/src/transforms.js b/packages/ses/src/transforms.js index 64a46cb533..4847e7d0e5 100644 --- a/packages/ses/src/transforms.js +++ b/packages/ses/src/transforms.js @@ -248,7 +248,8 @@ export const mandatoryTransforms = source => { * @returns {string} */ export const applyTransforms = (source, transforms) => { - for (const transform of transforms) { + for (let i = 0, l = transforms.length; i < l; i += 1) { + const transform = transforms[i]; source = transform(source); } return source; diff --git a/packages/ses/test/_meaning.js b/packages/ses/test/_meaning.js new file mode 100644 index 0000000000..7a4e8a723a --- /dev/null +++ b/packages/ses/test/_meaning.js @@ -0,0 +1 @@ +export default 42; diff --git a/packages/ses/test/_xs.js b/packages/ses/test/_xs.js new file mode 100644 index 0000000000..64f0b14e3c --- /dev/null +++ b/packages/ses/test/_xs.js @@ -0,0 +1,226 @@ +// This is a test fixture for minimal spot checks of the XS-specific variant of +// SES. +// The script ../scripts/generate-test-xs.js generates the _meaning.pre-mjs.json +// module by precompiling _meaning.js, then bundles this module with the "xs" +// package export/import condition so that it entrains ../src-xs/shim.js instead +// of the ordinary SES shim. +// This generates ../tmp/test-xs.js, which can be run with xst directly for +// validation of the XS environment under SES-for-XS. + +/* global print */ + +// Eslint does not know about package reflexive imports (importing your own +// package), which in this case is necessary to go through the conditional +// export in package.json. +// eslint-disable-next-line import/no-extraneous-dependencies +import 'ses'; + +// The dependency below is generated by ../scripts/generate-test-xs.js +// eslint-disable-next-line import/no-unresolved +import precompiledModuleSource from '../tmp/_meaning.pre-mjs.json'; + +lockdown(); + +// spot checks +assert(Object.isFrozen(Object)); + +print('# shim compartment can import a shim precompiled module source'); +{ + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + assert.equal( + shimCompartment.importNow('.').default, + 42, + 'can import precompiled module source', + ); +} + +print('# native compartment can import a native ModuleSource'); +{ + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + + assert( + nativeCompartment.importNow('.').default === 42, + 'can import native module source', + ); +} + +print('# shim compartment cannot import a native ModuleSource'); +// fail to import a native module source in a shim compartment +{ + let threw = null; + try { + new Compartment({ + __options__: true, + modules: { + '.': { + source: new ModuleSource(''), + }, + }, + }).importNow('.'); + } catch (error) { + threw = error; + } + assert( + threw, + 'attempting to import a native module source on a shim compartment should fail', + ); +} + +print('# native compartment cannot import a shim precompiled module source'); +{ + let threw = null; + try { + new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }).importNow('.'); + } catch (error) { + threw = error; + } + assert( + threw, + 'attempting to import a precompiled module source in a native compartment should fail', + ); +} + +print('# shim compartment can link to another shim compartment'); +{ + const shimCompartment1 = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + const shimCompartment2 = new Compartment({ + __options__: true, + modules: { + '.': { + compartment: shimCompartment1, + namespace: '.', + }, + }, + }); + assert.equal( + shimCompartment2.importNow('.').default, + 42, + 'can link shim compartments', + ); +} + +print('# native compartment can link to another native compartment'); +{ + const nativeCompartment1 = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + const nativeCompartment2 = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + compartment: nativeCompartment1, + namespace: '.', + }, + }, + }); + assert.equal( + nativeCompartment2.importNow('.').default, + 42, + 'can link native compartments', + ); +} + +print('# shim compartment cannot link a native compartment'); +{ + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + compartment: nativeCompartment, + namespace: '.', + }, + }, + }); + let threw = null; + try { + shimCompartment.importNow('.'); + } catch (error) { + threw = error; + } + assert(threw, 'cannot link native from shim compartment'); +} + +print('# native compartment cannot link shim compartment'); +{ + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + compartment: shimCompartment, + namespace: '.', + }, + }, + }); + let threw = null; + try { + nativeCompartment.importNow('.'); + } catch (error) { + threw = error; + } + assert(threw, 'cannot link shim from native compartment'); +} + +print('ok'); + +// To be continued in hardened262... diff --git a/typedoc.json b/typedoc.json index 5515034bf1..ddbad8f65a 100644 --- a/typedoc.json +++ b/typedoc.json @@ -3,11 +3,13 @@ "packages/*" ], "exclude": [ + "**/tmp/**", "packages/cjs-module-analyzer", "packages/cli", "packages/compartment-mapper", "packages/daemon", "packages/evasive-transform", + "packages/hardened262", "packages/lp32", "packages/memoize", "packages/module-source", From c5f5d433b1df60ef0f74b1a5458675a4e37806e0 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Fri, 11 Oct 2024 10:29:03 -0700 Subject: [PATCH 2/4] feat(compartment-mapper): Export all parsers --- packages/compartment-mapper/NEWS.md | 21 ++++++++++++++ .../import-archive-all-parsers.js | 1 + packages/compartment-mapper/package.json | 7 ++++- .../src/import-archive-all-parsers.js | 29 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 packages/compartment-mapper/import-archive-all-parsers.js create mode 100644 packages/compartment-mapper/src/import-archive-all-parsers.js diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index 5124764413..baac250180 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -15,6 +15,27 @@ User-visible changes to `@endo/compartment-mapper`: Correct interpretation of `peerDependencies` is not distributed evenly, so this behavior is no longer the default. +Experimental: + +- The module `@endo/compartment-mapper/import-archive-parsers.js` does not + support modules in archives in their original ESM (`mjs`) or CommonJS (`cjs`) + formats because they entrain Babel and a full JavaScript lexer that are + not suitable for use in all environments, specifically XS. + This version introduces an elective + `@endo/compartment-mapper/import-archive-all-parsers.js` that has all of the + precompiled module parsers (`pre-cjs-json` and `pre-mjs-json`) that Endo's + bundler currently produces by default and additionally parsers for original + sources (`mjs`, `cjs`). + Also, provided the `xs` package condition, + `@endo/compartment-mapper/import-archive-parsers.js` now falls through to the + native `ModuleSource` and safely includes `mjs` and `cjs` without entraining + Babel, but is only supported in conjunction with the `__native__` option + for `Compartment`, `importArchive`, `parseArchive`, and `importBundle`. + With the `node` package condition (present by default when running ESM on + `node`), `@endo/compartment-mapper/import-archive-parsers.js` also now + includes `mjs` and `cjs` by entraining Babel, which performs adequately on + that platform. + # v1.4.0 (2024-11-13) - Adds options `languageForExtension`, `moduleLanguageForExtension`, diff --git a/packages/compartment-mapper/import-archive-all-parsers.js b/packages/compartment-mapper/import-archive-all-parsers.js new file mode 100644 index 0000000000..f31eb40d75 --- /dev/null +++ b/packages/compartment-mapper/import-archive-all-parsers.js @@ -0,0 +1 @@ +export { defaultParserForLanguage } from './src/import-archive-all-parsers.js'; diff --git a/packages/compartment-mapper/package.json b/packages/compartment-mapper/package.json index 3cb623d0b6..60de4e8416 100644 --- a/packages/compartment-mapper/package.json +++ b/packages/compartment-mapper/package.json @@ -32,7 +32,12 @@ "./capture-lite.js": "./capture-lite.js", "./import-archive.js": "./import-archive.js", "./import-archive-lite.js": "./import-archive-lite.js", - "./import-archive-parsers.js": "./import-archive-parsers.js", + "./import-archive-parsers.js": { + "xs": "./import-archive-all-parsers.js", + "node": "./import-archive-all-parsers.js", + "default": "./import-archive-parsers.js" + }, + "./import-archive-all-parsers.js": "./import-archive-all-parsers.js", "./bundle.js": "./bundle.js", "./node-powers.js": "./node-powers.js", "./node-modules.js": "./node-modules.js", diff --git a/packages/compartment-mapper/src/import-archive-all-parsers.js b/packages/compartment-mapper/src/import-archive-all-parsers.js new file mode 100644 index 0000000000..336084d007 --- /dev/null +++ b/packages/compartment-mapper/src/import-archive-all-parsers.js @@ -0,0 +1,29 @@ +/* Provides a set of default language behaviors (parsers) suitable for + * evaluating archives (zip files with a `compartment-map.json` and a file for + * each module) with pre-compiled _or_ original ESM and CommonJS. + * + * This module does not entrain a dependency on Babel on XS, but does on other + * platforms like Node.js. + */ +/** @import {ParserForLanguage} from './types.js' */ + +import parserPreCjs from './parse-pre-cjs.js'; +import parserJson from './parse-json.js'; +import parserText from './parse-text.js'; +import parserBytes from './parse-bytes.js'; +import parserPreMjs from './parse-pre-mjs.js'; +import parserMjs from './parse-mjs.js'; +import parserCjs from './parse-cjs.js'; + +/** @satisfies {Readonly} */ +export const defaultParserForLanguage = Object.freeze( + /** @type {const} */ ({ + 'pre-cjs-json': parserPreCjs, + 'pre-mjs-json': parserPreMjs, + cjs: parserCjs, + mjs: parserMjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); From aa1e77f9af62dca40a357e92fd869ecdb8a35379 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Tue, 8 Oct 2024 22:52:59 -0700 Subject: [PATCH 3/4] feat(compartment-mapper): Thread native flag to opt-in for native XS runtime modules --- packages/compartment-mapper/NEWS.md | 7 +++++++ packages/compartment-mapper/README.md | 14 ++++++++++++++ .../compartment-mapper/src/import-archive-lite.js | 4 ++++ packages/compartment-mapper/src/import-lite.js | 3 +++ packages/compartment-mapper/src/link.js | 2 ++ packages/compartment-mapper/src/types/external.ts | 2 ++ packages/compartment-mapper/src/types/internal.ts | 1 + 7 files changed, 33 insertions(+) diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index baac250180..40d7389eaf 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -35,6 +35,13 @@ Experimental: `node`), `@endo/compartment-mapper/import-archive-parsers.js` also now includes `mjs` and `cjs` by entraining Babel, which performs adequately on that platform. +- Adds a `__native__: true` option to all paths to import, that indicates that + the application will fall through to the native implementation of + Compartment, currently only available on XS, which lacks support for + precompiled module sources (as exist in many archived applications, + particularly Agoric smart contract bundles) and instead supports loading + modules from original sources (which is not possible at runtime on XS). + # v1.4.0 (2024-11-13) diff --git a/packages/compartment-mapper/README.md b/packages/compartment-mapper/README.md index 6cfbdfb6b9..7a7fd0ad19 100644 --- a/packages/compartment-mapper/README.md +++ b/packages/compartment-mapper/README.md @@ -328,6 +328,20 @@ These will be appended to each module from the archive, for debugging purposes. The `@endo/bundle-source` and `@endo/import-bundle` tools integrate source maps for an end-to-end debugging experience. +# XS (experimental) + +The Compartment Mapper can use native XS `Compartment` and `ModuleSource` under +certain conditions: + +1. The application must be an XS script that was compiled with the `xs` + package condition. + This causes `ses`, `@endo/module-source`, and `@endo/import-bundle` to + provide slightly different implementations that can fall through to native + behavior. +2. The application must opt-in with the `__native__: true` option on any + of the compartment mapper methods that import modules like `importLocation` + and `importArchive`. + # Design Each of the workflows the compartment mapper executes a portion of one sequence diff --git a/packages/compartment-mapper/src/import-archive-lite.js b/packages/compartment-mapper/src/import-archive-lite.js index 5c50c41358..a0728c6dcc 100644 --- a/packages/compartment-mapper/src/import-archive-lite.js +++ b/packages/compartment-mapper/src/import-archive-lite.js @@ -257,6 +257,7 @@ export const parseArchive = async ( modules = undefined, importHook: exitModuleImportHook = undefined, parserForLanguage: parserForLanguageOption = {}, + __native__ = false, } = options; const parserForLanguage = freeze( @@ -343,6 +344,7 @@ export const parseArchive = async ( }), ), Compartment: CompartmentParseOption, + __native__, }); await pendingJobsPromise; @@ -362,6 +364,7 @@ export const parseArchive = async ( transforms, __shimTransforms__, Compartment: CompartmentOption = CompartmentParseOption, + __native__, importHook: exitModuleImportHook, } = options || {}; @@ -388,6 +391,7 @@ export const parseArchive = async ( transforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, }); await pendingJobsPromise; diff --git a/packages/compartment-mapper/src/import-lite.js b/packages/compartment-mapper/src/import-lite.js index 71d748f992..51c4a0e06b 100644 --- a/packages/compartment-mapper/src/import-lite.js +++ b/packages/compartment-mapper/src/import-lite.js @@ -152,6 +152,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { transforms, __shimTransforms__, Compartment: CompartmentOption = LoadCompartmentOption, + __native__, importHook: exitModuleImportHook, } = options; const compartmentExitModuleImportHook = exitModuleImportHookMaker({ @@ -201,6 +202,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { syncModuleTransforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, })); } else { // sync module transforms are allowed, because they are "compatible" @@ -215,6 +217,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { syncModuleTransforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, })); } diff --git a/packages/compartment-mapper/src/link.js b/packages/compartment-mapper/src/link.js index b2b1414f5d..b2f70f9652 100644 --- a/packages/compartment-mapper/src/link.js +++ b/packages/compartment-mapper/src/link.js @@ -258,6 +258,7 @@ export const link = ( moduleTransforms, syncModuleTransforms, __shimTransforms__ = [], + __native__ = false, archiveOnly = false, Compartment = defaultCompartment, } = options; @@ -362,6 +363,7 @@ export const link = ( transforms, __shimTransforms__, __options__: true, + __native__, }); if (!archiveOnly) { diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index 7216547568..c4568d11e3 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -29,6 +29,7 @@ export type ExecuteOptions = Partial<{ __shimTransforms__: Array; attenuations: Record; Compartment: typeof Compartment; + __native__: boolean; }> & ModulesOption & ExitModuleImportHookOption; @@ -38,6 +39,7 @@ export type ParseArchiveOptions = Partial<{ computeSha512: HashFn; computeSourceLocation: ComputeSourceLocationHook; computeSourceMapLocation: ComputeSourceMapLocationHook; + __native__: boolean; }> & ModulesOption & CompartmentOption & diff --git a/packages/compartment-mapper/src/types/internal.ts b/packages/compartment-mapper/src/types/internal.ts index 8cce0cd5f4..bb6c6c26f3 100644 --- a/packages/compartment-mapper/src/types/internal.ts +++ b/packages/compartment-mapper/src/types/internal.ts @@ -44,6 +44,7 @@ export type LinkOptions = { moduleTransforms?: ModuleTransforms; syncModuleTransforms?: SyncModuleTransforms; archiveOnly?: boolean; + __native__?: boolean; } & ExecuteOptions; export type LinkResult = { From bd79347776ce1baa0db41bc5509101001bc60d60 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Tue, 29 Oct 2024 12:00:44 -0700 Subject: [PATCH 4/4] chore(compartment-mapper): Improve makeBundle diagnostics --- packages/compartment-mapper/src/bundle.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/compartment-mapper/src/bundle.js b/packages/compartment-mapper/src/bundle.js index fe63135f1b..bea4f6a657 100644 --- a/packages/compartment-mapper/src/bundle.js +++ b/packages/compartment-mapper/src/bundle.js @@ -117,8 +117,14 @@ const sortedModules = ( const source = compartmentSources[compartmentName][moduleSpecifier]; if (source !== undefined) { const { record, parser, deferredError, bytes } = source; - assert(parser !== undefined); - assert(bytes !== undefined); + assert( + bytes !== undefined, + `No bytes for ${moduleSpecifier} in ${compartmentName}`, + ); + assert( + parser !== undefined, + `No parser for ${moduleSpecifier} in ${compartmentName}`, + ); if (deferredError) { throw Error( `Cannot bundle: encountered deferredError ${deferredError}`,