From bba941ab4dfba12037d93d7f3dd7bdc7f6eacf67 Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Wed, 4 Dec 2024 10:05:43 -0500 Subject: [PATCH] fix(core): move resolving plugins back to main thread (#29176) ## Current Behavior Resolving custom plugins to their paths to be loaded involves loading our default plugins. When plugin isolation, this loads the default plugins for each and every plugin worker. ## Expected Behavior Resolution of the plugins is the same and easily serializable to send to the worker. Resolving plugins on the main thread allows Nx to reuse the default plugins and decrease the memory usage greatly. ## Related Issue(s) Fixes # --- .../src/project-graph/plugins/get-plugins.ts | 19 +++-- .../plugins/isolation/messaging.ts | 3 + .../plugins/isolation/plugin-pool.ts | 8 ++- .../plugins/isolation/plugin-worker.ts | 26 +++++-- .../plugins/load-resolved-plugin.ts | 26 +++++++ .../nx/src/project-graph/plugins/loader.ts | 72 +++++++++---------- 6 files changed, 104 insertions(+), 50 deletions(-) create mode 100644 packages/nx/src/project-graph/plugins/load-resolved-plugin.ts diff --git a/packages/nx/src/project-graph/plugins/get-plugins.ts b/packages/nx/src/project-graph/plugins/get-plugins.ts index 83489795852a3..1a245d318d598 100644 --- a/packages/nx/src/project-graph/plugins/get-plugins.ts +++ b/packages/nx/src/project-graph/plugins/get-plugins.ts @@ -5,6 +5,9 @@ import { workspaceRoot } from '../../utils/workspace-root'; let currentPluginsConfigurationHash: string; let loadedPlugins: LoadedNxPlugin[]; +let pendingPluginsPromise: + | Promise void]> + | undefined; let cleanup: () => void; export async function getPlugins() { @@ -24,11 +27,10 @@ export async function getPlugins() { cleanup(); } + pendingPluginsPromise ??= loadNxPlugins(pluginsConfiguration, workspaceRoot); + currentPluginsConfigurationHash = pluginsConfigurationHash; - const [result, cleanupFn] = await loadNxPlugins( - pluginsConfiguration, - workspaceRoot - ); + const [result, cleanupFn] = await pendingPluginsPromise; cleanup = cleanupFn; loadedPlugins = result; return result; @@ -36,6 +38,9 @@ export async function getPlugins() { let loadedDefaultPlugins: LoadedNxPlugin[]; let cleanupDefaultPlugins: () => void; +let pendingDefaultPluginPromise: + | Promise void]> + | undefined; export async function getOnlyDefaultPlugins() { // If the plugins configuration has not changed, reuse the current plugins @@ -48,13 +53,17 @@ export async function getOnlyDefaultPlugins() { cleanupDefaultPlugins(); } - const [result, cleanupFn] = await loadNxPlugins([], workspaceRoot); + pendingDefaultPluginPromise ??= loadNxPlugins([], workspaceRoot); + + const [result, cleanupFn] = await pendingDefaultPluginPromise; cleanupDefaultPlugins = cleanupFn; loadedPlugins = result; return result; } export function cleanupPlugins() { + pendingPluginsPromise = undefined; + pendingDefaultPluginPromise = undefined; cleanup(); cleanupDefaultPlugins(); } diff --git a/packages/nx/src/project-graph/plugins/isolation/messaging.ts b/packages/nx/src/project-graph/plugins/isolation/messaging.ts index 950957aab51ae..f8e6e69a4378f 100644 --- a/packages/nx/src/project-graph/plugins/isolation/messaging.ts +++ b/packages/nx/src/project-graph/plugins/isolation/messaging.ts @@ -14,6 +14,9 @@ export interface PluginWorkerLoadMessage { payload: { plugin: PluginConfiguration; root: string; + name: string; + pluginPath: string; + shouldRegisterTSTranspiler: boolean; }; } diff --git a/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts b/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts index 740f8562ac68c..bb912362dd45e 100644 --- a/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts +++ b/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts @@ -16,6 +16,8 @@ import { isPluginWorkerResult, sendMessageOverSocket, } from './messaging'; +import { getNxRequirePaths } from '../../../utils/installation-directory'; +import { resolveNxPlugin } from '../loader'; const cleanupFunctions = new Set<() => void>(); @@ -59,6 +61,10 @@ export async function loadRemoteNxPlugin( if (nxPluginWorkerCache.has(cacheKey)) { return [nxPluginWorkerCache.get(cacheKey), () => {}]; } + const moduleName = typeof plugin === 'string' ? plugin : plugin.plugin; + + const { name, pluginPath, shouldRegisterTSTranspiler } = + await resolveNxPlugin(moduleName, root, getNxRequirePaths(root)); const { worker, socket } = await startPluginWorker(); @@ -77,7 +83,7 @@ export async function loadRemoteNxPlugin( const pluginPromise = new Promise((res, rej) => { sendMessageOverSocket(socket, { type: 'load', - payload: { plugin, root }, + payload: { plugin, root, name, pluginPath, shouldRegisterTSTranspiler }, }); // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`); diff --git a/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts b/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts index 5bc4477a3bfb0..349ab00347916 100644 --- a/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts +++ b/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts @@ -4,6 +4,7 @@ import { consumeMessagesFromSocket } from '../../../utils/consume-messages-from- import { createServer } from 'net'; import { unlinkSync } from 'fs'; +import { registerPluginTSTranspiler } from '../loader'; if (process.env.NX_PERF_LOGGING === 'true') { require('../../../utils/perf-logging'); @@ -35,13 +36,30 @@ const server = createServer((socket) => { return; } return consumeMessage(socket, message, { - load: async ({ plugin: pluginConfiguration, root }) => { + load: async ({ + plugin: pluginConfiguration, + root, + name, + pluginPath, + shouldRegisterTSTranspiler, + }) => { if (loadTimeout) clearTimeout(loadTimeout); process.chdir(root); try { - const { loadNxPlugin } = await import('../loader'); - const [promise] = loadNxPlugin(pluginConfiguration, root); - plugin = await promise; + const { loadResolvedNxPluginAsync } = await import( + '../load-resolved-plugin' + ); + + // Register the ts-transpiler if we are pointing to a + // plain ts file that's not part of a plugin project + if (shouldRegisterTSTranspiler) { + registerPluginTSTranspiler(); + } + plugin = await loadResolvedNxPluginAsync( + pluginConfiguration, + pluginPath, + name + ); return { type: 'load-result', payload: { diff --git a/packages/nx/src/project-graph/plugins/load-resolved-plugin.ts b/packages/nx/src/project-graph/plugins/load-resolved-plugin.ts new file mode 100644 index 0000000000000..a77337581a26f --- /dev/null +++ b/packages/nx/src/project-graph/plugins/load-resolved-plugin.ts @@ -0,0 +1,26 @@ +import type { PluginConfiguration } from '../../config/nx-json'; +import { LoadedNxPlugin } from './internal-api'; +import { NxPlugin } from './public-api'; + +export async function loadResolvedNxPluginAsync( + pluginConfiguration: PluginConfiguration, + pluginPath: string, + name: string +) { + const plugin = await importPluginModule(pluginPath); + plugin.name ??= name; + return new LoadedNxPlugin(plugin, pluginConfiguration); +} + +async function importPluginModule(pluginPath: string): Promise { + const m = await import(pluginPath); + if ( + m.default && + ('createNodes' in m.default || + 'createNodesV2' in m.default || + 'createDependencies' in m.default) + ) { + return m.default; + } + return m; +} diff --git a/packages/nx/src/project-graph/plugins/loader.ts b/packages/nx/src/project-graph/plugins/loader.ts index 05661bcc93525..0e90af76855de 100644 --- a/packages/nx/src/project-graph/plugins/loader.ts +++ b/packages/nx/src/project-graph/plugins/loader.ts @@ -24,13 +24,13 @@ import { logger } from '../../utils/logger'; import type * as ts from 'typescript'; import { extname } from 'node:path'; -import type { NxPlugin } from './public-api'; import type { PluginConfiguration } from '../../config/nx-json'; import { retrieveProjectConfigurationsWithoutPluginInference } from '../utils/retrieve-workspace-files'; import { LoadedNxPlugin } from './internal-api'; import { LoadPluginError } from '../error-types'; import path = require('node:path/posix'); import { readTsConfig } from '../../plugins/js/utils/typescript'; +import { loadResolvedNxPluginAsync } from './load-resolved-plugin'; export function readPluginPackageJson( pluginName: string, @@ -200,18 +200,18 @@ export function getPluginPathAndName( root: string ) { let pluginPath: string; - let registerTSTranspiler = false; + let shouldRegisterTSTranspiler = false; try { pluginPath = require.resolve(moduleName, { paths, }); const extension = path.extname(pluginPath); - registerTSTranspiler = extension === '.ts'; + shouldRegisterTSTranspiler = extension === '.ts'; } catch (e) { if (e.code === 'MODULE_NOT_FOUND') { const plugin = resolveLocalNxPlugin(moduleName, projects, root); if (plugin) { - registerTSTranspiler = true; + shouldRegisterTSTranspiler = true; const main = readPluginMainFromProjectConfiguration( plugin.projectConfig ); @@ -226,18 +226,12 @@ export function getPluginPathAndName( } const packageJsonPath = path.join(pluginPath, 'package.json'); - // Register the ts-transpiler if we are pointing to a - // plain ts file that's not part of a plugin project - if (registerTSTranspiler) { - registerPluginTSTranspiler(); - } - const { name } = !['.ts', '.js'].some((x) => extname(moduleName) === x) && // Not trying to point to a ts or js file existsSync(packageJsonPath) // plugin has a package.json ? readJsonFile(packageJsonPath) // read name from package.json : { name: moduleName }; - return { pluginPath, name }; + return { pluginPath, name, shouldRegisterTSTranspiler }; } let projectsWithoutInference: Record; @@ -249,6 +243,27 @@ export function loadNxPlugin(plugin: PluginConfiguration, root: string) { ] as const; } +export async function resolveNxPlugin( + moduleName: string, + root: string, + paths: string[] +) { + try { + require.resolve(moduleName); + } catch { + // If a plugin cannot be resolved, we will need projects to resolve it + projectsWithoutInference ??= + await retrieveProjectConfigurationsWithoutPluginInference(root); + } + const { pluginPath, name, shouldRegisterTSTranspiler } = getPluginPathAndName( + moduleName, + paths, + projectsWithoutInference, + root + ); + return { pluginPath, name, shouldRegisterTSTranspiler }; +} + export async function loadNxPluginAsync( pluginConfiguration: PluginConfiguration, paths: string[], @@ -259,37 +274,14 @@ export async function loadNxPluginAsync( ? pluginConfiguration : pluginConfiguration.plugin; try { - try { - require.resolve(moduleName); - } catch { - // If a plugin cannot be resolved, we will need projects to resolve it - projectsWithoutInference ??= - await retrieveProjectConfigurationsWithoutPluginInference(root); - } - const { pluginPath, name } = getPluginPathAndName( - moduleName, - paths, - projectsWithoutInference, - root - ); - const plugin = await importPluginModule(pluginPath); - plugin.name ??= name; + const { pluginPath, name, shouldRegisterTSTranspiler } = + await resolveNxPlugin(moduleName, root, paths); - return new LoadedNxPlugin(plugin, pluginConfiguration); + if (shouldRegisterTSTranspiler) { + registerPluginTSTranspiler(); + } + return loadResolvedNxPluginAsync(pluginConfiguration, pluginPath, name); } catch (e) { throw new LoadPluginError(moduleName, e); } } - -async function importPluginModule(pluginPath: string): Promise { - const m = await import(pluginPath); - if ( - m.default && - ('createNodes' in m.default || - 'createNodesV2' in m.default || - 'createDependencies' in m.default) - ) { - return m.default; - } - return m; -}