diff --git a/src/AureliaDependenciesPlugin.ts b/src/AureliaDependenciesPlugin.ts index 1fbb8da..24ac1f2 100644 --- a/src/AureliaDependenciesPlugin.ts +++ b/src/AureliaDependenciesPlugin.ts @@ -1,5 +1,6 @@ import { IncludeDependency } from "./IncludeDependency"; import BasicEvaluatedExpression = require("webpack/lib/BasicEvaluatedExpression"); +import { preserveModuleName } from "./PreserveModuleNamePlugin"; const TAP_NAME = "Aurelia:Dependencies"; @@ -13,7 +14,13 @@ class AureliaDependency extends IncludeDependency { class Template { apply(dep: AureliaDependency, source: Webpack.Source) { - source.replace(dep.range[0], dep.range[1] - 1, "'" + dep.request.replace(/^async(?:\?[^!]*)?!/, "") + "'"); + // Get the module id, fallback to using the module request + let moduleId = dep.request; + if (dep.module && typeof dep.module[preserveModuleName] === "string") { + moduleId = dep.module[preserveModuleName]; + } + + source.replace(dep.range[0], dep.range[1] - 1, "'" + moduleId.replace(/^async(?:\?[^!]*)?!/, "") + "'"); }; } diff --git a/src/PreserveModuleNamePlugin.ts b/src/PreserveModuleNamePlugin.ts index e586448..2591703 100644 --- a/src/PreserveModuleNamePlugin.ts +++ b/src/PreserveModuleNamePlugin.ts @@ -1,7 +1,100 @@ +/** + * The purpose of this plugin is to track down where exactly each included dependency lives and build a module + * name from that. Since projects and webpack configurations can be vary, we do our best to guess but expect edge-cases + * to be hit and changes needed. + * + * Process: + * - Each AureliaDependency contains the preserveModuleName symbol to notify this plugin to track the dependency + * - AureliaDependenciesPlugin searches and includes all PLATFORM.moduleName() dependencies and stores as + * an AureliaDependency + * - ConventionDependenciesPlugin searches for all relative includes and stores as an AureliaDependency + * - At this point, webpack has resolved all included modules regardless of using a relative or absolute path + * - Now we want to normalize each include so that we can reliably replace the include string to match the webpacks + * module id + * - We then unwrap all the modules from ModuleConcatenationPlugin to get the raw dependencies + * - For each raw dependency that is included via node_modules/, read from its location: + * - Example Path: /home/usr/pkg/node_modules/mod/dir/file.js + * - The path to the module itself: /home/usr/pkg/node_modules + * - The module name: mod + * - The relative path: dir/file.js + * - Map the parsed path data to nodeModuleResourcesMap by the parsed module name and its resource location + * - Example Map: + * { + * 'aurelia-templating-router': { + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/router-view.js': { + * path: '/home/usr/pkg/node_modules/aurelia-templating-router', + * name: 'aurelia-templating-router', + * relative: '/dist/native-modules/router-view.js', + * }, + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/route-href.js': { + * path: '/home/usr/pkg/node_modules/aurelia-templating-router', + * name: 'aurelia-templating-router', + * relative: '/dist/native-modules/route-href.js', + * }, + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/aurelia-templating-router.js': { + * path: '/home/usr/pkg/node_modules/aurelia-templating-router', + * name: 'aurelia-templating-router', + * relative: '/dist/native-modules/aurelia-templating-router.js', + * }, + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/route-loader.js': { + * path: '/home/usr/pkg/node_modules/aurelia-templating-router', + * name: 'aurelia-templating-router', + * relative: '/dist/native-modules/route-loader.js', + * }, + * } + * } + * - For each mapped node_module, look at the included resources and normalize: + * - Look at the modules included resources and find a common path and its entry point + * - Entry point conditions: + * - Resource name matches 'index' + * OR Resource name matches the module name + * OR It is the only included module resource + * - If there are multiple entry points: + * - Pick the most shallow resource + * - If they are both as shallow as possible choose index over module name match + * - Map the normalized module id to nodeModuleResourceIdMap by the resource file + * - Example Map: + * { + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/router-view.js': 'aurelia-templating-router/router-view.js', + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/route-href.js': 'aurelia-templating-router/route-href.js', + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/aurelia-templating-router.js': 'aurelia-templating-router', + * '/home/usr/pkg/node_modules/aurelia-templating-router/dist/native-modules/route-loader.js': 'aurelia-templating-router/route-loader.js', + * } + * - Look at all webpack modules and for each module that includes or has a dependency that includes an AureliaDependency: + * - Handling module ids can be a bit tricky. Modules can be included in any of the following ways: + * import { Module } from 'module' + * 'module/submodule' + * './module' + * 'folder/module' + * 'alias/folder/module' + * 'alias$' + * '@scope/module' + * @decorator(PLATFORM.moduleName('module')) + * ... + * - The problem arises when aurelia-loader has to know the module to use at runtime. Webpack Mappings: + * - Absolute Module: 'module' -> 'module' + * - Relative Module: './module' -> 'folder/module' + * - Absolute Path: 'folder/module' -> 'folder/module' + * - Aliased Path: 'alias/folder/module' -> 'alias/folder/module' + * - Absolute Alias Path: 'alias$' -> 'alias$' + * - In order to have the aurelia-loader work correctly, we need to coerce everything to normalized ids + * - If the module is in the node_module/ map, use the normalized module id + * - If the module exists inside a webpack resolver path, use the relative path as the module id + * - If the module exists inside a webpack alias path, use the relative path from the alias path as the module id + * - Set the modules preserveModuleName Symbol export so the AureliaDependenciesPlugin can read it later + * - In AureliaDependenciesPlugin, for each AureliaDependency, replace the original and now incorrect PLATFORM.moduleName() + * path with the normalized path. + * - Source files now contain the normalized module paths with map correctly to the webpack module ids so that the + * aurelia-loader can find them during runtime. + */ + import path = require("path"); -import ModuleDependency = require("webpack/lib/dependencies/ModuleDependency"); export const preserveModuleName = Symbol(); +// node_module maps +const nodeModuleResourcesMap: NodeModule.ResourcesMap = {}; +const nodeModuleResourceIdMap: NodeModule.ResourceIdMap = {}; + const TAP_NAME = "Aurelia:PreserveModuleName"; // This plugins preserves the module names of IncludeDependency and @@ -29,7 +122,11 @@ export class PreserveModuleNamePlugin { if (m.constructor.name === "ConcatenatedModule") modulesBeforeConcat.splice(i--, 1, ...m["modules"]); } - + + // Map and parse the modules if needed + modulesBeforeConcat.forEach(mapNodeModule); + parseNodeModules(); + for (let module of getPreservedModules(modules)) { // Even though it's imported by Aurelia, it's still possible that the module // became the _root_ of a ConcatenatedModule. @@ -38,124 +135,340 @@ export class PreserveModuleNamePlugin { if (module.constructor.name === "ConcatenatedModule") realModule = module["rootModule"]; - let preserve: string | true | undefined = realModule[preserveModuleName]; - let id = typeof preserve === "string" ? preserve : null; - - // No absolute request to preserve, we try to normalize the module resource - if (!id && realModule.resource) - id = fixNodeModule(realModule, modulesBeforeConcat) || - makeModuleRelative(roots, realModule.resource) || - aliasRelative(alias, realModule.resource); - - if (!id) - throw new Error(`Can't figure out a normalized module name for ${realModule.rawRequest}, please call PLATFORM.moduleName() somewhere to help.`); - - // Remove default extensions + // Get the module id + let id = getModuleId(realModule, roots, alias); + + // Remove default extensions normalizers.forEach(n => id = id!.replace(n, "")); - + // Keep "async!" in front of code splits proxies, they are used by aurelia-loader - if (/^async[?!]/.test(realModule.rawRequest)) + if (/^async[?!]/.test(realModule.rawRequest)) id = "async!" + id; - + id = id.replace(/\\/g, "/"); if (module.buildMeta) // meta can be null if the module contains errors module.buildMeta["aurelia-id"] = id; - if (!this.isDll) + if (!this.isDll) { + module[preserveModuleName] = id; module.id = id; + } } }) }); } }; -function getPreservedModules(modules: Webpack.Module[]) { +function getPreservedModules(modules: Webpack.Module[]) { return new Set( modules.filter(m => { // Some modules might have [preserveModuleName] already set, see ConventionDependenciesPlugin. - let value = m[preserveModuleName]; - for (let r of m.reasons) { - if (!r.dependency || !r.dependency[preserveModuleName]) continue; - value = true; - let req = removeLoaders((r.dependency as ModuleDependency).request); - // We try to find an absolute string and set that as the module [preserveModuleName], as it's the best id. - if (req && !req.startsWith(".")) { - m[preserveModuleName] = req; - return true; - } + if (m[preserveModuleName]) { + return true; } - return !!value; + + // Preserve the module if its dependencies are also preserved + return m.reasons.some(reason => (reason.dependency && reason.dependency[preserveModuleName])); }) ); } -function aliasRelative(aliases: {[key: string]: string } | null, resource: string) { - // We consider that aliases point to local folder modules. - // For example: `"my-lib": "../my-lib/src"`. - // Basically we try to make the resource relative to the alias target, - // and if it works we build the id from the alias name. - // So file `../my-lib/src/index.js` becomes `my-lib/index.js`. - // Note that this only works with aliases pointing to folders, not files. - // To have a "default" file in the folder, the following construct works: - // alias: { "mod$": "src/index.js", "mod": "src" } - if (!aliases) return null; - for (let name in aliases) { - let root = path.resolve(aliases[name]); - let relative = path.relative(root, resource); - if (relative.startsWith("..")) continue; - name = name.replace(/\$$/, ""); // A trailing $ indicates an exact match in webpack - return relative ? name + "/" + relative : name; - } - return null; +function isNodeModule(module: Webpack.Module) { + return (module.resource && /\bnode_modules\b/i.test(module.resource)); } -function makeModuleRelative(roots: string[], resource: string) { - for (let root of roots) { - let relative = path.relative(root, resource); - if (!relative.startsWith("..")) return relative; +/** + * Get module data if it exists in node_modules/ + * + * @param {Webpack.Module} module The module to get the data for + * + * @return {NodeModule.Data|null} The module data if available, null otherwise + */ +function getNodeModuleData(module: Webpack.Module): NodeModule.Data | null { + // Not a node_module? + if (!isNodeModule(module)) { + return null; } - return null; -} -function fixNodeModule(module: Webpack.Module, allModules: Webpack.Module[]) { - if (!/\bnode_modules\b/i.test(module.resource)) return null; - // The problem with node_modules is that often the root of the module is not /node_modules/my-lib - // Webpack is going to look for `main` in `project.json` to find where the main file actually is. - // And this can of course be configured differently with `resolve.alias`, `resolve.mainFields` & co. - - // Our best hope is that the file was not required as a relative path, then we can just preserve that. - // We just need to be careful with loaders (e.g. async!) - let request = removeLoaders(module.rawRequest)!; // we assume that Aurelia dependencies always have a rawRequest - if (!request.startsWith(".")) return request; - - // Otherwise we need to build the relative path from the module root, which as explained above is hard to find. - // Ideally we could use webpack resolvers, but they are currently async-only, which can't be used in before-modules-id - // See https://github.com/webpack/webpack/issues/1634 - // Instead, we'll look for the root library module, because it should have been requested somehow and work from there. // Note that the negative lookahead (?!.*node_modules) ensures that we only match the last node_modules/ folder in the path, // in case the package was located in a sub-node_modules (which can occur in special circumstances). // We also need to take care of scoped modules. If the name starts with @ we must keep two parts, // so @corp/bar is the proper module name. - let name = /\bnode_modules[\\/](?!.*\bnode_modules\b)((?:@[^\\/]+[\\/])?[^\\/]+)/i.exec(module.resource)![1]; - name = name.replace("\\", "/"); // normalize \ to / for scoped modules - let entry = allModules.find(m => removeLoaders(m.rawRequest) === name); - if (entry) - return name + "/" + path.relative(path.dirname(entry.resource), module.resource); - // We could not find the raw module. Let's try to find another a more complex entry point. - for (let m of allModules) { - let req = removeLoaders(m.rawRequest); - if (!req || !req.startsWith(name) || !m.resource) continue; - let i = m.resource.replace(/\\/g, "/").lastIndexOf(req.substr(name.length)); - if (i < 0) continue; - let root = m.resource.substr(0, i); - return name + "/" + path.relative(root, module.resource); - } - throw new Error("PreserveModuleNamePlugin: Unable to find root of module " + name); + const modulePaths = module.resource.match(/(.*\bnode_modules[\\/](?!.*\bnode_modules\b)((?:@[^\\/]+[\\/])?[^\\/]+))(.*)/i); + if (!modulePaths || modulePaths.length !== 4) { + return null; + } + + return { + path: modulePaths[1], + name: modulePaths[2], + relative: modulePaths[3], + }; +} + +/** + * If a module exists in node_modules/ map its data + * + * @param {Webpack.Module} module The module to map + * + * @return {undefined} + */ +function mapNodeModule(module: Webpack.Module) { + // Not a node_module? + if (!isNodeModule(module)) { + return; + } + + // Get the module data + const moduleData = getNodeModuleData(module); + if (!moduleData) { + return; + } + + // Map it + if (!nodeModuleResourcesMap[moduleData.name]) { + nodeModuleResourcesMap[moduleData.name] = {}; + } + nodeModuleResourcesMap[moduleData.name][module.resource] = moduleData; +} + +/** + * Parse the resource map for modules that exist in node_modules/ + * + * Since a module can be imported in any number of ways, we cannot rely + * on the module request or any other dynamic scenarios. This gets even + * more tricky when modules themselves can import their own relative or + * even node_modules/. + * + * In order to remedy this, we look at every module that is resolved in + * webpack and combine modules who share a common module path. While this + * approach works, it gets complicated when you have to detect the modules + * entry point. Luckily, using PLATFORM.moduleName() on the modules entry + * point will allow webpack to include the resource when parsing if needed. + * + * However, we need to know exactly which resource is the module entry. + * In order to do this, we make some assumptions: + * - The resource name matches 'index', discounting the extenstion + * - The resource name matches the module key exactly + * - If there is only one module resource, use that as the entry + * - If there are multiple entry points: (index, exact match) + * - Pick the resource that is most shallow to the modules root + * - Otherwise, choose the 'index' resource + * + * @return {undefined} + */ +function parseNodeModules() { + if (!nodeModuleResourcesMap || !Object.keys(nodeModuleResourcesMap).length) { + return; + } + + // Parse each module + for (const moduleKey in nodeModuleResourcesMap) { + const moduleResources: NodeModule.ModuleResource = nodeModuleResourcesMap[moduleKey]; + + // Keep track of the common resource path and possible module entry points + let commonPathParts: string[] = []; + let possibleEntryPoints: string[] = []; + + // Parse each resource in the module + for (const resource in moduleResources) { + const data: NodeModule.Data = moduleResources[resource]; + const pathParts = data.relative.split("/"); + const resourceFile: string | null = pathParts.splice(-1)[0]; + if (!resourceFile) { + continue; + } + + // Entry? + const resourceName = resourceFile.replace(/\..*/, ""); + if (resourceName === moduleKey || resourceName === "index") { + possibleEntryPoints.push(resource); + } + + // Initial or only resource? + if (!commonPathParts.length) { + commonPathParts = pathParts.slice(); + continue; + } + + // Remove uncommon paths + let cont = true; + commonPathParts = commonPathParts.reduce((common: string[], segment: string, idx: number): string[] => { + // Same? + if (cont && segment === pathParts[idx]) { + common.push(segment); + } + else { + cont = false; + } + return common; + }, []); + } + + // Convert common path to string + let commonPath = commonPathParts.join("/"); + commonPath = (commonPath.startsWith("/")) ? commonPath : `/${commonPath}`; + + // If there is more than one possible entry point, use the most shallow resource + let moduleEntry: string | null = null; + possibleEntryPoints.forEach((resource) => { + const data: NodeModule.Data = moduleResources[resource]; + + // No entry yet? + if (!moduleEntry) { + moduleEntry = data.relative; + } + + // Shallow? + else if (moduleEntry.split("/").length > data.relative.split("/").length) { + moduleEntry = data.relative; + } + + // This is an odd edge-case, both are as shallow as possible + // We attempt to use index over moduleKey + else if (!(/\bindex\b/i.test(moduleEntry)) && /\bindex\b/i.test(data.relative)) { + moduleEntry = data.relative; + } + }); + + // If an entry point still hasnt been found and there is only one resource, use that + const resourceKeys = Object.keys(moduleResources); + if (!moduleEntry && resourceKeys.length === 1) { + moduleEntry = moduleResources[resourceKeys[0]].relative; + } + + // Map the resources to the module id + resourceKeys.forEach((resource) => { + const data: NodeModule.Data = moduleResources[resource]; + + // Entry? + if (moduleEntry === data.relative) { + nodeModuleResourceIdMap[resource] = moduleKey; + return; + } + + // Build the id from the resources common path + let key = data.relative.replace(new RegExp(`^${escapeString(commonPath)}`), ""); + key = (key.startsWith("/")) ? key : `/${key}`; + nodeModuleResourceIdMap[resource] = `${moduleKey}${key}`; + }); + } +} + +/** + * Find the path to a modules resource relative to the webpack + * resolve module paths + * + * @param {Webpack.Module} module The module to get the relative path + * @param {string[]} paths The webpack resolver module paths + * + * @return {string|null} The relative path if available, null otherwise + */ +function getRelativeModule(module: Webpack.Module, paths: string[]): string | null { + if (!module.resource || !paths || !paths.length) { + return null; + } + + // Try to find the module in the resolver paths + for (let i = 0, len = paths.length; i < len; i++) { + const relative = path.relative(paths[i], module.resource); + if (!relative.startsWith("..")) { + return relative; + } + } + + // Nothing relative + return null; +} + +/** + * Find the path to a modules resource relative to webpack aliases + * + * @param {Webpack.Module} module The module to alias + * @param {[{ [key: string]: string }|null} aliases The webpack aliases + * + * @return {string|null} The alias path if available, null otherwise + */ +function getAliasModule(module: Webpack.Module, aliases: { [key: string]: string } | null): string | null { + if (!module.resource || !aliases || !Object.keys(aliases).length) { + return null; + } + + // Look for the module in each alias + for (let alias in aliases) { + const relative = path.relative(path.resolve(aliases[alias]), module.resource); + if (relative.startsWith("..")) { + continue; + } + + // Absolute alias? + alias = alias.replace(/\$$/, ""); + return (relative && relative.length) ? `${alias}/${relative}` : alias; + } + + // Nothing aliased + return null; +} + +/** + * Get the module id based on its resource + * + * @param {Webpack.Module} module The module to get the id for + * @param {string[]} paths The webpack resolver module paths + * @param {[{ [key: string]: string }|null} aliases The webpack aliases + * + * @return {string} The module id + */ +function getModuleId(module: Webpack.Module, paths: string[], aliases: { [key: string]: string } | null): string { + // Handling module ids can be a bit tricky + // Modules can be included in any of the following ways: + // import { Module } from 'module' + // 'module/submodule' + // './module' + // 'folder/module' + // 'alias/folder/module' + // 'alias$' + // '@scope/module' + // + // @decorator(PLATFORM.moduleName('module')) + // ... + // + // The problem arises when aurelia-loader has to know the module to use at runtime. + // Webpack Mappings: + // Absolute Module: 'module' -> 'module' + // Relative Module: './module' -> 'folder/module' + // Absolute Path: 'folder/module' -> 'folder/module' + // Aliased Path: 'alias/folder/module' -> 'alias/folder/module' + // Absolute Alias Path: 'alias$' -> 'alias$' + // + // In order to have the aurelia-loader work correctly, we need to coerce everything to absolute ids + // Is it a node_module? + if (isNodeModule(module)) { + return nodeModuleResourceIdMap[module.resource]; + } + + // Get the module relative to the webpack resolver paths + let moduleId = getRelativeModule(module, paths); + + // Fallback to parsing aliases if needed + if (!moduleId) { + moduleId = getAliasModule(module, aliases); + } + + // Still nothing? + if (!moduleId) { + throw new Error(`Can't figure out a normalized module name for ${module.rawRequest}, please call PLATFORM.moduleName() somewhere to help.`); + } + + return moduleId; } -function removeLoaders(request: string | undefined) { - // We have to be careful, as it seems that in the allModules.find() call above - // some modules might have m.rawRequst === undefined - if (!request) return request; - let lastBang = request.lastIndexOf("!"); - return lastBang < 0 ? request : request.substr(lastBang + 1); +/** + * Escape a string to pass to a regular expression + * + * @param {string} str The string to escape + * + * @return {string|null} The escaped string + */ +function escapeString(str: string): string | null { + return str.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&"); } diff --git a/src/webpack.d.ts b/src/webpack.d.ts index 348903f..d3fddc6 100644 --- a/src/webpack.d.ts +++ b/src/webpack.d.ts @@ -57,10 +57,15 @@ declare namespace Webpack { range: [number, number]; // Those types are not correct, but that's enough to compile this project property: IdentifierExpression; - object: { name: string; type: string; } & MemberExpression; + object: { name: string; type: string; } & (MemberExpression | Identifier); type: "MemberExpression"; } + export class Identifier { + name: string; + type: "Identifier"; + } + export class IdentifierExpression { range: [number, number]; name: string; @@ -202,16 +207,36 @@ declare namespace Webpack { } } +declare namespace NodeModule { + interface ModuleResource { + [key: string]: Data; + } + + interface ResourcesMap { + [key: string]: ModuleResource; + } + + interface ResourceIdMap { + [key: string]: string; + } + + interface Data { + path: string; + name: string; + relative: string; + } +} + declare module "webpack" { export class DefinePlugin { constructor(hash: any); apply(compiler: Webpack.Compiler): void; } - export class DllPlugin { + export class DllPlugin { } - export class DllReferencePlugin { + export class DllReferencePlugin { } }