From 49b724dc63917593c41bf6d82b9e3933dc887b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Wed, 4 Sep 2024 12:02:55 -0400 Subject: [PATCH 1/7] breaking: use webpack builtin cache --- src/cacheHandler.js | 99 +++++++++++++++++++ src/index.js | 43 ++++---- test/cache.test.js | 48 ++++++--- .../uncacheable-passthrough-loader.cjs | 4 + 4 files changed, 165 insertions(+), 29 deletions(-) create mode 100644 src/cacheHandler.js create mode 100644 test/fixtures/uncacheable-passthrough-loader.cjs diff --git a/src/cacheHandler.js b/src/cacheHandler.js new file mode 100644 index 00000000..06a82a0d --- /dev/null +++ b/src/cacheHandler.js @@ -0,0 +1,99 @@ +const serialize = require("./serialize"); +const transform = require("./transform"); +const { promisify } = require("node:util"); + +/** @typedef {import("webpack").Compilation} Compilation */ +/** @typedef {import("webpack").LoaderContext<{}>} LoaderContext */ +/** @typedef {ReturnType} WebpackLogger */ +/** @typedef {ReturnType} CacheFacade */ + +const addTimestamps = async function (externalDependencies, getFileTimestamp) { + for (const depAndEmptyTimestamp of externalDependencies) { + try { + const [dep] = depAndEmptyTimestamp; + const { timestamp } = await getFileTimestamp(dep); + depAndEmptyTimestamp.push(timestamp); + } catch { + // ignore errors if timestamp is not available + } + } +}; + +const areExternalDependenciesModified = async function ( + externalDepsWithTimestamp, + getFileTimestamp, +) { + for (const depAndTimestamp of externalDepsWithTimestamp) { + const [dep, timestamp] = depAndTimestamp; + let newTimestamp; + try { + newTimestamp = (await getFileTimestamp(dep)).timestamp; + } catch { + return true; + } + if (timestamp !== newTimestamp) { + return true; + } + } + return false; +}; + +/** + * @this {LoaderContext} + * @param {string} filename The input resource path + * @param {string} source The input source + * @param {object} options The Babel transform options + * @param {CacheFacade} cacheFacade The webpack cache facade instance + * @param {string} cacheIdentifier The extra cache identifier + * @param {WebpackLogger} logger + */ +async function handleCache( + filename, + source, + options = {}, + cacheFacade, + cacheIdentifier, + logger, +) { + const getFileTimestamp = promisify((path, cb) => { + this._compilation.fileSystemInfo.getFileTimestamp(path, cb); + }); + const hash = this.utils.createHash( + this._compilation.outputOptions.hashFunction, + ); + const cacheKey = hash + .update(serialize([options, source, cacheIdentifier])) + .digest("hex"); + logger.debug(`getting cache for '${filename}', cachekey '${cacheKey}'`); + + const itemCache = cacheFacade.getItemCache(cacheKey, null); + + let result = await itemCache.getPromise(); + logger.debug( + result ? `found cache for '${filename}'` : `missed cache for '${filename}'`, + ); + if (result) { + if ( + await areExternalDependenciesModified( + result.externalDependencies, + getFileTimestamp, + ) + ) { + logger.debug( + `discarded cache for '${filename}' due to changes in external dependencies`, + ); + result = null; + } + } + + if (!result) { + logger.debug("applying Babel transform"); + result = await transform(source, options); + await addTimestamps(result.externalDependencies, getFileTimestamp); + logger.debug(`caching result for '${filename}'`); + await itemCache.storePromise(result); + logger.debug(`cached result for '${filename}'`); + } +} + +module.exports = handleCache; diff --git a/src/index.js b/src/index.js index 96e5fd48..d33cd11c 100644 --- a/src/index.js +++ b/src/index.js @@ -20,13 +20,12 @@ if (/^6\./.test(babel.version)) { } const { version } = require("../package.json"); -const cache = require("./cache"); +const cacheHandler = require("./cacheHandler"); const transform = require("./transform"); const injectCaller = require("./injectCaller"); const schema = require("./schema"); const { isAbsolute } = require("path"); -const { promisify } = require("util"); function subscribe(subscriber, metadata, context) { if (context[subscriber]) { @@ -112,6 +111,23 @@ async function loader(source, inputSourceMap, overrides) { ), ); } + if (typeof loaderOptions.cacheDirectory === "string") { + this.emitWarning( + new Error( + "babel-loader does not support customizing the cacheDirectory since it now uses the webpack builtin cache. You can use cacheDirectory: true and specify the webpack cache location instead via the webpack option `cache.cacheDirectory`.", + ), + ); + } + if ("cacheCompression" in loaderOptions) { + this.emitWarning( + new Error( + "The option `cacheCompression` has been removed since the babel-loader now uses the webpack builtin cache." + + loaderOptions.cacheCompression + ? " You can specify the webpack option `cache.compression` to 'gzip' or 'brotli'." + : "", + ), + ); + } logger.debug("normalizing loader options"); // Standardize on 'sourceMaps' as the key passed through to Webpack, so that @@ -147,7 +163,6 @@ async function loader(source, inputSourceMap, overrides) { delete programmaticOptions.customize; delete programmaticOptions.cacheDirectory; delete programmaticOptions.cacheIdentifier; - delete programmaticOptions.cacheCompression; delete programmaticOptions.metadataSubscribers; logger.debug("resolving Babel configs"); @@ -176,32 +191,24 @@ async function loader(source, inputSourceMap, overrides) { } const { - cacheDirectory = null, + cacheDirectory, cacheIdentifier = "core" + transform.version + "," + "loader" + version, - cacheCompression = true, metadataSubscribers = [], } = loaderOptions; let result; if (cacheDirectory) { logger.debug("cache is enabled"); - const getFileTimestamp = promisify((path, cb) => { - this._compilation.fileSystemInfo.getFileTimestamp(path, cb); - }); - const hash = this.utils.createHash( - this._compilation.outputOptions.hashFunction, - ); - result = await cache({ + const cacheFacade = this._compilation.getCache("babel-loader"); + result = await cacheHandler.call( + this, + filename, source, options, - transform, - cacheDirectory, + cacheFacade, cacheIdentifier, - cacheCompression, - hash, - getFileTimestamp, logger, - }); + ); } else { logger.debug("cache is disabled, applying Babel transform"); result = await transform(source, options); diff --git a/test/cache.test.js b/test/cache.test.js index eb7b4efe..b5b56910 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -2,6 +2,7 @@ import test from "node:test"; import fs from "node:fs"; import path from "node:path"; import assert from "node:assert/strict"; +import webpack from "webpack"; import { webpackAsync } from "./helpers/webpackAsync.js"; import createTestDirectory from "./helpers/createTestDirectory.js"; import { fileURLToPath } from "node:url"; @@ -403,6 +404,9 @@ test("should cache result when there are external dependencies", async () => { test("should output debug logs when stats.loggingDebug includes babel-loader", async () => { const config = Object.assign({}, globalConfig, { + cache: { + type: "memory", + }, output: { path: context.directory, }, @@ -410,12 +414,20 @@ test("should output debug logs when stats.loggingDebug includes babel-loader", a rules: [ { test: /\.jsx?/, - loader: babelLoader, + exclude: /node_modules/, - options: { - cacheDirectory: true, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + presets: ["@babel/preset-env"], + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, @@ -424,10 +436,24 @@ test("should output debug logs when stats.loggingDebug includes babel-loader", a }, }); - const stats = await webpackAsync(config); - - assert.match( - stats.toString(config.stats), - /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+reading cache file.+\n\s+discarded cache as it can not be read\n\s+creating cache folder.+\n\s+applying Babel transform\n\s+writing result to cache file.+\n\s+added '.+babel.config.json' to webpack dependencies/, - ); + return new Promise((resolve, reject) => { + const compiler = webpack(config); + compiler.run((err, stats) => { + if (err) reject(err); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\n\s+added '.+babel.config.json' to webpack dependencies/, + "The first run stat does not match the snapshot regex", + ); + compiler.run((err, newStats) => { + if (err) reject(err); + assert.match( + newStats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+\n\s+added '.+babel.config.json' to webpack dependencies/, + "The second run stat does not match the snapshot regex", + ); + resolve(); + }); + }); + }); }); diff --git a/test/fixtures/uncacheable-passthrough-loader.cjs b/test/fixtures/uncacheable-passthrough-loader.cjs new file mode 100644 index 00000000..16df1017 --- /dev/null +++ b/test/fixtures/uncacheable-passthrough-loader.cjs @@ -0,0 +1,4 @@ +module.exports = function UncacheablePassthroughLoader(source) { + this.cacheable(false); + return source; +}; From 5e435d0a4f59ef53149e316a2cc549a8c86cf42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 5 Sep 2024 14:58:25 -0400 Subject: [PATCH 2/7] fix test cases --- src/cacheHandler.js | 2 + test/cache.test.js | 401 ++++++++++++++++++++++++------------------ test/loader.test.js | 10 +- test/metadata.test.js | 3 +- 4 files changed, 240 insertions(+), 176 deletions(-) diff --git a/src/cacheHandler.js b/src/cacheHandler.js index 06a82a0d..2a510ed1 100644 --- a/src/cacheHandler.js +++ b/src/cacheHandler.js @@ -94,6 +94,8 @@ async function handleCache( await itemCache.storePromise(result); logger.debug(`cached result for '${filename}'`); } + + return result; } module.exports = handleCache; diff --git a/test/cache.test.js b/test/cache.test.js index b5b56910..4ebd6488 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -9,10 +9,7 @@ import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const defaultCacheDir = path.join( - __dirname, - "../node_modules/.cache/babel-loader", -); +const defaultCacheDir = path.join(__dirname, "../node_modules/.cache/webpack"); const cacheDir = path.join(__dirname, "output/cache/cachefiles"); const outputDir = path.join(__dirname, "output/cache"); const babelLoader = path.join(__dirname, "../lib"); @@ -31,9 +28,10 @@ const globalConfig = { }, }; -// Cache filename is either SHA256 or MD5 hash -const UNCOMPRESSED_CACHE_FILE_REGEX = /^[0-9a-f]{32}(?:[0-9a-f]{32})?\.json$/; -const CACHE_FILE_REGEX = /^[0-9a-f]{32}(?:[0-9a-f]{32})?\.json\.gz$/; +// example: 0.pack, index.pack +const UNCOMPRESSED_CACHE_FILE_REGEX = /.+\.pack$/; +// example: 0.pack.gz, index.pack.gz +const CACHE_FILE_REGEX = /.+\.pack\.gz$/; // Create a separate directory for each test so that the tests // can run in parallel @@ -50,25 +48,37 @@ test.beforeEach(() => fs.rmSync(defaultCacheDir, { recursive: true, force: true }), ); test.afterEach(() => { + fs.rmSync(defaultCacheDir, { recursive: true, force: true }); fs.rmSync(context.directory, { recursive: true, force: true }); fs.rmSync(context.cacheDirectory, { recursive: true, force: true }); }); -test("should output files to cache directory", async () => { +test("should output files to cache directory when cache type is filesystem", async () => { const config = Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + cacheDirectory: context.cacheDirectory, + }, output: { path: context.directory, }, module: { rules: [ { - test: /\.js$/, - loader: babelLoader, + test: /\.jsx?/, exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + presets: ["@babel/preset-env"], + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, @@ -82,8 +92,12 @@ test("should output files to cache directory", async () => { assert.ok(files.length > 0); }); -test("should output json.gz files to standard cache dir by default", async () => { +test("should output pack.gz files to standard cache dir when cache.compression is gzip", async () => { const config = Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + compression: "gzip", + }, output: { path: context.directory, }, @@ -91,12 +105,19 @@ test("should output json.gz files to standard cache dir by default", async () => rules: [ { test: /\.jsx?/, - loader: babelLoader, exclude: /node_modules/, - options: { - cacheDirectory: true, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + presets: ["@babel/preset-env"], + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, @@ -106,13 +127,17 @@ test("should output json.gz files to standard cache dir by default", async () => assert.deepEqual(stats.compilation.errors, []); assert.deepEqual(stats.compilation.warnings, []); - let files = fs.readdirSync(defaultCacheDir); + let files = fs.readdirSync(defaultCacheDir, { recursive: true }); files = files.filter(file => CACHE_FILE_REGEX.test(file)); assert.ok(files.length > 0); }); -test("should output non-compressed files to standard cache dir when cacheCompression is set to false", async () => { +test("should output non-compressed files to standard cache dir when cache.compression is set to false", async () => { const config = Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + compression: false, + }, output: { path: context.directory, }, @@ -124,8 +149,8 @@ test("should output non-compressed files to standard cache dir when cacheCompres exclude: /node_modules/, options: { cacheDirectory: true, - cacheCompression: false, presets: ["@babel/preset-env"], + configFile: false, }, }, ], @@ -133,42 +158,16 @@ test("should output non-compressed files to standard cache dir when cacheCompres }); await webpackAsync(config); - let files = fs.readdirSync(defaultCacheDir); + let files = fs.readdirSync(defaultCacheDir, { recursive: true }); files = files.filter(file => UNCOMPRESSED_CACHE_FILE_REGEX.test(file)); assert.ok(files.length > 0); }); -test("should output files to standard cache dir if set to true in query", async () => { +test("should read from cache directory if cached exists", async () => { const config = Object.assign({}, globalConfig, { - output: { - path: context.directory, - }, - module: { - rules: [ - { - test: /\.jsx?/, - loader: babelLoader, - exclude: /node_modules/, - options: { - cacheDirectory: true, - presets: ["@babel/preset-env"], - }, - }, - ], + cache: { + type: "filesystem", }, - }); - - const stats = await webpackAsync(config); - assert.deepEqual(stats.compilation.errors, []); - assert.deepEqual(stats.compilation.warnings, []); - - let files = fs.readdirSync(defaultCacheDir); - files = files.filter(file => CACHE_FILE_REGEX.test(file)); - assert.ok(files.length > 0); -}); - -test("should read from cache directory if cached file exists", async () => { - const config = Object.assign({}, globalConfig, { output: { path: context.directory, }, @@ -176,78 +175,56 @@ test("should read from cache directory if cached file exists", async () => { rules: [ { test: /\.jsx?/, - loader: babelLoader, exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + presets: ["@babel/preset-env"], + configFile: false, + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, + stats: { + loggingDebug: ["babel-loader"], + }, }); - // @TODO Find a way to know if the file as correctly read without relying on - // Istanbul for coverage. - const stats = await webpackAsync(config); + let stats = await webpackAsync(config); assert.deepEqual(stats.compilation.errors, []); assert.deepEqual(stats.compilation.warnings, []); - - await webpackAsync(config); - const files = fs.readdirSync(context.cacheDirectory); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\n/, + "The first run stat does not match the snapshot regex", + ); + let files = fs.readdirSync(defaultCacheDir, { recursive: true }); + files = files.filter(file => UNCOMPRESSED_CACHE_FILE_REGEX.test(file)); assert.ok(files.length > 0); -}); - -test("should have one file per module", async () => { - const config = Object.assign({}, globalConfig, { - output: { - path: context.directory, - }, - module: { - rules: [ - { - test: /\.jsx?/, - loader: babelLoader, - exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - presets: ["@babel/preset-env"], - }, - }, - ], - }, - }); - const stats = await webpackAsync(config); + stats = await webpackAsync(config); assert.deepEqual(stats.compilation.errors, []); assert.deepEqual(stats.compilation.warnings, []); - - const files = fs.readdirSync(context.cacheDirectory); - assert.strictEqual(files.length, 3); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+\n/, + "The second run stat does not match the snapshot regex", + ); }); -test("should generate a new file if the identifier changes", async () => { - const configs = [ +test("should not reuse cache if the identifier changes", async () => { + const configFactory = cacheIdentifier => Object.assign({}, globalConfig, { - output: { - path: context.directory, + cache: { + type: "filesystem", }, - module: { - rules: [ - { - test: /\.jsx?/, - loader: babelLoader, - exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - cacheIdentifier: "a", - presets: ["@babel/preset-env"], - }, - }, - ], - }, - }), - Object.assign({}, globalConfig, { output: { path: context.directory, }, @@ -255,35 +232,64 @@ test("should generate a new file if the identifier changes", async () => { rules: [ { test: /\.jsx?/, - loader: babelLoader, exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - cacheIdentifier: "b", - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + cacheIdentifier, + presets: ["@babel/preset-env"], + configFile: false, + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, - }), - ]; + stats: { + loggingDebug: ["babel-loader"], + }, + }); - await Promise.allSettled( - configs.map(async config => { - const stats = await webpackAsync(config); - assert.deepEqual(stats.compilation.errors, []); - assert.deepEqual(stats.compilation.warnings, []); - }), + let config = configFactory("a"); + let stats = await webpackAsync(config); + assert.deepEqual(stats.compilation.errors, []); + assert.deepEqual(stats.compilation.warnings, []); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+/, + "The first run stat does not match the snapshot regex", ); + let files = fs.readdirSync(defaultCacheDir, { recursive: true }); + files = files.filter(file => UNCOMPRESSED_CACHE_FILE_REGEX.test(file)); + assert.ok(files.length > 0); - const files = fs.readdirSync(context.cacheDirectory); - assert.strictEqual(files.length, 6); + config = configFactory("b"); + stats = await webpackAsync(config); + assert.deepEqual(stats.compilation.errors, []); + assert.deepEqual(stats.compilation.warnings, []); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+/, + "The second run stat does not match the snapshot regex", + ); + files = fs.readdirSync(defaultCacheDir, { recursive: true }); + files = files.filter(file => UNCOMPRESSED_CACHE_FILE_REGEX.test(file)); + assert.ok(files.length > 0); }); test("should allow to specify the .babelrc file", async () => { - const config = [ + const configs = [ Object.assign({}, globalConfig, { entry: path.join(__dirname, "fixtures/constant.js"), + cache: { + type: "filesystem", + cacheDirectory: context.cacheDirectory, + }, output: { path: context.directory, }, @@ -291,19 +297,34 @@ test("should allow to specify the .babelrc file", async () => { rules: [ { test: /\.jsx?/, - loader: babelLoader, exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - extends: path.join(__dirname, "fixtures/babelrc"), - babelrc: false, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + extends: path.join(__dirname, "fixtures/babelrc"), + babelrc: false, + presets: ["@babel/preset-env"], + configFile: false, + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, + stats: { + loggingDebug: ["babel-loader"], + }, }), Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + cacheDirectory: context.cacheDirectory, + }, entry: path.join(__dirname, "fixtures/constant.js"), output: { path: context.directory, @@ -312,27 +333,47 @@ test("should allow to specify the .babelrc file", async () => { rules: [ { test: /\.jsx?/, - loader: babelLoader, exclude: /node_modules/, - options: { - cacheDirectory: context.cacheDirectory, - presets: ["@babel/preset-env"], - }, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + presets: ["@babel/preset-env"], + configFile: false, + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, + stats: { + loggingDebug: ["babel-loader"], + }, }), ]; - const multiStats = await webpackAsync(config); - assert.deepEqual(multiStats.stats[0].compilation.errors, []); - assert.deepEqual(multiStats.stats[0].compilation.warnings, []); - assert.deepEqual(multiStats.stats[1].compilation.errors, []); - assert.deepEqual(multiStats.stats[1].compilation.warnings, []); + let stats = await webpackAsync(configs[0]); + assert.deepEqual(stats.compilation.errors, []); + assert.deepEqual(stats.compilation.warnings, []); + assert.match( + stats.toString(configs[0].stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\s+added '.+fixtures\/babelrc' to webpack dependencies/, + "The first run stat does not match the snapshot regex", + ); - const files = fs.readdirSync(context.cacheDirectory); - // The two configs resolved to same Babel config because "fixtures/babelrc" - // is { "presets": ["@babel/preset-env"] } - assert.strictEqual(files.length, 1); + // The cache is reused because the two configs resolved to same Babel + // config as "fixtures/babelrc" is { "presets": ["@babel/preset-env"] } + stats = await webpackAsync(configs[1]); + assert.deepEqual(stats.compilation.errors, []); + assert.deepEqual(stats.compilation.warnings, []); + assert.match( + stats.toString(configs[1].stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+\n/, + "The second run stat does not match the snapshot regex", + ); }); test("should cache result when there are external dependencies", async () => { @@ -343,6 +384,10 @@ test("should cache result when there are external dependencies", async () => { let counter = 0; const config = Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + cacheDirectory: context.cacheDirectory, + }, entry: path.join(__dirname, "fixtures/constant.js"), output: { path: context.directory, @@ -350,30 +395,40 @@ test("should cache result when there are external dependencies", async () => { module: { rules: [ { - test: /\.js$/, - loader: babelLoader, - options: { - babelrc: false, - configFile: false, - cacheDirectory: context.cacheDirectory, - plugins: [ - api => { - api.cache.never(); - api.addExternalDependency(dep); - return { - visitor: { - BooleanLiteral(path) { - counter++; - path.replaceWith( - api.types.stringLiteral(fs.readFileSync(dep, "utf8")), - ); - path.stop(); - }, + test: /\.jsx?/, + exclude: /node_modules/, + use: [ + { + loader: babelLoader, + options: { + babelrc: false, + configFile: false, + cacheDirectory: true, + plugins: [ + api => { + api.cache.never(); + api.addExternalDependency(dep); + return { + visitor: { + BooleanLiteral(path) { + counter++; + path.replaceWith( + api.types.stringLiteral( + fs.readFileSync(dep, "utf8"), + ), + ); + path.stop(); + }, + }, + }; }, - }; + ], }, - ], - }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], }, ], }, @@ -402,7 +457,7 @@ test("should cache result when there are external dependencies", async () => { assert.strictEqual(counter, 2); }); -test("should output debug logs when stats.loggingDebug includes babel-loader", async () => { +test("should work with memory type webpack cache", async () => { const config = Object.assign({}, globalConfig, { cache: { type: "memory", @@ -414,13 +469,13 @@ test("should output debug logs when stats.loggingDebug includes babel-loader", a rules: [ { test: /\.jsx?/, - exclude: /node_modules/, use: [ { loader: babelLoader, options: { cacheDirectory: true, + configFile: false, presets: ["@babel/preset-env"], }, }, @@ -442,14 +497,14 @@ test("should output debug logs when stats.loggingDebug includes babel-loader", a if (err) reject(err); assert.match( stats.toString(config.stats), - /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\n\s+added '.+babel.config.json' to webpack dependencies/, + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+/, "The first run stat does not match the snapshot regex", ); compiler.run((err, newStats) => { if (err) reject(err); assert.match( newStats.toString(config.stats), - /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+\n\s+added '.+babel.config.json' to webpack dependencies/, + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+/, "The second run stat does not match the snapshot regex", ); resolve(); diff --git a/test/loader.test.js b/test/loader.test.js index d672daac..f27c2d95 100644 --- a/test/loader.test.js +++ b/test/loader.test.js @@ -210,7 +210,15 @@ test("should output debug logs when stats.loggingDebug includes babel-loader", a }, }); - const stats = await webpackAsync(config); + // The webpack memory cache will be cleaned during compiler shutdown, + // so the development default memory cache will not be reused. + let stats = await webpackAsync(config); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is disabled, applying Babel transform/, + ); + + stats = await webpackAsync(config); assert.match( stats.toString(config.stats), /normalizing loader options\n\s+resolving Babel configs\n\s+cache is disabled, applying Babel transform/, diff --git a/test/metadata.test.js b/test/metadata.test.js index ee049a97..003c9e47 100644 --- a/test/metadata.test.js +++ b/test/metadata.test.js @@ -9,7 +9,6 @@ const { NormalModule } = webpack; import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const cacheDir = path.join(__dirname, "output/cache/cachefiles"); const outputDir = path.join(__dirname, "output/metadata"); const babelLoader = path.join(__dirname, "../lib"); @@ -114,7 +113,7 @@ test("should obtain metadata from the transform result with cache", async () => test: /\.js/, loader: babelLoader, options: { - cacheDirectory: cacheDir, + cacheDirectory: true, metadataSubscribers: [WebpackMetadataSubscriberPlugin.subscriber], plugins: [babelMetadataProvierPlugin], babelrc: false, From 22214265de704fba8692c8c542f827c839f38838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 5 Sep 2024 15:03:40 -0400 Subject: [PATCH 3/7] remove unused old cache lib --- package.json | 5 +- src/cache.js | 227 --------------------------------------------------- yarn.lock | 1 - 3 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 src/cache.js diff --git a/package.json b/package.json index 9d1edb3c..bc75607d 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,6 @@ "engines": { "node": "^18.20.0 || ^20.10.0 || >=22.0.0" }, - "dependencies": { - "find-up": "^5.0.0" - }, "peerDependencies": { "@babel/core": "^7.12.0", "webpack": ">=5.61.0" @@ -98,4 +95,4 @@ ] }, "packageManager": "yarn@3.6.4" -} \ No newline at end of file +} diff --git a/src/cache.js b/src/cache.js deleted file mode 100644 index 7d953f3b..00000000 --- a/src/cache.js +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Filesystem Cache - * - * Given a file and a transform function, cache the result into files - * or retrieve the previously cached files if the given file is already known. - * - * @see https://github.com/babel/babel-loader/issues/34 - * @see https://github.com/babel/babel-loader/pull/41 - */ -const os = require("os"); -const path = require("path"); -const zlib = require("zlib"); -const { promisify } = require("util"); -const { readFile, writeFile, mkdir } = require("fs/promises"); -const { sync: findUpSync } = require("find-up"); -const { env } = process; -const transform = require("./transform"); -const serialize = require("./serialize"); -let defaultCacheDirectory = null; - -const gunzip = promisify(zlib.gunzip); -const gzip = promisify(zlib.gzip); - -/** - * Read the contents from the compressed file. - * - * @async - * @params {String} filename - * @params {Boolean} compress - */ -const read = async function (filename, compress) { - const data = await readFile(filename + (compress ? ".gz" : "")); - const content = compress ? await gunzip(data) : data; - - return JSON.parse(content.toString()); -}; - -/** - * Write contents into a compressed file. - * - * @async - * @params {String} filename - * @params {Boolean} compress - * @params {String} result - */ -const write = async function (filename, compress, result) { - const content = JSON.stringify(result); - - const data = compress ? await gzip(content) : content; - return await writeFile(filename + (compress ? ".gz" : ""), data); -}; - -/** - * Build the filename for the cached file - * - * @params {String} source File source code - * @params {Object} options Options used - * - * @return {String} - */ -const filename = function (source, identifier, options, hash) { - hash.update(serialize([options, source, identifier])); - - return hash.digest("hex") + ".json"; -}; - -const addTimestamps = async function (externalDependencies, getFileTimestamp) { - for (const depAndEmptyTimestamp of externalDependencies) { - try { - const [dep] = depAndEmptyTimestamp; - const { timestamp } = await getFileTimestamp(dep); - depAndEmptyTimestamp.push(timestamp); - } catch { - // ignore errors if timestamp is not available - } - } -}; - -const areExternalDependenciesModified = async function ( - externalDepsWithTimestamp, - getFileTimestamp, -) { - for (const depAndTimestamp of externalDepsWithTimestamp) { - const [dep, timestamp] = depAndTimestamp; - let newTimestamp; - try { - newTimestamp = (await getFileTimestamp(dep)).timestamp; - } catch { - return true; - } - if (timestamp !== newTimestamp) { - return true; - } - } - return false; -}; - -/** - * Handle the cache - * - * @params {String} directory - * @params {Object} params - */ -const handleCache = async function (directory, params) { - const { - source, - options = {}, - cacheIdentifier, - cacheDirectory, - cacheCompression, - hash, - getFileTimestamp, - logger, - } = params; - - const file = path.join( - directory, - filename(source, cacheIdentifier, options, hash), - ); - - try { - // No errors mean that the file was previously cached - // we just need to return it - logger.debug(`reading cache file '${file}'`); - const result = await read(file, cacheCompression); - if ( - !(await areExternalDependenciesModified( - result.externalDependencies, - getFileTimestamp, - )) - ) { - logger.debug(`validated cache file '${file}'`); - return result; - } - logger.debug( - `discarded cache file '${file}' due to changes in external dependencies`, - ); - } catch { - // conitnue if cache can't be read - logger.debug(`discarded cache as it can not be read`); - } - - const fallback = - typeof cacheDirectory !== "string" && directory !== os.tmpdir(); - - // Make sure the directory exists. - try { - // overwrite directory if exists - logger.debug(`creating cache folder '${directory}'`); - await mkdir(directory, { recursive: true }); - } catch (err) { - if (fallback) { - return handleCache(os.tmpdir(), params); - } - - throw err; - } - - // Otherwise just transform the file - // return it to the user asap and write it in cache - logger.debug(`applying Babel transform`); - const result = await transform(source, options); - await addTimestamps(result.externalDependencies, getFileTimestamp); - - try { - logger.debug(`writing result to cache file '${file}'`); - await write(file, cacheCompression, result); - } catch (err) { - if (fallback) { - // Fallback to tmpdir if node_modules folder not writable - return handleCache(os.tmpdir(), params); - } - - throw err; - } - - return result; -}; - -/** - * Retrieve file from cache, or create a new one for future reads - * - * @async - * @param {Object} params - * @param {String} params.cacheDirectory Directory to store cached files - * @param {String} params.cacheIdentifier Unique identifier to bust cache - * @param {Boolean} params.cacheCompression Whether compressing cached files - * @param {String} params.source Original contents of the file to be cached - * @param {Object} params.options Options to be given to the transform fn - * - * @example - * - * const result = await cache({ - * cacheDirectory: '.tmp/cache', - * cacheIdentifier: 'babel-loader-cachefile', - * cacheCompression: false, - * source: *source code from file*, - * options: { - * experimental: true, - * runtime: true - * }, - * }); - */ - -module.exports = async function (params) { - let directory; - - if (typeof params.cacheDirectory === "string") { - directory = params.cacheDirectory; - } else { - defaultCacheDirectory ??= findCacheDir("babel-loader"); - directory = defaultCacheDirectory; - } - - return await handleCache(directory, params); -}; - -function findCacheDir(name) { - if (env.CACHE_DIR && !["true", "false", "1", "0"].includes(env.CACHE_DIR)) { - return path.join(env.CACHE_DIR, name); - } - const rootPkgJSONPath = path.dirname(findUpSync("package.json")); - if (rootPkgJSONPath) { - return path.join(rootPkgJSONPath, "node_modules", ".cache", name); - } - return os.tmpdir(); -} diff --git a/yarn.lock b/yarn.lock index 8c992d54..f1cb6530 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1959,7 +1959,6 @@ __metadata: eslint: ^9.6.0 eslint-config-prettier: ^9.1.0 eslint-plugin-prettier: ^5.1.3 - find-up: ^5.0.0 globals: ^15.8.0 husky: ^9.1.5 lint-staged: ^15.2.9 From c6a894e6594c0dd78c0e357f9e872fc4a811f90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 5 Sep 2024 15:14:33 -0400 Subject: [PATCH 4/7] fix windows CI error --- test/cache.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cache.test.js b/test/cache.test.js index 4ebd6488..8f5debf5 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -360,7 +360,7 @@ test("should allow to specify the .babelrc file", async () => { assert.deepEqual(stats.compilation.warnings, []); assert.match( stats.toString(configs[0].stats), - /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\s+added '.+fixtures\/babelrc' to webpack dependencies/, + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+\s+added '.+fixtures[\\/]babelrc' to webpack dependencies/, "The first run stat does not match the snapshot regex", ); From 44c5b1508b8326f7e1ccd2fb234af4878881e4d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 5 Sep 2024 15:27:43 -0400 Subject: [PATCH 5/7] add a new test case for removing external deps --- test/cache.test.js | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/test/cache.test.js b/test/cache.test.js index 8f5debf5..43888f38 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -457,6 +457,89 @@ test("should cache result when there are external dependencies", async () => { assert.strictEqual(counter, 2); }); +test("should burst cache when the external dependency is removed from filesystem", async () => { + const dep = path.join(cacheDir, "externalDependency.txt"); + + fs.writeFileSync(dep, "first update"); + + let counter = 0; + + const config = Object.assign({}, globalConfig, { + cache: { + type: "filesystem", + cacheDirectory: context.cacheDirectory, + }, + entry: path.join(__dirname, "fixtures/constant.js"), + output: { + path: context.directory, + }, + module: { + rules: [ + { + test: /\.jsx?/, + exclude: /node_modules/, + use: [ + { + loader: babelLoader, + options: { + babelrc: false, + configFile: false, + cacheDirectory: true, + plugins: [ + api => { + api.cache.never(); + api.addExternalDependency(dep); + return { + visitor: { + BooleanLiteral(path) { + counter++; + let depContent = "dep is removed"; + try { + depContent = fs.readFileSync(dep, "utf8"); + } catch { + // ignore if dep is removed + } + path.replaceWith(api.types.stringLiteral(depContent)); + path.stop(); + }, + }, + }; + }, + ], + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], + }, + ], + }, + }); + + let stats = await webpackAsync(config); + assert.deepEqual(stats.compilation.warnings, []); + assert.deepEqual(stats.compilation.errors, []); + + assert.ok(stats.compilation.fileDependencies.has(dep)); + assert.strictEqual(counter, 1); + + stats = await webpackAsync(config); + assert.deepEqual(stats.compilation.warnings, []); + assert.deepEqual(stats.compilation.errors, []); + + assert.ok(stats.compilation.fileDependencies.has(dep)); + assert.strictEqual(counter, 1); + + fs.rmSync(dep); + stats = await webpackAsync(config); + assert.deepEqual(stats.compilation.warnings, []); + assert.deepEqual(stats.compilation.errors, []); + + assert.ok(stats.compilation.fileDependencies.has(dep)); + assert.strictEqual(counter, 2); +}); + test("should work with memory type webpack cache", async () => { const config = Object.assign({}, globalConfig, { cache: { From 7717fb33606d097a43eded4c50f64681b7d64d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 6 Sep 2024 12:24:52 -0400 Subject: [PATCH 6/7] Add test with custom webpack cache plugin --- test/cache.test.js | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/cache.test.js b/test/cache.test.js index 43888f38..e8c95834 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -75,6 +75,10 @@ test("should output files to cache directory when cache type is filesystem", asy presets: ["@babel/preset-env"], }, }, + // when cache.type is filesystem, webpack will try to cache the loader result if they + // are cacheable (by default). The webpack cache will then be hit before the babel-loader + // cache. To test the babel-loader cache behaviour, we have to mark the loader results + // as uncacheable { loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", }, @@ -595,3 +599,104 @@ test("should work with memory type webpack cache", async () => { }); }); }); + +test("it should work with custom webpack cache plugin", async () => { + class CustomCachePlugin { + /** + * Apply the plugin + * @param {import("webpack").Compiler} compiler the compiler instance + */ + apply(compiler) { + let cache = Object.create(null); + const pluginName = this.constructor.name; + + compiler.cache.hooks.store.tap(pluginName, (identifier, etag, data) => { + cache[identifier] = { etag, data }; + }); + + compiler.cache.hooks.get.tap( + pluginName, + (identifier, etag, gotHandlers) => { + if (!(identifier in cache)) { + return null; + } else if (cache[identifier] != null) { + const cacheEntry = cache[identifier]; + if (cacheEntry.etag === etag) { + return cacheEntry.data; + } else { + return null; + } + } + gotHandlers.push((result, callback) => { + if (result === undefined) { + cache[identifier] = null; + } else { + cache[identifier] = { etag, data: result }; + } + return callback(); + }); + }, + ); + + compiler.cache.hooks.shutdown.tap( + pluginName, + () => (cache = Object.create(null)), + ); + } + } + + const config = Object.assign({}, globalConfig, { + // disable builtin webpack cache so that CustomCachePlugin can provide the cache backend + cache: false, + entry: path.join(__dirname, "fixtures/constant.js"), + output: { + path: context.directory, + }, + module: { + rules: [ + { + test: /\.jsx?/, + exclude: /node_modules/, + use: [ + { + loader: babelLoader, + options: { + cacheDirectory: true, + configFile: false, + presets: ["@babel/preset-env"], + }, + }, + { + loader: "./test/fixtures/uncacheable-passthrough-loader.cjs", + }, + ], + }, + ], + }, + plugins: [new CustomCachePlugin()], + stats: { + loggingDebug: ["babel-loader", CustomCachePlugin.name], + }, + }); + + return new Promise((resolve, reject) => { + const compiler = webpack(config); + compiler.run((err, stats) => { + if (err) reject(err); + assert.match( + stats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+missed cache for.+\n\s+applying Babel transform\n\s+caching result for.+\n\s+cached result for.+/, + "The first run stat does not match the snapshot regex", + ); + compiler.run((err, newStats) => { + if (err) reject(err); + assert.match( + newStats.toString(config.stats), + /normalizing loader options\n\s+resolving Babel configs\n\s+cache is enabled\n\s+getting cache for.+\n\s+found cache for.+/, + "The second run stat does not match the snapshot regex", + ); + resolve(); + }); + }); + }); +}); From 96c4d92568d8bba4c29e6e45a91ed9925ff26e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 23 Sep 2024 15:38:37 -0400 Subject: [PATCH 7/7] update loader docs --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 76abe55f..c4852277 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,20 @@ The `options` passed here will be [merged](https://babeljs.io/docs/configuration This loader also supports the following loader-specific option: -* `cacheDirectory`: Default `false`. When set, the given directory will be used to cache the results of the loader. Future webpack builds will attempt to read from the cache to avoid needing to run the potentially expensive Babel recompilation process on each run. If the value is set to `true` in options (`{cacheDirectory: true}`), the loader will use the default cache directory in `node_modules/.cache/babel-loader` or fallback to the default OS temporary file directory if no `node_modules` folder could be found in any root directory. +* `cacheDirectory`: Default `false`. When set to `true`, Babel loader will use the [webpack builtin cache](https://webpack.js.org/configuration/cache/) to store the transformed code. + ```js + // webpack.config.js + module.exports = { + ... + cache: 'filesystem' // or 'memory' + } + ``` + Since webpack already caches loader results, it is recommended that you enable the webpack builtin cache and disable the babel-loader cache. In rare circumstances, such as when there is an uncacheable loader applied after babel-loader, the babel-loader cache can improve the build performance. + + If you want to implement your own webpack cache backend, such as redis or lmdb, see [`./test/loader.test.js`](./test/loader.test.js) and search `custom webpack cache plugin` for an example. * `cacheIdentifier`: Default is a string composed by the `@babel/core`'s version and the `babel-loader`'s version. The final cache id will be determined by the input file path, the [merged](https://babeljs.io/docs/configuration#how-babel-merges-config-items) Babel config via `Babel.loadPartialConfigAsync` and the `cacheIdentifier`. The merged Babel config will be determined by the `babel.config.js` or `.babelrc` file if they exist, or the value of the environment variable `BABEL_ENV` and `NODE_ENV`. `cacheIdentifier` can be set to a custom value to force cache busting if the identifier changes. -* `cacheCompression`: Default `true`. When set, each Babel transform output will be compressed with Gzip. If you want to opt-out of cache compression, set it to `false` -- your project may benefit from this if it transpiles thousands of files. - * `customize`: Default `null`. The path of a module that exports a `custom` callback [like the one that you'd pass to `.custom()`](#customized-loader). Since you already have to make a new file to use this, it is recommended that you instead use `.custom` to create a wrapper loader. Only use this if you _must_ continue using `babel-loader` directly, but still want to customize. * `metadataSubscribers`: Default `[]`. Takes an array of context function names. E.g. if you passed ['myMetadataPlugin'], you'd assign a subscriber function to `context.myMetadataPlugin` within your webpack plugin's hooks & that function will be called with `metadata`. See [`./test/metadata.test.js`](./test/metadata.test.js) for an example.