feat: Add an optional cache that speeds up consecutive runs (#38)

used like: ngx-translate-extract --cache-file node_modules/.i18n-cache/my-cache-file --input ./src --output ./src/i18n/{da,en}.json

README.md

@@ -69,6 +69,15 @@ Useful when loading multiple translation files in the same application and prefi
 ngx-translate-extract --input ./src --output ./src/i18n/{da,en}.json --strip-prefix 'PREFIX.'
 ```
 
+**Cache for consecutive runs**
+
+If your project grows rather large, runs can take seconds. With this cache, unchanged files don't need
+to be parsed again, keeping consecutive runs under .5 seconds.
+
+```bash
+ngx-translate-extract --cache-file node_modules/.i18n-cache/my-cache-file --input ./src --output ./src/i18n/{da,en}.json
+```
+
 ### JSON indentation
 
 Tabs are used by default for indentation when saving extracted strings in json formats:
@@ -117,6 +126,7 @@ Options:
                  multiple paths                                               [array] [required] [default: ["./"]]
   --output, -o   Paths where you would like to save extracted strings. You can use path expansion, glob
                  patterns and multiple paths                                                    [array] [required]
+  --cache-file   Cache parse results to speed up consecutive runs                                         [string]
   --marker, -m   Custom marker function name                                                              [string]
 
 Examples:

src/cache/cache-interface.ts

@@ -0,0 +1,4 @@
+export interface CacheInterface<RESULT extends object = object> {
+	persist(): void;
+	get<KEY extends string>(uniqueContents: KEY, generator: () => RESULT): RESULT;
+}

src/cache/file-cache.ts

@@ -0,0 +1,84 @@
+import type { CacheInterface } from './cache-interface.js';
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const getHash = (value: string) => crypto.createHash('sha256').update(value).digest('hex');
+
+export class FileCache<RESULT extends object = object> implements CacheInterface<RESULT> {
+	private tapped: Record<string, RESULT> = {};
+	private cached?: Readonly<Record<string, RESULT>> = undefined;
+	private originalCache?: string;
+	private versionHash?: string;
+
+	constructor(private cacheFile: string) {}
+
+	public get<KEY extends string>(uniqueContents: KEY, generator: () => RESULT): RESULT {
+		if (!this.cached) {
+			this.readCache();
+			this.versionHash = this.getVersionHash();
+		}
+
+		const key = getHash(`${this.versionHash}${uniqueContents}`);
+
+		if (key in this.cached) {
+			this.tapped[key] = this.cached[key];
+
+			return this.cached[key];
+		}
+
+		return (this.tapped[key] = generator());
+	}
+
+	public persist(): void {
+		const newCache = JSON.stringify(this.sortByKey(this.tapped), null, 2);
+		if (newCache === this.originalCache) {
+			return;
+		}
+
+		const file = this.getCacheFile();
+		const dir = path.dirname(file);
+
+		const stats = fs.statSync(dir, { throwIfNoEntry: false });
+		if (!stats) {
+			fs.mkdirSync(dir);
+		}
+
+		const tmpFile = `${file}~${getHash(newCache)}`;
+
+		fs.writeFileSync(tmpFile, newCache, { encoding: 'utf-8' });
+		fs.rmSync(file, { force: true, recursive: false });
+		fs.renameSync(tmpFile, file);
+	}
+
+	private sortByKey(unordered: Record<string, RESULT>): Record<string, RESULT> {
+		return Object.keys(unordered)
+			.sort()
+			.reduce((obj, key) => {
+				obj[key] = unordered[key];
+				return obj;
+			}, {} as Record<string, RESULT>);
+	}
+
+	private readCache(): void {
+		try {
+			this.originalCache = fs.readFileSync(this.getCacheFile(), { encoding: 'utf-8' });
+			this.cached = JSON.parse(this.originalCache) ?? {};
+		} catch {
+			this.originalCache = undefined;
+			this.cached = {};
+		}
+	}
+
+	private getVersionHash(): string {
+		const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+		const packageJson = fs.readFileSync(path.join(projectRoot, 'package.json'), { encoding: 'utf-8' });
+
+		return getHash(packageJson);
+	}
+
+	private getCacheFile(): string {
+		return `${this.cacheFile}-ngx-translate-extract-cache.json`;
+	}
+}

src/cache/null-cache.ts

@@ -0,0 +1,8 @@
+import { CacheInterface } from './cache-interface.js';
+
+export class NullCache<RESULT extends object = object> implements CacheInterface<RESULT> {
+	persist() {}
+	get<KEY extends string>(_uniqueContents: KEY, generator: () => RESULT): RESULT {
+		return generator();
+	}
+}

src/cli/cli.ts

@@ -18,6 +18,8 @@ import { StripPrefixPostProcessor } from '../post-processors/strip-prefix.post-p
 import { CompilerInterface } from '../compilers/compiler.interface.js';
 import { CompilerFactory } from '../compilers/compiler.factory.js';
 import { normalizePaths } from '../utils/fs-helpers.js';
+import { FileCache } from '../cache/file-cache.js';
+import { TranslationType } from '../utils/translation.collection.js';
 
 // First parsing pass to be able to access pattern argument for use input/output arguments
 const y = yargs().option('patterns', {
@@ -83,6 +85,10 @@ const cli = await y
 		describe: 'Remove obsolete strings after merge',
 		type: 'boolean'
 	})
+	.option('cache-file', {
+		describe: 'Cache parse results to speed up consecutive runs',
+		type: 'string'
+	})
 	.option('marker', {
 		alias: 'm',
 		describe: 'Name of a custom marker function for extracting strings',
@@ -139,6 +145,10 @@ if (cli.marker) {
 }
 extractTask.setParsers(parsers);
 
+if (cli.cacheFile) {
+	extractTask.setCache(new FileCache<TranslationType[]>(cli.cacheFile));
+}
+
 // Post processors
 const postProcessors: PostProcessorInterface[] = [];
 if (cli.clean) {

src/cli/tasks/extract.task.ts

@@ -3,11 +3,13 @@ import { globSync } from 'glob';
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { TranslationCollection } from '../../utils/translation.collection.js';
+import { TranslationCollection, TranslationType } from '../../utils/translation.collection.js';
 import { TaskInterface } from './task.interface.js';
 import { ParserInterface } from '../../parsers/parser.interface.js';
 import { PostProcessorInterface } from '../../post-processors/post-processor.interface.js';
 import { CompilerInterface } from '../../compilers/compiler.interface.js';
+import type { CacheInterface } from '../../cache/cache-interface.js';
+import { NullCache } from '../../cache/null-cache.js';
 
 export interface ExtractTaskOptionsInterface {
 	replace?: boolean;
@@ -21,6 +23,7 @@ export class ExtractTask implements TaskInterface {
 	protected parsers: ParserInterface[] = [];
 	protected postProcessors: PostProcessorInterface[] = [];
 	protected compiler: CompilerInterface;
+	protected cache: CacheInterface<TranslationType[]> = new NullCache<TranslationType[]>();
 
 	public constructor(protected inputs: string[], protected outputs: string[], options?: ExtractTaskOptionsInterface) {
 		this.inputs = inputs.map((input) => path.resolve(input));
@@ -83,13 +86,20 @@ export class ExtractTask implements TaskInterface {
 				throw e;
 			}
 		});
+
+		this.cache.persist();
 	}
 
 	public setParsers(parsers: ParserInterface[]): this {
 		this.parsers = parsers;
 		return this;
 	}
 
+	public setCache(cache: CacheInterface<TranslationType[]>): this {
+		this.cache = cache;
+		return this;
+	}
+
 	public setPostProcessors(postProcessors: PostProcessorInterface[]): this {
 		this.postProcessors = postProcessors;
 		return this;
@@ -104,20 +114,37 @@ export class ExtractTask implements TaskInterface {
 	 * Extract strings from specified input dirs using configured parsers
 	 */
 	protected extract(): TranslationCollection {
-		let collection: TranslationCollection = new TranslationCollection();
+		const collectionTypes: TranslationType[] = [];
+		let skipped = 0;
 		this.inputs.forEach((pattern) => {
 			this.getFiles(pattern).forEach((filePath) => {
-				this.out(dim('- %s'), filePath);
 				const contents: string = fs.readFileSync(filePath, 'utf-8');
-				this.parsers.forEach((parser) => {
-					const extracted = parser.extract(contents, filePath);
-					if (extracted instanceof TranslationCollection) {
-						collection = collection.union(extracted);
-					}
+				skipped += 1;
+				const cachedCollectionValues = this.cache.get(`${pattern}:${filePath}:${contents}`, () => {
+					skipped -= 1;
+					this.out(dim('- %s'), filePath);
+					return this.parsers
+						.map((parser) => {
+							const extracted = parser.extract(contents, filePath);
+							return extracted instanceof TranslationCollection ? extracted.values : undefined;
+						})
+						.filter((result): result is TranslationType => result && !!Object.keys(result).length);
 				});
+
+				collectionTypes.push(...cachedCollectionValues);
 			});
 		});
-		return collection;
+
+		if (skipped) {
+			this.out(dim('- %s unchanged files skipped via cache'), skipped);
+		}
+
+		const values: TranslationType = {};
+		for (const collectionType of collectionTypes) {
+			Object.assign(values, collectionType);
+		}
+
+		return new TranslationCollection(values);
 	}
 
 	/**