From 35af7ebef9f408e2cef87621590140325c29db8d Mon Sep 17 00:00:00 2001 From: Alexander von Weiss Date: Mon, 5 Feb 2024 11:14:15 +0100 Subject: [PATCH] feat: Add an optional cache that speeds up consecutive runs (#38) used like: ngx-translate-extract --cache-file node_modules/.i18n-cache/my-cache-file --input ./src --output ./src/i18n/{da,en}.json --- README.md | 10 +++++ src/cache/cache-interface.ts | 4 ++ src/cache/file-cache.ts | 84 +++++++++++++++++++++++++++++++++++ src/cache/null-cache.ts | 8 ++++ src/cli/cli.ts | 10 +++++ src/cli/tasks/extract.task.ts | 45 +++++++++++++++---- 6 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 src/cache/cache-interface.ts create mode 100644 src/cache/file-cache.ts create mode 100644 src/cache/null-cache.ts diff --git a/README.md b/README.md index 98d8f794..2ec7dde1 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,15 @@ Useful when loading multiple translation files in the same application and prefi ngx-translate-extract --input ./src --output ./src/i18n/{da,en}.json --strip-prefix 'PREFIX.' ``` +**Cache for consecutive runs** + +If your project grows rather large, runs can take seconds. With this cache, unchanged files don't need +to be parsed again, keeping consecutive runs under .5 seconds. + +```bash +ngx-translate-extract --cache-file node_modules/.i18n-cache/my-cache-file --input ./src --output ./src/i18n/{da,en}.json +``` + ### JSON indentation Tabs are used by default for indentation when saving extracted strings in json formats: @@ -117,6 +126,7 @@ Options: multiple paths [array] [required] [default: ["./"]] --output, -o Paths where you would like to save extracted strings. You can use path expansion, glob patterns and multiple paths [array] [required] + --cache-file Cache parse results to speed up consecutive runs [string] --marker, -m Custom marker function name [string] Examples: diff --git a/src/cache/cache-interface.ts b/src/cache/cache-interface.ts new file mode 100644 index 00000000..a63be81c --- /dev/null +++ b/src/cache/cache-interface.ts @@ -0,0 +1,4 @@ +export interface CacheInterface { + persist(): void; + get(uniqueContents: KEY, generator: () => RESULT): RESULT; +} diff --git a/src/cache/file-cache.ts b/src/cache/file-cache.ts new file mode 100644 index 00000000..d709a5eb --- /dev/null +++ b/src/cache/file-cache.ts @@ -0,0 +1,84 @@ +import type { CacheInterface } from './cache-interface.js'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const getHash = (value: string) => crypto.createHash('sha256').update(value).digest('hex'); + +export class FileCache implements CacheInterface { + private tapped: Record = {}; + private cached?: Readonly> = undefined; + private originalCache?: string; + private versionHash?: string; + + constructor(private cacheFile: string) {} + + public get(uniqueContents: KEY, generator: () => RESULT): RESULT { + if (!this.cached) { + this.readCache(); + this.versionHash = this.getVersionHash(); + } + + const key = getHash(`${this.versionHash}${uniqueContents}`); + + if (key in this.cached) { + this.tapped[key] = this.cached[key]; + + return this.cached[key]; + } + + return (this.tapped[key] = generator()); + } + + public persist(): void { + const newCache = JSON.stringify(this.sortByKey(this.tapped), null, 2); + if (newCache === this.originalCache) { + return; + } + + const file = this.getCacheFile(); + const dir = path.dirname(file); + + const stats = fs.statSync(dir, { throwIfNoEntry: false }); + if (!stats) { + fs.mkdirSync(dir); + } + + const tmpFile = `${file}~${getHash(newCache)}`; + + fs.writeFileSync(tmpFile, newCache, { encoding: 'utf-8' }); + fs.rmSync(file, { force: true, recursive: false }); + fs.renameSync(tmpFile, file); + } + + private sortByKey(unordered: Record): Record { + return Object.keys(unordered) + .sort() + .reduce((obj, key) => { + obj[key] = unordered[key]; + return obj; + }, {} as Record); + } + + private readCache(): void { + try { + this.originalCache = fs.readFileSync(this.getCacheFile(), { encoding: 'utf-8' }); + this.cached = JSON.parse(this.originalCache) ?? {}; + } catch { + this.originalCache = undefined; + this.cached = {}; + } + } + + private getVersionHash(): string { + const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); + const packageJson = fs.readFileSync(path.join(projectRoot, 'package.json'), { encoding: 'utf-8' }); + + return getHash(packageJson); + } + + private getCacheFile(): string { + return `${this.cacheFile}-ngx-translate-extract-cache.json`; + } +} diff --git a/src/cache/null-cache.ts b/src/cache/null-cache.ts new file mode 100644 index 00000000..b526b2e7 --- /dev/null +++ b/src/cache/null-cache.ts @@ -0,0 +1,8 @@ +import { CacheInterface } from './cache-interface.js'; + +export class NullCache implements CacheInterface { + persist() {} + get(_uniqueContents: KEY, generator: () => RESULT): RESULT { + return generator(); + } +} diff --git a/src/cli/cli.ts b/src/cli/cli.ts index 60052553..ded7af5d 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -18,6 +18,8 @@ import { StripPrefixPostProcessor } from '../post-processors/strip-prefix.post-p import { CompilerInterface } from '../compilers/compiler.interface.js'; import { CompilerFactory } from '../compilers/compiler.factory.js'; import { normalizePaths } from '../utils/fs-helpers.js'; +import { FileCache } from '../cache/file-cache.js'; +import { TranslationType } from '../utils/translation.collection.js'; // First parsing pass to be able to access pattern argument for use input/output arguments const y = yargs().option('patterns', { @@ -83,6 +85,10 @@ const cli = await y describe: 'Remove obsolete strings after merge', type: 'boolean' }) + .option('cache-file', { + describe: 'Cache parse results to speed up consecutive runs', + type: 'string' + }) .option('marker', { alias: 'm', describe: 'Name of a custom marker function for extracting strings', @@ -139,6 +145,10 @@ if (cli.marker) { } extractTask.setParsers(parsers); +if (cli.cacheFile) { + extractTask.setCache(new FileCache(cli.cacheFile)); +} + // Post processors const postProcessors: PostProcessorInterface[] = []; if (cli.clean) { diff --git a/src/cli/tasks/extract.task.ts b/src/cli/tasks/extract.task.ts index 492ce7c3..a7585ef1 100644 --- a/src/cli/tasks/extract.task.ts +++ b/src/cli/tasks/extract.task.ts @@ -3,11 +3,13 @@ import { globSync } from 'glob'; import * as fs from 'fs'; import * as path from 'path'; -import { TranslationCollection } from '../../utils/translation.collection.js'; +import { TranslationCollection, TranslationType } from '../../utils/translation.collection.js'; import { TaskInterface } from './task.interface.js'; import { ParserInterface } from '../../parsers/parser.interface.js'; import { PostProcessorInterface } from '../../post-processors/post-processor.interface.js'; import { CompilerInterface } from '../../compilers/compiler.interface.js'; +import type { CacheInterface } from '../../cache/cache-interface.js'; +import { NullCache } from '../../cache/null-cache.js'; export interface ExtractTaskOptionsInterface { replace?: boolean; @@ -21,6 +23,7 @@ export class ExtractTask implements TaskInterface { protected parsers: ParserInterface[] = []; protected postProcessors: PostProcessorInterface[] = []; protected compiler: CompilerInterface; + protected cache: CacheInterface = new NullCache(); public constructor(protected inputs: string[], protected outputs: string[], options?: ExtractTaskOptionsInterface) { this.inputs = inputs.map((input) => path.resolve(input)); @@ -83,6 +86,8 @@ export class ExtractTask implements TaskInterface { throw e; } }); + + this.cache.persist(); } public setParsers(parsers: ParserInterface[]): this { @@ -90,6 +95,11 @@ export class ExtractTask implements TaskInterface { return this; } + public setCache(cache: CacheInterface): this { + this.cache = cache; + return this; + } + public setPostProcessors(postProcessors: PostProcessorInterface[]): this { this.postProcessors = postProcessors; return this; @@ -104,20 +114,37 @@ export class ExtractTask implements TaskInterface { * Extract strings from specified input dirs using configured parsers */ protected extract(): TranslationCollection { - let collection: TranslationCollection = new TranslationCollection(); + const collectionTypes: TranslationType[] = []; + let skipped = 0; this.inputs.forEach((pattern) => { this.getFiles(pattern).forEach((filePath) => { - this.out(dim('- %s'), filePath); const contents: string = fs.readFileSync(filePath, 'utf-8'); - this.parsers.forEach((parser) => { - const extracted = parser.extract(contents, filePath); - if (extracted instanceof TranslationCollection) { - collection = collection.union(extracted); - } + skipped += 1; + const cachedCollectionValues = this.cache.get(`${pattern}:${filePath}:${contents}`, () => { + skipped -= 1; + this.out(dim('- %s'), filePath); + return this.parsers + .map((parser) => { + const extracted = parser.extract(contents, filePath); + return extracted instanceof TranslationCollection ? extracted.values : undefined; + }) + .filter((result): result is TranslationType => result && !!Object.keys(result).length); }); + + collectionTypes.push(...cachedCollectionValues); }); }); - return collection; + + if (skipped) { + this.out(dim('- %s unchanged files skipped via cache'), skipped); + } + + const values: TranslationType = {}; + for (const collectionType of collectionTypes) { + Object.assign(values, collectionType); + } + + return new TranslationCollection(values); } /**