From 1e3640496981bef0e4f81566744b0ffd57156236 Mon Sep 17 00:00:00 2001 From: Anton Vikulov Date: Tue, 19 Sep 2023 23:01:24 +0500 Subject: [PATCH] add cache flag, better api --- src/cmd/build/index.ts | 12 ++++++-- src/models.ts | 1 + src/resolvers/lintPage.ts | 9 +++--- src/resolvers/md2html.ts | 7 ++--- src/resolvers/md2md.ts | 7 ++--- src/services/cache/cache.ts | 49 +++++++++++++++++++++++++++------ src/services/cache/cacheFile.ts | 34 ++++++++--------------- src/services/cache/types.ts | 22 +++++++++++++++ src/utils/pluginEnvApi.ts | 2 +- src/workers/linter/index.ts | 2 +- 10 files changed, 96 insertions(+), 49 deletions(-) create mode 100644 src/services/cache/types.ts diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index d938a53d8..f9e679b4a 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -171,6 +171,12 @@ function builder(argv: Argv) { type: 'string', group: 'Build options:', }) + .option('cache', { + default: false, + describe: 'Enable cache', + type: 'boolean', + group: 'Build options:', + }) .check(argvValidator) .example('yfm -i ./input -o ./output', '') .demandOption(['input', 'output'], 'Please provide input and output arguments to work with this tool'); @@ -181,9 +187,9 @@ async function handler(args: Arguments) { const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); - cacheServiceLint.init(args.cacheDir); - cacheServiceBuildMd.init(args.cacheDir); - cacheServiceMdToHtml.init(args.cacheDir); + cacheServiceLint.init(args.cache, args.cacheDir); + cacheServiceBuildMd.init(args.cache, args.cacheDir); + cacheServiceMdToHtml.init(args.cache, args.cacheDir); try { ArgvService.init({ diff --git a/src/models.ts b/src/models.ts index 6a0658e53..374675879 100644 --- a/src/models.ts +++ b/src/models.ts @@ -46,6 +46,7 @@ export interface YfmArgv extends YfmConfig { rootInput: string; input: string; output: string; + cache: boolean; cacheDir: string; quiet: string; publish: boolean; diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index fe3faf1c9..0720f6824 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -12,7 +12,7 @@ import {ArgvService, PluginService} from '../services'; import {getVarsPerFileWithHash, getVarsPerRelativeFile} from '../utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; -import {CacheService, cacheServiceLint} from '../services/cache'; +import {cacheServiceLint} from '../services/cache'; import PluginEnvApi from '../utils/pluginEnvApi'; import {checkLogWithoutProblems, getLogState} from '../services/utils'; @@ -72,19 +72,18 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void const path: string = resolve(input, filePath); let preparedContent = content; - const contentHash = CacheService.getHash(content); - const cacheKey = CacheService.getHashKey(filePath, contentHash, varsHashList); + const cacheKey = cacheServiceLint.getHashKey({filename: filePath, content, varsHashList}); const cachedFile = cacheServiceLint.checkFile(cacheKey); if (cachedFile) { return; } - const cacheFile = cacheServiceLint.createFile(cacheKey, filePath, contentHash, varsHashList); + const cacheFile = cacheServiceLint.createFile(cacheKey); const envApi = PluginEnvApi.create({ root, - distRoot: '/dev/null', + distRoot: '', cacheFile, }); const logState = getLogState(log); diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 4c26cf320..24e6136ba 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -19,7 +19,7 @@ import {PROCESSING_FINISHED, Lang} from '../constants'; import {getAssetsPublicPath, getUpdatedMetadata} from '../services/metadata'; import {MarkdownItPluginCb} from '@doc-tools/transform/lib/plugins/typings'; import PluginEnvApi from '../utils/pluginEnvApi'; -import {CacheService, cacheServiceMdToHtml} from '../services/cache'; +import {cacheServiceMdToHtml} from '../services/cache'; import {checkLogWithoutProblems, getLogState} from '../services/utils'; export interface FileTransformOptions { @@ -137,8 +137,7 @@ async function MdFileTransformer(content: string, transformOptions: FileTransfor const root = resolve(input); const path: string = resolve(input, filePath); - const contentHash = CacheService.getHash(content); - const cacheKey = CacheService.getHashKey(filePath, contentHash, varsHashList); + const cacheKey = cacheServiceMdToHtml.getHashKey({filename: filePath, content, varsHashList}); const cachedFile = await cacheServiceMdToHtml.checkFileAsync(cacheKey); if (cachedFile) { @@ -146,7 +145,7 @@ async function MdFileTransformer(content: string, transformOptions: FileTransfor return cachedFile.getResult(); } - const cacheFile = cacheServiceMdToHtml.createFile(cacheKey, filePath, contentHash, varsHashList); + const cacheFile = cacheServiceMdToHtml.createFile(cacheKey); const envApi = PluginEnvApi.create({ root: resolve(input), distRoot: resolve(options.output), diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index 044ba39ba..4afac10a1 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -10,7 +10,7 @@ import {PluginOptions, ResolveMd2MdOptions} from '../models'; import {PROCESSING_FINISHED} from '../constants'; import {getContentWithUpdatedMetadata} from '../services/metadata'; import {ChangelogItem} from '@doc-tools/transform/lib/plugins/changelog/types'; -import {CacheService, cacheServiceBuildMd} from '../services/cache'; +import {cacheServiceBuildMd} from '../services/cache'; import PluginEnvApi from '../utils/pluginEnvApi'; import {checkLogWithoutProblems, getLogState} from '../services/utils'; @@ -21,9 +21,8 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const {vars, varsHashList} = getVarsPerFileWithHash(inputPath); const rawContent = readFileSync(resolvedInputPath, 'utf8'); - const rawContentHash = CacheService.getHash(rawContent); - const cacheKey = CacheService.getHashKey(inputPath, rawContentHash, varsHashList); + const cacheKey = cacheServiceBuildMd.getHashKey({filename: inputPath, content: rawContent, varsHashList}); let result: string; let changelogs: ChangelogItem[]; @@ -41,7 +40,7 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise vars.__system, ); - const cacheFile = cacheServiceBuildMd.createFile(cacheKey, inputPath, rawContentHash, varsHashList); + const cacheFile = cacheServiceBuildMd.createFile(cacheKey); const envApi = PluginEnvApi.create({ root: resolve(input), distRoot: resolve(output), diff --git a/src/services/cache/cache.ts b/src/services/cache/cache.ts index d29be4659..955be9f08 100644 --- a/src/services/cache/cache.ts +++ b/src/services/cache/cache.ts @@ -5,11 +5,14 @@ import {ArgvService} from '../index'; import {pick} from 'lodash'; import path from 'path'; import {fileExists} from '../../utils'; +import {HashKey} from './types'; const objHash = new WeakMap(); const fileHash = new Map(); const existsDir = new Set(); +type GetHashKeyProps = Omit & {content: string}; + let argsHash = ''; export class CacheService { @@ -44,7 +47,7 @@ export class CacheService { return hash; } - static getHashKey(filename: string, contentHash: string, varsHashList: string[], extra?: Record) { + static getHashKey({filename, content, varsHashList}: GetHashKeyProps): HashKey { if (!argsHash) { const args = ArgvService.getConfig(); const staticArgs = pick(args, [ @@ -55,21 +58,29 @@ export class CacheService { ]); argsHash = CacheService.getHash(JSON.stringify(staticArgs)); } - return this.getHash(JSON.stringify({filename, contentHash, varsHashList, argsHash, extra})); + const contentHash = CacheService.getHash(content); + return { + key: this.getHash(JSON.stringify({filename, contentHash, varsHashList, argsHash})), + filename, contentHash, varsHashList, + }; } private readonly storeName; private cacheDir = ''; + private disabled = false; constructor(storeName = 'main') { this.storeName = storeName; } - init(cacheDir: string) { + init(enabled: boolean, cacheDir: string) { + this.disabled = !enabled; this.cacheDir = path.resolve(cacheDir); } - checkFile(key: string) { + checkFile({key}: HashKey) { + if (this.disabled) { return; } + const filepath = this.getCacheFilepath(key); if (!fs.existsSync(filepath)) { return; @@ -78,14 +89,16 @@ export class CacheService { try { const dataJson = fs.readFileSync(filepath, 'utf-8'); const data = JSON.parse(dataJson); - file = CacheFile.from(data); + file = CacheFile.from(data, this.disabled); } catch (err) { return; } return file?.check() ? file : undefined; } - async checkFileAsync(key: string) { + async checkFileAsync({key}: HashKey) { + if (this.disabled) { return; } + const filepath = this.getCacheFilepath(key); const exists = await fileExists(filepath); if (!exists) { @@ -95,7 +108,7 @@ export class CacheService { try { const dataJson = await fs.promises.readFile(filepath, 'utf-8'); const data = JSON.parse(dataJson); - file = CacheFile.from(data); + file = CacheFile.from(data, this.disabled); } catch (err) { return; } @@ -103,11 +116,13 @@ export class CacheService { return isCorrect ? file : undefined; } - createFile(key: string, filename: string, contentHash: string, varsHashList: string[]) { - return new CacheFile({key, filename, contentHash, varsHashList}); + createFile(key: HashKey) { + return new CacheFile(key, this.disabled); } addFile(file: CacheFile) { + if (this.disabled) { return; } + const filepath = this.getCacheFilepath(file.getKey()); const place = path.dirname(filepath); if (!existsDir.has(place)) { @@ -118,6 +133,8 @@ export class CacheService { } async addFileAsync(file: CacheFile) { + if (this.disabled) { return; } + const filepath = this.getCacheFilepath(file.getKey()); const place = path.dirname(filepath); if (!existsDir.has(place)) { @@ -127,6 +144,20 @@ export class CacheService { await fs.promises.writeFile(filepath, JSON.stringify(file.toJSON())); } + getHashKey(props: GetHashKeyProps) { + if (this.disabled) { + const {filename, varsHashList} = props; + return { + key: '', + contentHash: '', + filename, + varsHashList, + }; + } + + return CacheService.getHashKey(props); + } + private getCacheFilepath(key: string) { return path.join(this.cacheDir, this.storeName, key.slice(0, 2), key); } diff --git a/src/services/cache/cacheFile.ts b/src/services/cache/cacheFile.ts index 5bffdc676..bd74064b9 100644 --- a/src/services/cache/cacheFile.ts +++ b/src/services/cache/cacheFile.ts @@ -5,38 +5,23 @@ import isEqual from 'lodash/isEqual'; import * as fs from 'fs'; import path from 'path'; import {asyncify, mapLimit, parallelLimit} from 'async'; +import {CacheFileData, CacheFileDataWithDeps, Deps} from './types'; const CUNCURRENCY = 1000; type CacheFileProps = CacheFileData & Partial; -type CacheFileDataWithDeps = CacheFileData & Deps; -type TargetLocation = string; -type SourceLocation = string; - -export interface CacheFileData { - key: string; - filename: string; - contentHash: string; - varsHashList: string[]; - result?: unknown; -} - -export interface Deps { - fileDeps: Record; - wroteFiles: Record; - copiedFiles: Record; - existsFiles: Record; - fileVarsDeps: Record; -} class CacheFile { - static from(data: CacheFileDataWithDeps) { - return new CacheFile(data); + static from(data: CacheFileDataWithDeps, disabled: boolean) { + return new CacheFile(data, disabled); } + disabled = false; + private data: CacheFileDataWithDeps; - constructor(data: CacheFileProps) { + constructor(data: CacheFileProps, disabled: boolean) { + this.disabled = disabled; this.data = { ...data, fileDeps: data.fileDeps || {}, @@ -47,6 +32,11 @@ class CacheFile { }; } + use() { + if (this.disabled) { return undefined; } + return this; + } + getKey() { return this.data.key; } diff --git a/src/services/cache/types.ts b/src/services/cache/types.ts new file mode 100644 index 000000000..02ec2549c --- /dev/null +++ b/src/services/cache/types.ts @@ -0,0 +1,22 @@ +export interface HashKey { + filename: string; + contentHash: string; + varsHashList: string[]; + key: string; +} + +export type CacheFileDataWithDeps = CacheFileData & Deps; +type TargetLocation = string; +type SourceLocation = string; + +export interface CacheFileData extends HashKey { + result?: unknown; +} + +export interface Deps { + fileDeps: Record; + wroteFiles: Record; + copiedFiles: Record; + existsFiles: Record; + fileVarsDeps: Record; +} diff --git a/src/utils/pluginEnvApi.ts b/src/utils/pluginEnvApi.ts index 60e19f874..46ff0a4d9 100644 --- a/src/utils/pluginEnvApi.ts +++ b/src/utils/pluginEnvApi.ts @@ -20,7 +20,7 @@ class PluginEnvApi { constructor({root, distRoot, cacheFile}: PluginEnvApiProps) { this.root = root; this.distRoot = distRoot; - this.cacheFile = cacheFile; + this.cacheFile = cacheFile?.use(); } copyFileSync(rawFrom: string, rawTo: string) { diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts index 321aaf557..33694fb35 100644 --- a/src/workers/linter/index.ts +++ b/src/workers/linter/index.ts @@ -28,7 +28,7 @@ async function run({ TocService.setNavigationPaths(navigationPaths); PluginService.setPlugins(); - cacheServiceLint.init(argvConfig.cacheDir); + cacheServiceLint.init(argvConfig.cache, argvConfig.cacheDir); TocService.getNavigationPaths().forEach((pathToFile) => { lintPage({