add cache flag, better api

diplodoc-platform · Sep 19, 2023 · 1e36404 · 1e36404
1 parent 8a6c99f
commit 1e36404
Show file tree

Hide file tree

Showing 10 changed files with 96 additions and 49 deletions.
diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts
@@ -171,6 +171,12 @@ function builder<T>(argv: Argv<T>) {
             type: 'string',
             group: 'Build options:',
         })
+        .option('cache', {
+            default: false,
+            describe: 'Enable cache',
+            type: 'boolean',
+            group: 'Build options:',
+        })
         .check(argvValidator)
         .example('yfm -i ./input -o ./output', '')
         .demandOption(['input', 'output'], 'Please provide input and output arguments to work with this tool');
@@ -181,9 +187,9 @@ async function handler(args: Arguments<any>) {
     const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
     const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
 
-    cacheServiceLint.init(args.cacheDir);
-    cacheServiceBuildMd.init(args.cacheDir);
-    cacheServiceMdToHtml.init(args.cacheDir);
+    cacheServiceLint.init(args.cache, args.cacheDir);
+    cacheServiceBuildMd.init(args.cache, args.cacheDir);
+    cacheServiceMdToHtml.init(args.cache, args.cacheDir);
 
     try {
         ArgvService.init({

diff --git a/src/models.ts b/src/models.ts
@@ -46,6 +46,7 @@ export interface YfmArgv extends YfmConfig {
     rootInput: string;
     input: string;
     output: string;
+    cache: boolean;
     cacheDir: string;
     quiet: string;
     publish: boolean;

diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts
@@ -12,7 +12,7 @@ import {ArgvService, PluginService} from '../services';
 import {getVarsPerFileWithHash, getVarsPerRelativeFile} from '../utils';
 import {liquidMd2Html} from './md2html';
 import {liquidMd2Md} from './md2md';
-import {CacheService, cacheServiceLint} from '../services/cache';
+import {cacheServiceLint} from '../services/cache';
 import PluginEnvApi from '../utils/pluginEnvApi';
 import {checkLogWithoutProblems, getLogState} from '../services/utils';
 
@@ -72,19 +72,18 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void
     const path: string = resolve(input, filePath);
     let preparedContent = content;
 
-    const contentHash = CacheService.getHash(content);
-    const cacheKey = CacheService.getHashKey(filePath, contentHash, varsHashList);
+    const cacheKey = cacheServiceLint.getHashKey({filename: filePath, content, varsHashList});
 
     const cachedFile = cacheServiceLint.checkFile(cacheKey);
     if (cachedFile) {
         return;
     }
 
-    const cacheFile = cacheServiceLint.createFile(cacheKey, filePath, contentHash, varsHashList);
+    const cacheFile = cacheServiceLint.createFile(cacheKey);
 
     const envApi = PluginEnvApi.create({
         root,
-        distRoot: '/dev/null',
+        distRoot: '',
         cacheFile,
     });
     const logState = getLogState(log);

diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts
@@ -19,7 +19,7 @@ import {PROCESSING_FINISHED, Lang} from '../constants';
 import {getAssetsPublicPath, getUpdatedMetadata} from '../services/metadata';
 import {MarkdownItPluginCb} from '@doc-tools/transform/lib/plugins/typings';
 import PluginEnvApi from '../utils/pluginEnvApi';
-import {CacheService, cacheServiceMdToHtml} from '../services/cache';
+import {cacheServiceMdToHtml} from '../services/cache';
 import {checkLogWithoutProblems, getLogState} from '../services/utils';
 
 export interface FileTransformOptions {
@@ -137,16 +137,15 @@ async function MdFileTransformer(content: string, transformOptions: FileTransfor
     const root = resolve(input);
     const path: string = resolve(input, filePath);
 
-    const contentHash = CacheService.getHash(content);
-    const cacheKey = CacheService.getHashKey(filePath, contentHash, varsHashList);
+    const cacheKey = cacheServiceMdToHtml.getHashKey({filename: filePath, content, varsHashList});
 
     const cachedFile = await cacheServiceMdToHtml.checkFileAsync(cacheKey);
     if (cachedFile) {
         await cachedFile.extractCacheAsync();
         return cachedFile.getResult<Output>();
     }
 
-    const cacheFile = cacheServiceMdToHtml.createFile(cacheKey, filePath, contentHash, varsHashList);
+    const cacheFile = cacheServiceMdToHtml.createFile(cacheKey);
     const envApi = PluginEnvApi.create({
         root: resolve(input),
         distRoot: resolve(options.output),

diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts
@@ -10,7 +10,7 @@ import {PluginOptions, ResolveMd2MdOptions} from '../models';
 import {PROCESSING_FINISHED} from '../constants';
 import {getContentWithUpdatedMetadata} from '../services/metadata';
 import {ChangelogItem} from '@doc-tools/transform/lib/plugins/changelog/types';
-import {CacheService, cacheServiceBuildMd} from '../services/cache';
+import {cacheServiceBuildMd} from '../services/cache';
 import PluginEnvApi from '../utils/pluginEnvApi';
 import {checkLogWithoutProblems, getLogState} from '../services/utils';
 
@@ -21,9 +21,8 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void>
     const {vars, varsHashList} = getVarsPerFileWithHash(inputPath);
 
     const rawContent = readFileSync(resolvedInputPath, 'utf8');
-    const rawContentHash = CacheService.getHash(rawContent);
 
-    const cacheKey = CacheService.getHashKey(inputPath, rawContentHash, varsHashList);
+    const cacheKey = cacheServiceBuildMd.getHashKey({filename: inputPath, content: rawContent, varsHashList});
 
     let result: string;
     let changelogs: ChangelogItem[];
@@ -41,7 +40,7 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void>
             vars.__system,
         );
 
-        const cacheFile = cacheServiceBuildMd.createFile(cacheKey, inputPath, rawContentHash, varsHashList);
+        const cacheFile = cacheServiceBuildMd.createFile(cacheKey);
         const envApi = PluginEnvApi.create({
             root: resolve(input),
             distRoot: resolve(output),

diff --git a/src/services/cache/cache.ts b/src/services/cache/cache.ts
@@ -5,11 +5,14 @@ import {ArgvService} from '../index';
 import {pick} from 'lodash';
 import path from 'path';
 import {fileExists} from '../../utils';
+import {HashKey} from './types';
 
 const objHash = new WeakMap<Object, string>();
 const fileHash = new Map<string, string>();
 const existsDir = new Set<string>();
 
+type GetHashKeyProps = Omit<HashKey, 'key' | 'contentHash'> & {content: string};
+
 let argsHash = '';
 
 export class CacheService {
@@ -44,7 +47,7 @@ export class CacheService {
         return hash;
     }
 
-    static getHashKey(filename: string, contentHash: string, varsHashList: string[], extra?: Record<string, unknown>) {
+    static getHashKey({filename, content, varsHashList}: GetHashKeyProps): HashKey {
         if (!argsHash) {
             const args = ArgvService.getConfig();
             const staticArgs = pick(args, [
@@ -55,21 +58,29 @@ export class CacheService {
             ]);
             argsHash = CacheService.getHash(JSON.stringify(staticArgs));
         }
-        return this.getHash(JSON.stringify({filename, contentHash, varsHashList, argsHash, extra}));
+        const contentHash = CacheService.getHash(content);
+        return {
+            key: this.getHash(JSON.stringify({filename, contentHash, varsHashList, argsHash})),
+            filename, contentHash, varsHashList,
+        };
     }
 
     private readonly storeName;
     private cacheDir = '';
+    private disabled = false;
 
     constructor(storeName = 'main') {
         this.storeName = storeName;
     }
 
-    init(cacheDir: string) {
+    init(enabled: boolean, cacheDir: string) {
+        this.disabled = !enabled;
         this.cacheDir = path.resolve(cacheDir);
     }
 
-    checkFile(key: string) {
+    checkFile({key}: HashKey) {
+        if (this.disabled) { return; }
+
         const filepath = this.getCacheFilepath(key);
         if (!fs.existsSync(filepath)) {
             return;
@@ -78,14 +89,16 @@ export class CacheService {
         try {
             const dataJson = fs.readFileSync(filepath, 'utf-8');
             const data = JSON.parse(dataJson);
-            file = CacheFile.from(data);
+            file = CacheFile.from(data, this.disabled);
         } catch (err) {
             return;
         }
         return file?.check() ? file : undefined;
     }
 
-    async checkFileAsync(key: string) {
+    async checkFileAsync({key}: HashKey) {
+        if (this.disabled) { return; }
+
         const filepath = this.getCacheFilepath(key);
         const exists = await fileExists(filepath);
         if (!exists) {
@@ -95,19 +108,21 @@ export class CacheService {
         try {
             const dataJson = await fs.promises.readFile(filepath, 'utf-8');
             const data = JSON.parse(dataJson);
-            file = CacheFile.from(data);
+            file = CacheFile.from(data, this.disabled);
         } catch (err) {
             return;
         }
         const isCorrect = await file?.checkAsync();
         return isCorrect ? file : undefined;
     }
 
-    createFile(key: string, filename: string, contentHash: string, varsHashList: string[]) {
-        return new CacheFile({key, filename, contentHash, varsHashList});
+    createFile(key: HashKey) {
+        return new CacheFile(key, this.disabled);
     }
 
     addFile(file: CacheFile) {
+        if (this.disabled) { return; }
+
         const filepath = this.getCacheFilepath(file.getKey());
         const place = path.dirname(filepath);
         if (!existsDir.has(place)) {
@@ -118,6 +133,8 @@ export class CacheService {
     }
 
     async addFileAsync(file: CacheFile) {
+        if (this.disabled) { return; }
+
         const filepath = this.getCacheFilepath(file.getKey());
         const place = path.dirname(filepath);
         if (!existsDir.has(place)) {
@@ -127,6 +144,20 @@ export class CacheService {
         await fs.promises.writeFile(filepath, JSON.stringify(file.toJSON()));
     }
 
+    getHashKey(props: GetHashKeyProps) {
+        if (this.disabled) {
+            const {filename, varsHashList} = props;
+            return {
+                key: '',
+                contentHash: '',
+                filename,
+                varsHashList,
+            };
+        }
+
+        return CacheService.getHashKey(props);
+    }
+
     private getCacheFilepath(key: string) {
         return path.join(this.cacheDir, this.storeName, key.slice(0, 2), key);
     }

diff --git a/src/services/cache/cacheFile.ts b/src/services/cache/cacheFile.ts
@@ -5,38 +5,23 @@ import isEqual from 'lodash/isEqual';
 import * as fs from 'fs';
 import path from 'path';
 import {asyncify, mapLimit, parallelLimit} from 'async';
+import {CacheFileData, CacheFileDataWithDeps, Deps} from './types';
 
 const CUNCURRENCY = 1000;
 
 type CacheFileProps = CacheFileData & Partial<Deps>;
-type CacheFileDataWithDeps = CacheFileData & Deps;
-type TargetLocation = string;
-type SourceLocation = string;
-
-export interface CacheFileData {
-    key: string;
-    filename: string;
-    contentHash: string;
-    varsHashList: string[];
-    result?: unknown;
-}
-
-export interface Deps {
-    fileDeps: Record<SourceLocation, string>;
-    wroteFiles: Record<TargetLocation, string>;
-    copiedFiles: Record<TargetLocation, SourceLocation>;
-    existsFiles: Record<SourceLocation, boolean>;
-    fileVarsDeps: Record<SourceLocation, string[]>;
-}
 
 class CacheFile {
-    static from(data: CacheFileDataWithDeps) {
-        return new CacheFile(data);
+    static from(data: CacheFileDataWithDeps, disabled: boolean) {
+        return new CacheFile(data, disabled);
     }
 
+    disabled = false;
+
     private data: CacheFileDataWithDeps;
 
-    constructor(data: CacheFileProps) {
+    constructor(data: CacheFileProps, disabled: boolean) {
+        this.disabled = disabled;
         this.data = {
             ...data,
             fileDeps: data.fileDeps || {},
@@ -47,6 +32,11 @@ class CacheFile {
         };
     }
 
+    use() {
+        if (this.disabled) { return undefined; }
+        return this;
+    }
+
     getKey() {
         return this.data.key;
     }

diff --git a/src/services/cache/types.ts b/src/services/cache/types.ts
@@ -0,0 +1,22 @@
+export interface HashKey {
+    filename: string;
+    contentHash: string;
+    varsHashList: string[];
+    key: string;
+}
+
+export type CacheFileDataWithDeps = CacheFileData & Deps;
+type TargetLocation = string;
+type SourceLocation = string;
+
+export interface CacheFileData extends HashKey {
+    result?: unknown;
+}
+
+export interface Deps {
+    fileDeps: Record<SourceLocation, string>;
+    wroteFiles: Record<TargetLocation, string>;
+    copiedFiles: Record<TargetLocation, SourceLocation>;
+    existsFiles: Record<SourceLocation, boolean>;
+    fileVarsDeps: Record<SourceLocation, string[]>;
+}
diff --git a/src/utils/pluginEnvApi.ts b/src/utils/pluginEnvApi.ts
@@ -20,7 +20,7 @@ class PluginEnvApi {
     constructor({root, distRoot, cacheFile}: PluginEnvApiProps) {
         this.root = root;
         this.distRoot = distRoot;
-        this.cacheFile = cacheFile;
+        this.cacheFile = cacheFile?.use();
     }
 
     copyFileSync(rawFrom: string, rawTo: string) {

diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts
@@ -28,7 +28,7 @@ async function run({
     TocService.setNavigationPaths(navigationPaths);
     PluginService.setPlugins();
 
-    cacheServiceLint.init(argvConfig.cacheDir);
+    cacheServiceLint.init(argvConfig.cache, argvConfig.cacheDir);
 
     TocService.getNavigationPaths().forEach((pathToFile) => {
         lintPage({