feat(cache): Add fs cache

src/cmd/build/index.ts

@@ -27,6 +27,7 @@ import {Resources} from '../../models';
 import {copyFiles, logger} from '../../utils';
 import {upload as publishFilesToS3} from '../publish/upload';
 import glob from 'glob';
+import {cacheServiceBuildMd, cacheServiceLint, cacheServiceMdToHtml} from '../../services/cache';
 
 export const build = {
     command: ['build', '$0'],
@@ -166,6 +167,18 @@ function builder<T>(argv: Argv<T>) {
             type: 'boolean',
             group: 'Build options:',
         })
+        .option('cache-dir', {
+            default: resolve('cache'),
+            describe: 'Path to cache folder',
+            type: 'string',
+            group: 'Build options:',
+        })
+        .option('cache', {
+            default: false,
+            describe: 'Enable cache',
+            type: 'boolean',
+            group: 'Build options:',
+        })
         .check(argvValidator)
         .example('yfm -i ./input -o ./output', '')
         .demandOption(
@@ -179,6 +192,10 @@ async function handler(args: Arguments<any>) {
     const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
     const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
 
+    cacheServiceLint.init(args.cache, args.cacheDir);
+    cacheServiceBuildMd.init(args.cache, args.cacheDir);
+    cacheServiceMdToHtml.init(args.cache, args.cacheDir);
+
     try {
         ArgvService.init({
             ...args,
@@ -287,7 +304,7 @@ async function handler(args: Arguments<any>) {
             }
         }
     } catch (err) {
-        logger.error('', err.message);
+        logger.error('', (err as Error).message);
     } finally {
         processLogs(tmpInputFolder);
 

src/constants.ts

@@ -97,6 +97,8 @@ export const GETTING_ALL_CONTRIBUTORS = 'Getting all contributors.';
 export const ALL_CONTRIBUTORS_RECEIVED = 'All contributors received.';
 export const getMsgСonfigurationMustBeProvided = (repo: string) =>
     `Сonfiguration must be provided for ${repo} like env variables or in .yfm file`;
+export const CACHE_HIT = 'Cache hit:';
+export const LINT_CACHE_HIT = 'Lint cache hit:';
 
 export const FIRST_COMMIT_FROM_ROBOT_IN_GITHUB = '2dce14271359cd20d7e874956d604de087560cf4';
 

src/models.ts

@@ -4,6 +4,7 @@ import {LintConfig} from '@diplodoc/transform/lib/yfmlint';
 import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models';
 import {Lang, Stage, IncludeMode, ResourceType} from './constants';
 import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types';
+import PluginEnvApi from './utils/pluginEnvApi';
 
 export type VarsPreset = 'internal' | 'external';
 
@@ -48,6 +49,8 @@ export interface YfmArgv extends YfmConfig {
     rootInput: string;
     input: string;
     output: string;
+    cache: boolean;
+    cacheDir: string;
     quiet: string;
     publish: boolean;
     storageEndpoint: string;
@@ -203,6 +206,7 @@ export interface PluginOptions {
     collectOfPlugins?: (input: string, options: PluginOptions) => string;
     changelogs?: ChangelogItem[];
     extractChangelogs?: boolean;
+    envApi?: PluginEnvApi;
 }
 
 export interface Plugin {

src/resolvers/lintPage.ts

@@ -9,9 +9,13 @@ import {readFileSync} from 'fs';
 import {bold} from 'chalk';
 
 import {ArgvService, PluginService} from '../services';
-import {getVarsPerFile, getVarsPerRelativeFile} from '../utils';
+import {getVarsPerFileWithHash, getVarsPerRelativeFile, logger} from '../utils';
 import {liquidMd2Html} from './md2html';
 import {liquidMd2Md} from './md2md';
+import {cacheServiceLint} from '../services/cache';
+import PluginEnvApi from '../utils/pluginEnvApi';
+import {checkLogWithoutProblems, getLogState} from '../services/utils';
+import {LINT_CACHE_HIT} from '../constants';
 
 interface FileTransformOptions {
     path: string;
@@ -58,11 +62,28 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void
     const {path: filePath} = lintOptions;
 
     const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins();
-    const vars = getVarsPerFile(filePath);
+    const {vars, varsHashList} = getVarsPerFileWithHash(filePath);
     const root = resolve(input);
     const path: string = resolve(input, filePath);
     let preparedContent = content;
 
+    const cacheKey = cacheServiceLint.getHashKey({filename: filePath, content, varsHashList});
+
+    const cachedFile = cacheServiceLint.checkFile(cacheKey);
+    if (cachedFile) {
+        logger.info(filePath, LINT_CACHE_HIT);
+        return;
+    }
+
+    const cacheFile = cacheServiceLint.createFile(cacheKey);
+
+    const envApi = PluginEnvApi.create({
+        root,
+        distRoot: '',
+        cacheFile,
+    });
+    const logState = getLogState(log);
+
     /* Relative path from folder of .md file to root of user' output folder */
     const assetsPublicPath = relative(dirname(path), root);
 
@@ -79,6 +100,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void
             disableLiquid,
             log,
             getVarsPerFile: getVarsPerRelativeFile,
+            envApi,
         };
 
         yfmlint({
@@ -110,4 +132,9 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void
         path,
         sourceMap,
     });
+
+    const logIsOk = checkLogWithoutProblems(log, logState);
+    if (logIsOk) {
+        cacheServiceLint.addFile(cacheFile);
+    }
 }

src/resolvers/md2html.ts

@@ -12,12 +12,15 @@ import {
     generateStaticMarkup,
     logger,
     transformToc,
-    getVarsPerFile,
     getVarsPerRelativeFile,
+    getVarsPerFileWithHash,
 } from '../utils';
-import {PROCESSING_FINISHED, Lang} from '../constants';
+import {PROCESSING_FINISHED, Lang, CACHE_HIT} from '../constants';
 import {getAssetsPublicPath, getUpdatedMetadata} from '../services/metadata';
 import {MarkdownItPluginCb} from '@diplodoc/transform/lib/plugins/typings';
+import PluginEnvApi from '../utils/pluginEnvApi';
+import {cacheServiceMdToHtml} from '../services/cache';
+import {checkLogWithoutProblems, getLogState} from '../services/utils';
 
 export interface FileTransformOptions {
     path: string;
@@ -48,7 +51,7 @@ export async function resolveMd2HTML(options: ResolverOptions): Promise<ResolveM
     const content: string = readFileSync(resolvedPath, 'utf8');
 
     const transformFn: Function = FileTransformer[fileExtension];
-    const {result} = transformFn(content, {path: inputPath});
+    const {result} = await transformFn(content, {path: inputPath});
 
     const updatedMetadata =
         metadata && metadata.isContributorsEnabled
@@ -128,16 +131,36 @@ export function liquidMd2Html(input: string, vars: Record<string, unknown>, path
     });
 }
 
-function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output {
+async function MdFileTransformer(
+    content: string,
+    transformOptions: FileTransformOptions,
+): Promise<Output> {
     const {input, ...options} = ArgvService.getConfig();
     const {path: filePath} = transformOptions;
 
     const plugins = PluginService.getPlugins();
-    const vars = getVarsPerFile(filePath);
+    const {vars, varsHashList} = getVarsPerFileWithHash(filePath);
     const root = resolve(input);
     const path: string = resolve(input, filePath);
 
-    return transform(content, {
+    const cacheKey = cacheServiceMdToHtml.getHashKey({filename: filePath, content, varsHashList});
+
+    const cachedFile = await cacheServiceMdToHtml.checkFileAsync(cacheKey);
+    if (cachedFile) {
+        logger.info(filePath, CACHE_HIT);
+        await cachedFile.extractCacheAsync();
+        return cachedFile.getResult<Output>();
+    }
+
+    const cacheFile = cacheServiceMdToHtml.createFile(cacheKey);
+    const envApi = PluginEnvApi.create({
+        root: resolve(input),
+        distRoot: resolve(options.output),
+        cacheFile,
+    });
+    const logState = getLogState(log);
+
+    const result = transform(content, {
         ...options,
         plugins: plugins as MarkdownItPluginCb<unknown>[],
         vars,
@@ -146,5 +169,16 @@ function MdFileTransformer(content: string, transformOptions: FileTransformOptio
         assetsPublicPath: getAssetsPublicPath(filePath),
         getVarsPerFile: getVarsPerRelativeFile,
         extractTitle: true,
+        envApi,
     });
+
+    envApi.executeActions();
+
+    const logIsOk = checkLogWithoutProblems(log, logState);
+    if (logIsOk) {
+        cacheFile.setResult(result);
+        await cacheServiceMdToHtml.addFileAsync(cacheFile);
+    }
+
+    return result;
 }

src/resolvers/md2md.ts

@@ -1,40 +1,84 @@
-import {existsSync, readFileSync, writeFileSync} from 'fs';
-import {dirname, resolve, join, basename, extname} from 'path';
+import * as fs from 'fs';
+import {dirname, resolve, join, basename, extname, relative} from 'path';
 import shell from 'shelljs';
-import log from '@diplodoc/transform/lib/log';
+import log, {LogLevels} from '@diplodoc/transform/lib/log';
 import liquid from '@diplodoc/transform/lib/liquid';
 
 import {ArgvService, PluginService} from '../services';
-import {logger, getVarsPerFile} from '../utils';
+import {logger, getVarsPerFileWithHash} from '../utils';
 import {PluginOptions, ResolveMd2MdOptions} from '../models';
-import {PROCESSING_FINISHED} from '../constants';
+import {CACHE_HIT, PROCESSING_FINISHED} from '../constants';
 import {getContentWithUpdatedMetadata} from '../services/metadata';
 import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types';
+import {cacheServiceBuildMd} from '../services/cache';
+import PluginEnvApi from '../utils/pluginEnvApi';
+import {checkLogWithoutProblems, getLogState} from '../services/utils';
 
 export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void> {
     const {inputPath, outputPath, metadata} = options;
     const {input, output} = ArgvService.getConfig();
     const resolvedInputPath = resolve(input, inputPath);
-    const vars = getVarsPerFile(inputPath);
-
-    const content = await getContentWithUpdatedMetadata(
-        readFileSync(resolvedInputPath, 'utf8'),
-        metadata,
-        vars.__system,
-    );
-
-    const {result, changelogs} = transformMd2Md(content, {
-        path: resolvedInputPath,
-        destPath: outputPath,
-        root: resolve(input),
-        destRoot: resolve(output),
-        collectOfPlugins: PluginService.getCollectOfPlugins(),
-        vars,
-        log,
-        copyFile,
+    const {vars, varsHashList} = getVarsPerFileWithHash(inputPath);
+
+    const rawContent = fs.readFileSync(resolvedInputPath, 'utf8');
+
+    const cacheKey = cacheServiceBuildMd.getHashKey({
+        filename: inputPath,
+        content: rawContent,
+        varsHashList,
     });
 
-    writeFileSync(outputPath, result);
+    let result: string;
+    let changelogs: ChangelogItem[];
+
+    const cachedFile = await cacheServiceBuildMd.checkFileAsync(cacheKey);
+    if (cachedFile) {
+        logger.info(inputPath, CACHE_HIT);
+        await cachedFile.extractCacheAsync();
+        const results = cachedFile.getResult<{
+            result: string;
+            changelogs: ChangelogItem[];
+            logs: Record<LogLevels, string[]>;
+        }>();
+        result = results.result;
+        changelogs = results.changelogs;
+    } else {
+        const content = await getContentWithUpdatedMetadata(rawContent, metadata, vars.__system);
+
+        const cacheFile = cacheServiceBuildMd.createFile(cacheKey);
+        const envApi = PluginEnvApi.create({
+            root: resolve(input),
+            distRoot: resolve(output),
+            cacheFile,
+        });
+        const logState = getLogState(log);
+
+        const transformResult = transformMd2Md(content, {
+            path: resolvedInputPath,
+            destPath: outputPath,
+            root: resolve(input),
+            destRoot: resolve(output),
+            collectOfPlugins: PluginService.getCollectOfPlugins(),
+            vars,
+            log,
+            copyFile,
+            envApi,
+        });
+
+        result = transformResult.result;
+        changelogs = transformResult.changelogs;
+
+        envApi.executeActions();
+
+        const logIsOk = checkLogWithoutProblems(log, logState);
+        if (logIsOk) {
+            cacheFile.setResult(transformResult);
+            // not async cause race condition
+            cacheServiceBuildMd.addFile(cacheFile);
+        }
+    }
+
+    fs.writeFileSync(outputPath, result);
 
     if (changelogs?.length) {
         const mdFilename = basename(outputPath, extname(outputPath));
@@ -58,11 +102,11 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void>
 
             const changesPath = join(outputDir, `changes-${changesName}.json`);
 
-            if (existsSync(changesPath)) {
+            if (fs.existsSync(changesPath)) {
                 throw new Error(`Changelog ${changesPath} already exists!`);
             }
 
-            writeFileSync(
+            fs.writeFileSync(
                 changesPath,
                 JSON.stringify({
                     ...changes,
@@ -78,14 +122,28 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void>
 }
 
 function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) {
-    shell.mkdir('-p', dirname(targetDestPath));
-
     if (options) {
-        const sourceIncludeContent = readFileSync(targetPath, 'utf8');
+        const {envApi} = options;
+        let sourceIncludeContent: string;
+        if (envApi) {
+            sourceIncludeContent = envApi.readFile(
+                relative(envApi.root, targetPath),
+                'utf-8',
+            ) as string;
+        } else {
+            sourceIncludeContent = fs.readFileSync(targetPath, 'utf8');
+        }
+
         const {result} = transformMd2Md(sourceIncludeContent, options);
 
-        writeFileSync(targetDestPath, result);
+        if (envApi) {
+            envApi.writeFileAsync(relative(envApi.distRoot, targetDestPath), result);
+        } else {
+            fs.mkdirSync(dirname(targetDestPath), {recursive: true});
+            fs.writeFileSync(targetDestPath, result);
+        }
     } else {
+        fs.mkdirSync(dirname(targetDestPath), {recursive: true});
         shell.cp(targetPath, targetDestPath);
     }
 }
@@ -113,6 +171,7 @@ function transformMd2Md(input: string, options: PluginOptions) {
         collectOfPlugins,
         log: pluginLog,
         copyFile: pluginCopyFile,
+        envApi,
     } = options;
 
     let output = input;
@@ -136,6 +195,7 @@ function transformMd2Md(input: string, options: PluginOptions) {
             collectOfPlugins,
             changelogs,
             extractChangelogs: true,
+            envApi,
         });
     }