diff --git a/src/constants.ts b/src/constants.ts index a7c2ca03..aed431c3 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -97,6 +97,8 @@ export const GETTING_ALL_CONTRIBUTORS = 'Getting all contributors.'; export const ALL_CONTRIBUTORS_RECEIVED = 'All contributors received.'; export const getMsgСonfigurationMustBeProvided = (repo: string) => `Сonfiguration must be provided for ${repo} like env variables or in .yfm file`; +export const CACHE_HIT = 'Cache hit:'; +export const LINT_CACHE_HIT = 'Lint cache hit:'; export const FIRST_COMMIT_FROM_ROBOT_IN_GITHUB = '2dce14271359cd20d7e874956d604de087560cf4'; diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index 0720f682..788a57e4 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -9,12 +9,13 @@ import {readFileSync} from 'fs'; import {bold} from 'chalk'; import {ArgvService, PluginService} from '../services'; -import {getVarsPerFileWithHash, getVarsPerRelativeFile} from '../utils'; +import {getVarsPerFileWithHash, getVarsPerRelativeFile, logger} from '../utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; import {cacheServiceLint} from '../services/cache'; import PluginEnvApi from '../utils/pluginEnvApi'; import {checkLogWithoutProblems, getLogState} from '../services/utils'; +import {LINT_CACHE_HIT} from '../constants'; interface FileTransformOptions { path: string; @@ -76,6 +77,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void const cachedFile = cacheServiceLint.checkFile(cacheKey); if (cachedFile) { + logger.info(filePath, LINT_CACHE_HIT); return; } diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 24e6136b..071f3cfd 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -15,7 +15,7 @@ import { getVarsPerRelativeFile, getVarsPerFileWithHash, } from '../utils'; -import {PROCESSING_FINISHED, Lang} from '../constants'; +import {PROCESSING_FINISHED, Lang, CACHE_HIT} from '../constants'; import {getAssetsPublicPath, getUpdatedMetadata} from '../services/metadata'; import {MarkdownItPluginCb} from '@doc-tools/transform/lib/plugins/typings'; import PluginEnvApi from '../utils/pluginEnvApi'; @@ -141,6 +141,7 @@ async function MdFileTransformer(content: string, transformOptions: FileTransfor const cachedFile = await cacheServiceMdToHtml.checkFileAsync(cacheKey); if (cachedFile) { + logger.info(filePath, CACHE_HIT); await cachedFile.extractCacheAsync(); return cachedFile.getResult(); } @@ -165,6 +166,8 @@ async function MdFileTransformer(content: string, transformOptions: FileTransfor envApi, }); + await envApi.executeActionsAsync(); + const logIsOk = checkLogWithoutProblems(log, logState); if (logIsOk) { cacheFile.setResult(result); diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index 4afac10a..56e39cd3 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -1,18 +1,18 @@ -import {existsSync, readFileSync, writeFileSync} from 'fs'; import {dirname, resolve, join, basename, extname, relative} from 'path'; import shell from 'shelljs'; import log, {LogLevels} from '@doc-tools/transform/lib/log'; import liquid from '@doc-tools/transform/lib/liquid'; import {ArgvService, PluginService} from '../services'; -import {logger, getVarsPerFileWithHash} from '../utils'; +import {logger, getVarsPerFileWithHash, fileExists} from '../utils'; import {PluginOptions, ResolveMd2MdOptions} from '../models'; -import {PROCESSING_FINISHED} from '../constants'; +import {CACHE_HIT, PROCESSING_FINISHED} from '../constants'; import {getContentWithUpdatedMetadata} from '../services/metadata'; import {ChangelogItem} from '@doc-tools/transform/lib/plugins/changelog/types'; import {cacheServiceBuildMd} from '../services/cache'; import PluginEnvApi from '../utils/pluginEnvApi'; import {checkLogWithoutProblems, getLogState} from '../services/utils'; +import * as fs from 'fs'; export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise { const {inputPath, outputPath, metadata} = options; @@ -20,7 +20,7 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const resolvedInputPath = resolve(input, inputPath); const {vars, varsHashList} = getVarsPerFileWithHash(inputPath); - const rawContent = readFileSync(resolvedInputPath, 'utf8'); + const rawContent = await fs.promises.readFile(resolvedInputPath, 'utf8'); const cacheKey = cacheServiceBuildMd.getHashKey({filename: inputPath, content: rawContent, varsHashList}); @@ -29,6 +29,7 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const cachedFile = await cacheServiceBuildMd.checkFileAsync(cacheKey); if (cachedFile) { + logger.info(inputPath, CACHE_HIT); await cachedFile.extractCacheAsync(); const results = cachedFile.getResult<{result: string; changelogs: ChangelogItem[]; logs: Record}>(); result = results.result; @@ -63,20 +64,23 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise result = transformResult.result; changelogs = transformResult.changelogs; + await envApi.executeActionsAsync(); + const logIsOk = checkLogWithoutProblems(log, logState); if (logIsOk) { cacheFile.setResult(transformResult); - // not async cause race condition - cacheServiceBuildMd.addFile(cacheFile); + await cacheServiceBuildMd.addFileAsync(cacheFile); } } - writeFileSync(outputPath, result); + await fs.promises.writeFile(outputPath, result); if (changelogs?.length) { const mdFilename = basename(outputPath, extname(outputPath)); const outputDir = dirname(outputPath); - changelogs.forEach((changes, index) => { + for (let index = 0, len = changelogs.length; index < len; index++) { + const changes = changelogs[index]; + let changesName; const changesDate = changes.date as string | undefined; const changesIdx = changes.index as number | undefined; @@ -92,15 +96,16 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const changesPath = join(outputDir, `changes-${changesName}.json`); - if (existsSync(changesPath)) { + const isExists = await fileExists(changesPath); + if (isExists) { throw new Error(`Changelog ${changesPath} already exists!`); } - writeFileSync(changesPath, JSON.stringify({ + await fs.promises.writeFile(changesPath, JSON.stringify({ ...changes, source: mdFilename, })); - }); + } } logger.info(inputPath, PROCESSING_FINISHED); @@ -109,25 +114,25 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise } function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { - shell.mkdir('-p', dirname(targetDestPath)); - if (options) { const {envApi} = options; let sourceIncludeContent: string; if (envApi) { - sourceIncludeContent = envApi.readFileSync(relative(envApi.root, targetPath), 'utf-8'); + sourceIncludeContent = envApi.readFile(relative(envApi.root, targetPath), 'utf-8'); } else { - sourceIncludeContent = readFileSync(targetPath, 'utf8'); + sourceIncludeContent = fs.readFileSync(targetPath, 'utf8'); } const {result} = transformMd2Md(sourceIncludeContent, options); if (envApi) { - envApi.writeFileSync(relative(envApi.distRoot, targetDestPath), result); + envApi.writeFileAsync(relative(envApi.distRoot, targetDestPath), result); } else { - writeFileSync(targetDestPath, result); + fs.mkdirSync(dirname(targetDestPath), {recursive: true}); + fs.writeFileSync(targetDestPath, result); } } else { + fs.mkdirSync(dirname(targetDestPath), {recursive: true}); shell.cp(targetPath, targetDestPath); } } diff --git a/src/services/cache/cache.ts b/src/services/cache/cache.ts index 955be9f0..35a5c6a8 100644 --- a/src/services/cache/cache.ts +++ b/src/services/cache/cache.ts @@ -89,7 +89,7 @@ export class CacheService { try { const dataJson = fs.readFileSync(filepath, 'utf-8'); const data = JSON.parse(dataJson); - file = CacheFile.from(data, this.disabled); + file = CacheFile.from(data, this.disabled, this.getAssetsDir()); } catch (err) { return; } @@ -108,7 +108,7 @@ export class CacheService { try { const dataJson = await fs.promises.readFile(filepath, 'utf-8'); const data = JSON.parse(dataJson); - file = CacheFile.from(data, this.disabled); + file = CacheFile.from(data, this.disabled, this.getAssetsDir()); } catch (err) { return; } @@ -117,7 +117,7 @@ export class CacheService { } createFile(key: HashKey) { - return new CacheFile(key, this.disabled); + return new CacheFile(key, this.disabled, this.getAssetsDir()); } addFile(file: CacheFile) { @@ -129,6 +129,7 @@ export class CacheService { fs.mkdirSync(place, {recursive: true}); existsDir.add(place); } + file.writeAssets(); fs.writeFileSync(filepath, JSON.stringify(file.toJSON())); } @@ -141,7 +142,10 @@ export class CacheService { await fs.promises.mkdir(place, {recursive: true}); existsDir.add(place); } - await fs.promises.writeFile(filepath, JSON.stringify(file.toJSON())); + await Promise.all([ + file.writeAssetsAsync(), + fs.promises.writeFile(filepath, JSON.stringify(file.toJSON())), + ]); } getHashKey(props: GetHashKeyProps) { @@ -161,6 +165,10 @@ export class CacheService { private getCacheFilepath(key: string) { return path.join(this.cacheDir, this.storeName, key.slice(0, 2), key); } + + private getAssetsDir() { + return path.join(this.cacheDir, 'assets'); + } } export const cacheServiceLint = new CacheService('lint'); diff --git a/src/services/cache/cacheFile.ts b/src/services/cache/cacheFile.ts index bd74064b..04644954 100644 --- a/src/services/cache/cacheFile.ts +++ b/src/services/cache/cacheFile.ts @@ -8,19 +8,22 @@ import {asyncify, mapLimit, parallelLimit} from 'async'; import {CacheFileData, CacheFileDataWithDeps, Deps} from './types'; const CUNCURRENCY = 1000; +const existsDir = new Set(); type CacheFileProps = CacheFileData & Partial; class CacheFile { - static from(data: CacheFileDataWithDeps, disabled: boolean) { - return new CacheFile(data, disabled); + static from(data: CacheFileDataWithDeps, disabled: boolean, assetsDir: string) { + return new CacheFile(data, disabled, assetsDir); } disabled = false; - + private assetsDir: string; private data: CacheFileDataWithDeps; + private wroteFileData: Record = {}; - constructor(data: CacheFileProps, disabled: boolean) { + constructor(data: CacheFileProps, disabled: boolean, assetsDir: string) { + this.assetsDir = assetsDir; this.disabled = disabled; this.data = { ...data, @@ -108,39 +111,33 @@ class CacheFile { const tasks: (() => Promise)[] = []; - Object.entries(copiedFiles).forEach(([, from]) => { - tasks.push(asyncify(async () => { - const filepath = path.join(root, from); - const isExists = await fileExists(filepath); - if (!isExists) { - throw new Error('Aborted'); - } - })); - }); + Object.entries(copiedFiles).forEach(([, from]) => tasks.push(asyncify(async () => { + const filepath = path.join(root, from); + const isExists = await fileExists(filepath); + if (!isExists) { + throw new Error('Aborted'); + } + }))); - Object.entries(existsFiles).forEach(([filename, reqState]) => { - tasks.push(asyncify(async () => { - const filepath = path.join(root, filename as string); - const isExists = await fileExists(filepath); - if (isExists !== reqState as boolean) { - throw new Error('Aborted'); - } - })); - }); + Object.entries(existsFiles).forEach(([filename, reqState]) => tasks.push(asyncify(async () => { + const filepath = path.join(root, filename as string); + const isExists = await fileExists(filepath); + if (isExists !== reqState as boolean) { + throw new Error('Aborted'); + } + }))); - Object.entries(fileDeps).forEach(([filename, reqContentHash]) => { - tasks.push(asyncify(async () => { - const filepath = path.join(root, filename); - const isExists = await fileExists(filepath); - if (!isExists) { - throw new Error('Aborted'); - } - const contentHash = await CacheService.getFileHashAsync(filepath); - if (contentHash !== reqContentHash) { - throw new Error('Aborted'); - } - })); - }); + Object.entries(fileDeps).forEach(([filename, reqContentHash]) => tasks.push(asyncify(async () => { + const filepath = path.join(root, filename); + const isExists = await fileExists(filepath); + if (!isExists) { + throw new Error('Aborted'); + } + const contentHash = await CacheService.getFileHashAsync(filepath); + if (contentHash !== reqContentHash) { + throw new Error('Aborted'); + } + }))); try { await parallelLimit(tasks, CUNCURRENCY); @@ -172,8 +169,11 @@ class CacheFile { this.data.fileVarsDeps[filename] = varsHashList; } - addWriteFile(filename: string, data: string) { - this.data.wroteFiles[filename] = data; + addWriteFile(to: string, content: string | Uint8Array) { + const contentHash = CacheService.getHash(content); + + this.wroteFileData[contentHash] = content; + this.data.wroteFiles[to] = contentHash; } getResult() { @@ -196,17 +196,50 @@ class CacheFile { this.copyFiles(); } + writeAssets() { + const {wroteFileData} = this; + for (const filename in wroteFileData) { + if (!Object.hasOwnProperty.call(wroteFileData, filename)) { + continue; + } + + const data = wroteFileData[filename]; + const fullFilename = this.getAssetFilepath(filename); + const place = path.dirname(fullFilename); + if (!existsDir.has(place)) { + fs.mkdirSync(place, {recursive: true}); + } + fs.writeFileSync(fullFilename, data); + } + } + + async writeAssetsAsync() { + const {wroteFileData} = this; + + const tasks = Object.entries(wroteFileData).map(([filename, data]) => asyncify(async () => { + const fullFilename = this.getAssetFilepath(filename); + const place = path.dirname(fullFilename); + if (!existsDir.has(place)) { + await fs.promises.mkdir(place, {recursive: true}); + } + await fs.promises.writeFile(fullFilename, data); + })); + + await parallelLimit(tasks, CUNCURRENCY); + } + private writeData() { const {output} = ArgvService.getConfig(); const distRoot = path.resolve(output); const {wroteFiles} = this.data; - Object.entries(wroteFiles).forEach(([filename, content]) => { - const fullFilename = path.join(distRoot, filename); + Object.entries(wroteFiles).forEach(([to, assetName]) => { + const fullFrom = this.getAssetFilepath(assetName); + const fullTo = path.join(distRoot, to); - fs.mkdirSync(path.dirname(fullFilename), {recursive: true}); - fs.writeFileSync(fullFilename, content); + fs.mkdirSync(path.dirname(fullTo), {recursive: true}); + fs.copyFileSync(fullFrom, fullTo); }); } @@ -216,11 +249,12 @@ class CacheFile { const {wroteFiles} = this.data; - await mapLimit(Object.entries(wroteFiles), CUNCURRENCY, asyncify(async ([filename, content]: string[]) => { - const fullFilename = path.join(distRoot, filename); + await mapLimit(Object.entries(wroteFiles), CUNCURRENCY, asyncify(async ([to, assetName]: string[]) => { + const fullFrom = this.getAssetFilepath(assetName); + const fullTo = path.join(distRoot, to); - await fs.promises.mkdir(path.dirname(fullFilename), {recursive: true}); - await fs.promises.writeFile(fullFilename, content); + await fs.promises.mkdir(path.dirname(fullTo), {recursive: true}); + await fs.promises.copyFile(fullFrom, fullTo); })); } @@ -255,6 +289,10 @@ class CacheFile { await fs.promises.copyFile(fullFrom, fullTo); })); } + + private getAssetFilepath(key: string) { + return path.join(this.assetsDir, key.slice(0, 2), key); + } } export default CacheFile; diff --git a/src/utils/pluginEnvApi.ts b/src/utils/pluginEnvApi.ts index 46ff0a4d..77c2f03a 100644 --- a/src/utils/pluginEnvApi.ts +++ b/src/utils/pluginEnvApi.ts @@ -3,6 +3,19 @@ import * as path from 'path'; import CacheFile from '../services/cache/cacheFile'; import {getVarsPerFileWithHash} from './presets'; import {safeRelativePath} from './path'; +import {asyncify, mapLimit} from 'async'; + +const CUNCURRENCY = 1000; + +enum AsyncActionType { + Copy = 'copy', + Write = 'write', +} + +type CopyFileAsyncAction = {type: AsyncActionType.Copy; from: string; to: string}; +type WriteFileAsyncAction = {type: AsyncActionType.Write; to: string; data: string | Uint8Array}; + +type AsyncAction = CopyFileAsyncAction | WriteFileAsyncAction; interface PluginEnvApiProps { root: string; distRoot: string; cacheFile?: CacheFile; @@ -17,13 +30,15 @@ class PluginEnvApi { readonly distRoot: string; cacheFile: CacheFile | undefined; + private asyncActionQueue: AsyncAction[] = []; + constructor({root, distRoot, cacheFile}: PluginEnvApiProps) { this.root = root; this.distRoot = distRoot; this.cacheFile = cacheFile?.use(); } - copyFileSync(rawFrom: string, rawTo: string) { + copyFile(rawFrom: string, rawTo: string) { const from = safeRelativePath(rawFrom); const to = safeRelativePath(rawTo); @@ -37,7 +52,17 @@ class PluginEnvApi { } } - readFileSync(rawTarget: string, encoding: T) { + copyFileAsync(rawFrom: string, rawTo: string) { + const from = safeRelativePath(rawFrom); + const to = safeRelativePath(rawTo); + + this.asyncActionQueue.push({type: AsyncActionType.Copy, from, to}); + if (this.cacheFile) { + this.cacheFile.addCopyFile({from, to}); + } + } + + readFile(rawTarget: string, encoding: T) { const target = safeRelativePath(rawTarget); const fullTarget = path.join(this.root, target); @@ -48,7 +73,7 @@ class PluginEnvApi { return result; } - fileExistsSync(rawTarget: string) { + fileExists(rawTarget: string) { const target = safeRelativePath(rawTarget); const fullTarget = path.join(this.root, target); @@ -59,16 +84,26 @@ class PluginEnvApi { return result; } - writeFileSync(rawTo: string, data: string) { + writeFile(rawTo: string, data: string | Uint8Array) { const to = safeRelativePath(rawTo); const fullTo = path.join(this.distRoot, to); + fs.mkdirSync(path.dirname(fullTo), {recursive: true}); fs.writeFileSync(fullTo, data); if (this.cacheFile) { this.cacheFile.addWriteFile(to, data); } } + writeFileAsync(rawTo: string, data: string | Uint8Array) { + const to = safeRelativePath(rawTo); + + this.asyncActionQueue.push({type: AsyncActionType.Write, to, data}); + if (this.cacheFile) { + this.cacheFile.addWriteFile(to, data); + } + } + getFileVars(rawTarget: string) { const target = safeRelativePath(rawTarget); @@ -78,6 +113,56 @@ class PluginEnvApi { } return vars; } + + executeActions() { + this.asyncActionQueue.splice(0).forEach((action) => { + switch (action.type) { + case AsyncActionType.Copy: { + const {from, to} = action; + const fullFrom = path.join(this.root, from); + const fullTo = path.join(this.distRoot, to); + + fs.mkdirSync(path.dirname(fullTo), {recursive: true}); + fs.copyFileSync(fullFrom, fullTo); + break; + } + case AsyncActionType.Write: { + const {to, data} = action; + const fullTo = path.join(this.distRoot, to); + + fs.mkdirSync(path.dirname(fullTo), {recursive: true}); + fs.writeFileSync(fullTo, data); + break; + } + } + }); + } + + async executeActionsAsync() { + const {asyncActionQueue} = this; + + await mapLimit(asyncActionQueue.splice(0), CUNCURRENCY, asyncify(async (action: AsyncAction) => { + switch (action.type) { + case AsyncActionType.Copy: { + const {from, to} = action; + const fullFrom = path.join(this.root, from); + const fullTo = path.join(this.distRoot, to); + + await fs.promises.mkdir(path.dirname(fullTo), {recursive: true}); + await fs.promises.copyFile(fullFrom, fullTo); + break; + } + case AsyncActionType.Write: { + const {to, data} = action; + const fullTo = path.join(this.distRoot, to); + + await fs.promises.mkdir(path.dirname(fullTo), {recursive: true}); + await fs.promises.writeFile(fullTo, data); + break; + } + } + })); + } } export default PluginEnvApi;