From 2ed331abbb73a083aa7cb0d4206388f47338ee15 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 3 Dec 2024 21:45:02 +0300 Subject: [PATCH] chore: Rewrite presets on new architecture --- src/commands/build/core/vars/VarsService.ts | 121 +++++++ .../vars/__snapshots__/index.spec.ts.snap | 57 ++++ src/commands/build/core/vars/index.spec.ts | 294 ++++++++++++++++++ src/commands/build/core/vars/index.ts | 3 + src/commands/build/core/vars/types.ts | 10 + .../build/features/templating/index.ts | 17 + src/commands/build/handler.ts | 16 +- src/commands/build/index.ts | 1 - src/commands/build/run.ts | 20 +- src/services/preset.ts | 45 +-- src/steps/processServiceFiles.ts | 66 +--- src/utils/common.ts | 25 ++ .../md2html-with-metadata/input/presets.yaml | 11 +- .../md2md-with-metadata/input/presets.yaml | 11 +- 14 files changed, 587 insertions(+), 110 deletions(-) create mode 100644 src/commands/build/core/vars/VarsService.ts create mode 100644 src/commands/build/core/vars/__snapshots__/index.spec.ts.snap create mode 100644 src/commands/build/core/vars/index.spec.ts create mode 100644 src/commands/build/core/vars/index.ts create mode 100644 src/commands/build/core/vars/types.ts diff --git a/src/commands/build/core/vars/VarsService.ts b/src/commands/build/core/vars/VarsService.ts new file mode 100644 index 00000000..a64c188d --- /dev/null +++ b/src/commands/build/core/vars/VarsService.ts @@ -0,0 +1,121 @@ +import type {Preset, Presets} from './types'; + +import {dirname, join} from 'node:path'; +import {merge} from 'lodash'; +import {dump, load} from 'js-yaml'; + +import {Run} from '~/commands/build'; +import {freeze, own} from '~/utils'; +import {AsyncParallelHook, AsyncSeriesWaterfallHook} from 'tapable'; + +export type VarsServiceConfig = { + varsPreset: string; + vars: Hash; +}; + +type VarsServiceHooks = { + /** + * Async waterfall hook. + * Called after any presets.yaml was loaded. + */ + PresetsLoaded: AsyncSeriesWaterfallHook<[Presets, RelativePath]>; + /** + * Async parallel hook. + * Called after vars was resolved on any level. + * Vars data is sealed here. + */ + Resolved: AsyncParallelHook<[Preset, RelativePath]>; +}; + +export class VarsService { + hooks: VarsServiceHooks; + + private run: Run; + + private fs: Run['fs']; + + private logger: Run['logger']; + + private config: VarsServiceConfig; + + private cache: Record = {}; + + constructor(run: Run) { + this.run = run; + this.fs = run.fs; + this.logger = run.logger; + this.config = run.config; + this.hooks = { + PresetsLoaded: new AsyncSeriesWaterfallHook(['presets', 'path']), + Resolved: new AsyncParallelHook(['vars', 'path']), + }; + } + + async load(path: RelativePath) { + const varsPreset = this.config.varsPreset || 'default'; + const file = join(dirname(path), 'presets.yaml'); + + if (this.cache[file]) { + return this.cache[file]; + } + + this.logger.proc(path); + + const scopes = []; + + if (dirname(path) !== '.') { + scopes.push(await this.load(dirname(path))); + } + + try { + const presets = await this.hooks.PresetsLoaded.promise( + load(await this.fs.readFile(join(this.run.input, file), 'utf8')) as Presets, + file, + ); + + scopes.push(presets['default']); + + if (varsPreset && varsPreset !== 'default') { + scopes.push(presets[varsPreset] || {}); + } + } catch (error) { + if (!own(error, 'code') || error.code !== 'ENOENT') { + throw error; + } + } + + scopes.push(this.config.vars); + + this.cache[file] = freeze(merge({}, ...scopes)); + + await this.hooks.Resolved.promise(this.cache[file], file); + + return this.cache[file]; + } + + dump(presets: Hash): string; + dump(presets: Presets, options: {filtered: boolean} = {filtered: false}): string { + const {varsPreset} = this.config; + + if (options.filtered) { + const scopes = [ + {default: presets.default}, + varsPreset !== 'default' && + presets[varsPreset] && {[varsPreset]: presets[varsPreset]}, + ].filter(Boolean); + const result = scopes.reduce((acc, scope) => ({...acc, ...scope}), {}); + + return dump(result, { + lineWidth: 120, + }); + } + + return dump(presets, { + lineWidth: 120, + }); + } + + entries() { + return Object.entries(this.cache); + } +} diff --git a/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap b/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000..115d72ec --- /dev/null +++ b/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap @@ -0,0 +1,57 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`vars > service > load > should allow content extending in PresetsLoaded hook 1`] = ` +"field1: value1 +field2: value2 +" +`; + +exports[`vars > service > load > should allow content updating in PresetsLoaded hook 1`] = ` +"field1: value2 +" +`; + +exports[`vars > service > load > should load presets file default scope 1`] = ` +"field1: value1 +field2: value2 +" +`; + +exports[`vars > service > load > should load presets file target scope 1`] = ` +"field1: value3 +field2: value2 +" +`; + +exports[`vars > service > load > should load super layers 1`] = ` +"field1: value1 +override1: value1 +override2: value1 +override3: value1 +override4: value1 +field2: value1 +sub1: value1 +sub2: value1 +override5: value1 +override6: value1 +subsub1: value1 +subsub2: value1 +" +`; + +exports[`vars > service > load > should override default presets with vars 1`] = ` +"field1: value6 +field2: value2 +" +`; + +exports[`vars > service > load > should override target presets with vars 1`] = ` +"field1: value6 +field2: value2 +" +`; + +exports[`vars > service > load > should use vars if presets not found 1`] = ` +"field1: value6 +" +`; diff --git a/src/commands/build/core/vars/index.spec.ts b/src/commands/build/core/vars/index.spec.ts new file mode 100644 index 00000000..c12a68c2 --- /dev/null +++ b/src/commands/build/core/vars/index.spec.ts @@ -0,0 +1,294 @@ +import type {Run} from '~/commands/build'; +import type {VarsServiceConfig} from './VarsService'; + +import {join} from 'node:path'; +import {describe, expect, it, vi} from 'vitest'; +import {when} from 'vitest-when'; +import {dedent} from 'ts-dedent'; +import {YAMLException} from 'js-yaml'; + +import {VarsService} from './VarsService'; + +const ENOENT = Object.assign(new Error('ENOENT: no such file or directory'), { + code: 'ENOENT', +}); + +type Options = Partial; + +function prepare(content: string | Hash | Error, options: Options = {}) { + const input = '/dev/null/input' as AbsolutePath; + const output = '/dev/null/output' as AbsolutePath; + const run = { + input, + output, + config: { + varsPreset: options.varsPreset, + vars: options.vars || {}, + }, + logger: { + proc: vi.fn(), + }, + fs: { + readFile: vi.fn(), + }, + } as unknown as Run; + const service = new VarsService(run); + + if (content instanceof Error) { + when(run.fs.readFile) + .calledWith(join(input, './presets.yaml'), expect.anything()) + .thenReject(content); + } else { + if (typeof content === 'string') { + content = {'./presets.yaml': content}; + } + + for (const [file, data] of Object.entries(content)) { + when(run.fs.readFile) + .calledWith(join(input, file), expect.anything()) + .thenResolve(data); + } + } + + return service; +} + +async function call(content: string | Error, options: Options = {}) { + const service = prepare(content, options); + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); +} + +function test(name: string, content: string | Error, options: Options = {}) { + it(name, async () => call(content, options)); +} + +describe('vars', () => { + describe('service', () => { + describe('load', () => { + test( + 'should load presets file default scope', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + ); + + test( + 'should load presets file target scope', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {varsPreset: 'internal'}, + ); + + test( + 'should override default presets with vars', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {vars: {field1: 'value6'}}, + ); + + test( + 'should override target presets with vars', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {varsPreset: 'internal', vars: {field1: 'value6'}}, + ); + + test('should use vars if presets not found', ENOENT, {vars: {field1: 'value6'}}); + + it('should throw parse error', async () => { + await expect(() => call('!@#', {vars: {field1: 'value6'}})).rejects.toThrow( + YAMLException, + ); + }); + + it('should load super layers', async () => { + const service = prepare( + { + './presets.yaml': dedent` + default: + field1: value1 + override1: value2 + override2: value2 + override3: value2 + override4: value2 + internal: + field2: value1 + override1: value1 + `, + './subfolder/presets.yaml': dedent` + default: + sub1: value1 + sub2: value2 + override2: value1 + override5: value2 + internal: + sub2: value1 + override3: value1 + override6: value2 + `, + './subfolder/subfolder/subfolder/presets.yaml': dedent` + default: + subsub1: value2 + override4: value2 + override5: value1 + internal: + subsub1: value1 + subsub2: value1 + override4: value1 + override6: value1 + `, + }, + {varsPreset: 'internal'}, + ); + + const result = await service.load( + './subfolder/subfolder/subfolder/presets.yaml' as RelativePath, + ); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should call PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy = vi.fn(); + + service.hooks.PresetsLoaded.tap('Test', spy); + + await service.load('./presets.yaml' as RelativePath); + + expect(spy).toHaveBeenCalledWith({default: {field1: 'value1'}}, 'presets.yaml'); + }); + + it('should call Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy = vi.fn(); + + service.hooks.Resolved.tap('Test', spy); + + await service.load('./presets.yaml' as RelativePath); + + expect(spy).toHaveBeenCalledWith({field1: 'value1'}, 'presets.yaml'); + }); + + it('should allow content updating in PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.PresetsLoaded.tap('Test', (presets) => { + presets.default.field1 = 'value2'; + + return presets; + }); + + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should allow content extending in PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.PresetsLoaded.tap('Test', (presets) => { + presets.default.field2 = 'value2'; + + return presets; + }); + + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should reject content updating in Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.Resolved.tap('Test', (vars) => { + vars.field1 = 'value2'; + }); + + await expect(() => + service.load('./presets.yaml' as RelativePath), + ).rejects.toThrow(); + }); + + it('should reject content extending in Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.Resolved.tap('Test', (vars) => { + vars.field2 = 'value2'; + }); + + await expect(() => + service.load('./presets.yaml' as RelativePath), + ).rejects.toThrow(); + }); + + it('should load content only once', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy1 = vi.fn(); + const spy2 = vi.fn(); + + service.hooks.PresetsLoaded.tap('Test', spy1); + service.hooks.Resolved.tap('Test', spy2); + + await service.load('./presets.yaml' as RelativePath); + await service.load('./presets.yaml' as RelativePath); + + expect(spy1).toHaveBeenCalledOnce(); + expect(spy2).toHaveBeenCalledOnce(); + }); + }); + }); +}); diff --git a/src/commands/build/core/vars/index.ts b/src/commands/build/core/vars/index.ts new file mode 100644 index 00000000..7f666f0d --- /dev/null +++ b/src/commands/build/core/vars/index.ts @@ -0,0 +1,3 @@ +export type {Preset, Presets} from './types'; + +export {VarsService} from './VarsService'; diff --git a/src/commands/build/core/vars/types.ts b/src/commands/build/core/vars/types.ts new file mode 100644 index 00000000..2dbb23d8 --- /dev/null +++ b/src/commands/build/core/vars/types.ts @@ -0,0 +1,10 @@ +export type Presets = { + default: Preset; +} & { + [prop: string]: Preset; +}; + +export type Preset = { + __system?: Hash; + __metadata?: Hash; +} & Hash; diff --git a/src/commands/build/features/templating/index.ts b/src/commands/build/features/templating/index.ts index 6fab837f..0b1fb400 100644 --- a/src/commands/build/features/templating/index.ts +++ b/src/commands/build/features/templating/index.ts @@ -1,5 +1,8 @@ import type {Build} from '~/commands'; import type {Command} from '~/config'; + +import {join} from 'node:path'; + import {defined, valuable} from '~/config'; import {options} from './config'; @@ -90,5 +93,19 @@ export class Templating { return config; }); + + program.hooks.BeforeRun.for('md').tap('Build', (run) => { + const {substitutions, conditions} = run.config.template.features; + if (!substitutions || !conditions) { + run.vars.hooks.PresetsLoaded.tapPromise( + 'Build', + async (presets, path) => { + await run.write(join(run.output, path), run.vars.dump(presets, {filtered: true})); + + return presets; + }, + ); + } + }); } } diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index e37859fa..ab9eb08a 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -2,18 +2,21 @@ import type {Run} from './run'; import 'threads/register'; +import {glob} from 'glob'; + import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; import {ArgvService, Includers, SearchService} from '~/services'; import { initLinterWorkers, + preparingPresetFiles, + preparingTocFiles, processAssets, processChangelogs, processExcludedFiles, processLinter, processLogs, processPages, - processServiceFiles, } from '~/steps'; import {prepareMapFile} from '~/steps/processMapFile'; @@ -27,7 +30,16 @@ export async function handler(run: Run) { const {lintDisabled, buildDisabled, addMapFile} = ArgvService.getConfig(); - await processServiceFiles(); + const presets = (await glob('**/presets.yaml', { + cwd: run.input, + ignore: run.config.ignore, + })) as RelativePath[]; + for (const preset of presets) { + await run.vars.load(preset); + } + + await preparingPresetFiles(run); + await preparingTocFiles(run); processExcludedFiles(); if (addMapFile) { diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index daa26ba5..bffcaba9 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -2,7 +2,6 @@ import type {IProgram, ProgramArgs, ProgramConfig} from '~/program'; import type {DocAnalytics} from '@diplodoc/client'; import {ok} from 'node:assert'; -import {join} from 'node:path'; import {pick} from 'lodash'; import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index 7359aff9..8d196c7a 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -13,9 +13,10 @@ import { TMP_OUTPUT_FOLDER, YFM_CONFIG_FILENAME, } from '~/constants'; -import {Logger} from '~/logger'; +import {LogLevel, Logger} from '~/logger'; import {BuildConfig} from '.'; // import {InsecureAccessError} from './errors'; +import {VarsService} from './core/vars'; type FileSystem = { access: typeof access; @@ -27,6 +28,10 @@ type FileSystem = { writeFile: typeof writeFile; }; +class RunLogger extends Logger { + proc = this.topic(LogLevel.INFO, 'PROC'); +} + /** * This is transferable context for build command. * Use this context to communicate with lower data processing levels. @@ -42,12 +47,14 @@ export class Run { readonly legacyConfig: YfmArgv; - readonly logger: Logger; + readonly logger: RunLogger; readonly config: BuildConfig; readonly fs: FileSystem = {access, stat, link, unlink, mkdir, readFile, writeFile}; + readonly vars: VarsService; + get bundlePath() { return join(this.output, BUNDLE_FOLDER); } @@ -72,6 +79,11 @@ export class Run { this.input = resolve(config.output, TMP_INPUT_FOLDER); this.output = resolve(config.output, TMP_OUTPUT_FOLDER); + this.logger = new RunLogger(config, [ + (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), + ]); + + this.vars = new VarsService(this); this.legacyConfig = { rootInput: this.originalInput, input: this.input, @@ -119,10 +131,6 @@ export class Run { included: config.mergeIncludes, }; - - this.logger = new Logger(config, [ - (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), - ]); } write = async (path: AbsolutePath, content: string | Buffer) => { diff --git a/src/services/preset.ts b/src/services/preset.ts index c24533ba..13694c85 100644 --- a/src/services/preset.ts +++ b/src/services/preset.ts @@ -1,43 +1,30 @@ import {dirname, normalize} from 'path'; -import {DocPreset, YfmPreset} from '../models'; +import {YfmPreset} from '../models'; +import {VarsService} from '~/commands/build/core/vars'; export type PresetStorage = Map; let presetStorage: PresetStorage = new Map(); -function add(parsedPreset: DocPreset, path: string, varsPreset: string) { - const combinedValues = { - ...(parsedPreset.default || {}), - ...(parsedPreset[varsPreset] || {}), - __metadata: parsedPreset.__metadata, - } as YfmPreset; - - const key = dirname(normalize(path)); - presetStorage.set(key, combinedValues); +function init(vars: VarsService) { + for (const [path, values] of vars.entries()) { + presetStorage.set(dirname(path), values); + } } function get(path: string): YfmPreset { - let combinedValues: YfmPreset = {}; - let localPath = normalize(path); - - while (localPath !== '.') { - const presetValues: YfmPreset = presetStorage.get(localPath) || {}; - localPath = dirname(localPath); - - combinedValues = { - ...presetValues, - ...combinedValues, - }; + let vars = presetStorage.get(normalize(path)); + while (!vars) { + path = dirname(path); + vars = presetStorage.get(normalize(path)); + + if (path === '.') { + break; + } } - // Add root' presets - combinedValues = { - ...presetStorage.get('.'), - ...combinedValues, - }; - - return combinedValues; + return vars || {}; } function getPresetStorage(): Map { @@ -49,7 +36,7 @@ function setPresetStorage(preset: Map): void { } export default { - add, + init, get, getPresetStorage, setPresetStorage, diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts index 7c315264..df1a3bfc 100644 --- a/src/steps/processServiceFiles.ts +++ b/src/steps/processServiceFiles.ts @@ -1,13 +1,8 @@ -import {dirname, resolve} from 'path'; import walkSync from 'walk-sync'; -import {readFileSync, writeFileSync} from 'fs'; -import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; import {ArgvService, PresetService, TocService} from '../services'; -import {logger} from '../utils'; -import {DocPreset} from '../models'; -import shell from 'shelljs'; +import {Run} from '~/commands/build'; const getFilePathsByGlobals = (globs: string[]): string[] => { const {input, ignore = []} = ArgvService.getConfig(); @@ -20,64 +15,11 @@ const getFilePathsByGlobals = (globs: string[]): string[] => { }); }; -export async function processServiceFiles(): Promise { - await preparingPresetFiles(); - await preparingTocFiles(); +export async function preparingPresetFiles(run: Run) { + PresetService.init(run.vars); } -async function preparingPresetFiles() { - const { - input: inputFolderPath, - varsPreset = '', - outputFormat, - applyPresets, - resolveConditions, - } = ArgvService.getConfig(); - - try { - const presetsFilePaths = getFilePathsByGlobals(['**/presets.yaml']); - - for (const path of presetsFilePaths) { - logger.proc(path); - - const pathToPresetFile = resolve(inputFolderPath, path); - const content = readFileSync(pathToPresetFile, 'utf8'); - const parsedPreset = load(content) as DocPreset; - - PresetService.add(parsedPreset, path, varsPreset); - - if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) { - // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false - saveFilteredPresets(path, parsedPreset); - } - } - } catch (error) { - log.error(`Preparing presets.yaml files failed. Error: ${error}`); - throw error; - } -} - -function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { - const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig(); - - const outputPath = resolve(outputFolderPath, path); - const filteredPreset: Record = { - default: parsedPreset.default, - }; - - if (parsedPreset[varsPreset]) { - filteredPreset[varsPreset] = parsedPreset[varsPreset]; - } - - const outputPreset = dump(filteredPreset, { - lineWidth: 120, - }); - - shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputPreset); -} - -async function preparingTocFiles(): Promise { +export async function preparingTocFiles(): Promise { try { const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); await TocService.init(tocFilePaths); diff --git a/src/utils/common.ts b/src/utils/common.ts index 569226ce..bae1fff7 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -50,3 +50,28 @@ export function checkPathExists(path: string, parentFilePath: string) { return isFileExists(includePath); } + +export function own(box: unknown, field: T): box is {[p in T]: unknown} { + return ( + Boolean(box && typeof box === 'object') && Object.prototype.hasOwnProperty.call(box, field) + ); +} + +export function freeze(target: T, visited = new Set()): T { + if (!visited.has(target)) { + visited.add(target); + + if (Array.isArray(target)) { + target.forEach((item) => freeze(item, visited)); + } + + if (isObject(target) && !Object.isSealed(target)) { + Object.freeze(target); + Object.keys(target).forEach((key) => + freeze(target[key as keyof typeof target], visited), + ); + } + } + + return target; +} diff --git a/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml b/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml index a5dec198..d02a09a9 100644 --- a/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml +++ b/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml @@ -1,5 +1,6 @@ -__metadata: - - name: test-yfm - content: inline test - - name: yfm-config - content: config test +default: + __metadata: + - name: test-yfm + content: inline test + - name: yfm-config + content: config test diff --git a/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml b/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml index a5dec198..d02a09a9 100644 --- a/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml +++ b/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml @@ -1,5 +1,6 @@ -__metadata: - - name: test-yfm - content: inline test - - name: yfm-config - content: config test +default: + __metadata: + - name: test-yfm + content: inline test + - name: yfm-config + content: config test