From f65e1b116d41fba9a040aaa118c7d6cf352cb42c Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 10 Dec 2024 12:11:10 +0300 Subject: [PATCH] wip --- src/commands/build/core/toc/TocService.ts | 199 ++++++++ .../core/toc/__snapshots__/index.spec.ts.snap | 147 ++++++ .../build/core/toc/includers/generic.spec.ts | 39 ++ .../build/core/toc/includers/generic.ts | 88 ++++ src/commands/build/core/toc/index.spec.ts | 385 +++++++++++++++ src/commands/build/core/toc/index.ts | 6 + src/commands/build/core/toc/loader.ts | 349 ++++++++++++++ src/commands/build/core/toc/types.ts | 47 ++ src/commands/build/core/toc/utils.ts | 3 + src/commands/build/core/vars/VarsService.ts | 2 +- src/commands/build/features/html/index.ts | 32 ++ .../build/features/singlepage/index.ts | 60 +++ src/commands/build/handler.ts | 15 +- src/commands/build/index.ts | 27 ++ src/commands/build/run.ts | 8 + src/commands/build/types.ts | 2 + src/commands/build/utils.ts | 37 ++ src/globals.d.ts | 21 +- src/index.ts | 9 +- src/models.ts | 2 - src/services/includers/batteries/common.ts | 46 -- src/services/includers/batteries/generic.ts | 150 ------ src/services/includers/batteries/index.ts | 3 - .../includers/batteries/sourcedocs.ts | 36 -- src/services/includers/batteries/unarchive.ts | 110 ----- src/services/includers/index.ts | 159 ------- src/services/index.ts | 1 - src/services/metadata/vcsMetadata.ts | 10 +- src/services/tocs.ts | 441 +----------------- src/services/utils.ts | 20 - src/steps/index.ts | 1 - src/steps/processExcludedFiles.ts | 15 +- src/steps/processPages.ts | 43 +- src/steps/processServiceFiles.ts | 25 - src/utils/common.ts | 37 ++ src/utils/index.ts | 1 - src/utils/singlePage.ts | 33 +- src/utils/toc.ts | 71 --- tests/utils.ts | 2 +- 39 files changed, 1525 insertions(+), 1157 deletions(-) create mode 100644 src/commands/build/core/toc/TocService.ts create mode 100644 src/commands/build/core/toc/__snapshots__/index.spec.ts.snap create mode 100644 src/commands/build/core/toc/includers/generic.spec.ts create mode 100644 src/commands/build/core/toc/includers/generic.ts create mode 100644 src/commands/build/core/toc/index.spec.ts create mode 100644 src/commands/build/core/toc/index.ts create mode 100644 src/commands/build/core/toc/loader.ts create mode 100644 src/commands/build/core/toc/types.ts create mode 100644 src/commands/build/core/toc/utils.ts create mode 100644 src/commands/build/features/html/index.ts create mode 100644 src/commands/build/types.ts create mode 100644 src/commands/build/utils.ts delete mode 100644 src/services/includers/batteries/common.ts delete mode 100644 src/services/includers/batteries/generic.ts delete mode 100644 src/services/includers/batteries/index.ts delete mode 100644 src/services/includers/batteries/sourcedocs.ts delete mode 100644 src/services/includers/batteries/unarchive.ts delete mode 100644 src/services/includers/index.ts delete mode 100644 src/steps/processServiceFiles.ts delete mode 100644 src/utils/toc.ts diff --git a/src/commands/build/core/toc/TocService.ts b/src/commands/build/core/toc/TocService.ts new file mode 100644 index 00000000..e4596475 --- /dev/null +++ b/src/commands/build/core/toc/TocService.ts @@ -0,0 +1,199 @@ +import type { BuildConfig, Run } from '~/commands/build'; +import type { IncluderOptions, RawToc, RawTocItem, WithItems } from './types'; + +import { ok } from 'node:assert'; +import { basename, dirname, join } from 'node:path'; +import { dump, load } from 'js-yaml'; +import { AsyncParallelHook, AsyncSeriesWaterfallHook, HookMap } from 'tapable'; +import { freeze, intercept, isExternalHref, own } from '~/utils'; + +import loader, {IncludeMode, LoaderContext} from './loader'; + +export type TocServiceConfig = { + ignoreStage: BuildConfig['ignoreStage']; + template: BuildConfig['template']; + removeHiddenTocItems: BuildConfig['removeHiddenTocItems']; +}; + +type WalkStepResult = RawTocItem | RawTocItem[] | void; + +type TocServiceHooks = { + /** + * Called before item data processing (but after data interpolation) + */ + Item: AsyncSeriesWaterfallHook<[RawTocItem, RelativePath]>; + Includer: HookMap>; + Resolved: AsyncParallelHook<[Toc, RelativePath]>; + Included: AsyncParallelHook<[Toc, LoadFrom]>; +}; + +type IncludeInfo = { + from: RelativePath; + mode: IncludeMode; + mergeBase?: RelativePath; +}; + +// TODO: addSourcePath(fileContent, sourcePath); +export class TocService { + hooks: TocServiceHooks; + + private run: Run; + + private logger: Run['logger']; + + private vars: Run['vars']; + + private config: TocServiceConfig; + + private tocs: Map = new Map(); + + private entries: Set = new Set(); + + constructor(run: Run) { + this.run = run; + this.logger = run.logger; + this.vars = run.vars; + this.config = run.config; + this.hooks = intercept('TocService', { + Item: new AsyncSeriesWaterfallHook(['item', 'path']), + Includer: new HookMap(() => new AsyncSeriesWaterfallHook(['toc', 'options', 'path'])), + Resolved: new AsyncParallelHook(['toc', 'path']), + Included: new AsyncParallelHook(['toc', 'from']), + }); + } + + // TODO: remove after metadate refactoring + async realpath(path: RelativePath) { + return this.run.realpath(join(this.run.input, path)); + } + + async load(path: RelativePath, include?: IncludeInfo) { + this.logger.proc(path); + + const file = join(this.run.input, path); + + ok(file.startsWith(this.run.input), `Requested toc '${file}' is out of project scope.`); + + const context: LoaderContext = { + root: this.run.input, + mode: include?.mode || IncludeMode.RootMerge, + base: include?.from, + mergeBase: include?.mergeBase, + path, + vars: await this.vars.load(path), + toc: this, + options: { + ignoreStage: this.config.ignoreStage, + resolveConditions: this.config.template.features.conditions, + resolveSubstitutions: this.config.template.features.substitutions, + removeHiddenItems: this.config.removeHiddenTocItems, + }, + }; + + const content = load(await this.run.read(file)) as RawToc; + + if (include && [IncludeMode.RootMerge, IncludeMode.Merge].includes(include.mode)) { + const from = dirname(file); + const to = join(this.run.input, include.mergeBase || dirname(include.from)); + await this.run.copy(from, to, [basename(file)]); + } + + const toc = await loader.call(context, content); + + // If this is not a part of other toc.yaml + if (!include) { + // TODO: we don't need to store tocs in future + // All processing should subscribe on toc.hooks.Resolved + this.tocs.set(path, toc); + await this.walkItems([toc], (item) => { + if (own(item, 'href') && !isExternalHref(item.href)) { + this.entries.add(join(dirname(path), item.href)); + } + + return item; + }); + + freeze(toc); + + await this.hooks.Resolved.promise(toc, path); + } else { + await this.hooks.Included.promise(toc, include); + } + + return toc; + } + + dump(toc: Toc) { + return dump(toc); + } + + async walkItems( + items: T, + actor: (item: RawTocItem) => Promise | WalkStepResult, + ): Promise { + if (!items || !items.length) { + return items; + } + + const results: RawTocItem[] = []; + const queue = [...items]; + while (queue.length) { + const item = queue.shift() as RawTocItem; + + const result = await actor(item); + if (result !== undefined) { + if (Array.isArray(result)) { + results.push(...result); + } else { + results.push(result); + } + } + + if (hasItems(result)) { + result.items = await this.walkItems(result.items, actor); + } + } + + return results as T; + } + + /** + * Resolves toc path and data for any page path + * + * @param {RelativePath} path - any page path + * + * @returns [RelativePath, Toc] + */ + for(path: RelativePath): [RelativePath, Toc] { + // TODO: assert relative + + if (!path) { + throw new Error('Error while finding toc dir.'); + } + + const tocPath = join(dirname(path), 'toc.yaml'); + + if (this.tocs.has(tocPath)) { + return [tocPath, this.tocs.get(tocPath)]; + } + + return this.for(dirname(path)); + } + + async applyIncluder(path: RelativePath, name: string, options: IncluderOptions) { + const hook = this.hooks.Includer.get(name); + + ok(hook, `Includer with name '${name}' is not registered.`); + + const toc = await hook.promise({}, options, path); + + await this.run.write( + join(this.run.input, options.path), + this.run.toc.dump(toc) + ); + } +} + +function hasItems(item: any): item is WithItems { + return item && typeof item === 'object' && item.items && item.items.length; +} diff --git a/src/commands/build/core/toc/__snapshots__/index.spec.ts.snap b/src/commands/build/core/toc/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000..7adcb130 --- /dev/null +++ b/src/commands/build/core/toc/__snapshots__/index.spec.ts.snap @@ -0,0 +1,147 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`toc-loader > includes > should merge deep includes in link mode 1`] = ` +"items: + - name: Outer Item + items: + - name: Inner Item 1 + href: _includes/core/item-1.md + - name: Inner Item 2 + items: + - name: Inner Lib Item 1 + href: _includes/lib/item-1.md +" +`; + +exports[`toc-loader > includes > should merge deep includes in merge mode 1`] = ` +"items: + - name: Outer Item + items: + - name: Inner Item 1 + href: _includes/core/item-1.md + - name: Inner Item 2 + items: + - name: Inner Merge Item 1 + href: _includes/core/merge-item-1.md + - name: Inner Deep Merge Item 1 + href: _includes/core/deep-merge-item-1.md + - name: Inner Sub Item 1 + href: _includes/core/sub-item-1.md +" +`; + +exports[`toc-loader > includes > should merge includes in flat link mode 1`] = ` +"items: + - name: Inner Item 1 +" +`; + +exports[`toc-loader > includes > should merge includes in link mode 1`] = ` +"items: + - name: Outer Item + items: + - name: Inner Item 1 +" +`; + +exports[`toc-loader > includes > should rebase items href for includes in link mode 1`] = ` +"items: + - name: Outer Item + items: + - name: Inner Item 1 + href: _includes/core/item-1.md + items: + - name: Inner Sub Item 1 + href: _includes/core/sub-item-1.md + - name: Inner Item 2 + href: _includes/core/item-2.md + - name: Inner Item 3 + href: _includes/core/sub/item-3.md + - name: Inner Item 4 + href: _includes/core/sub/item-4.md + - name: Inner Item 5 + href: _includes/item-5.md + - name: Inner Item 6 + href: https://example.com + - name: Inner Item 7 + href: //example.com +" +`; + +exports[`toc-loader > should filter hidden item 1`] = ` +"items: + - name: Visible Item 1 + - name: Visible Item 2 +" +`; + +exports[`toc-loader > should filter item with declined rule 1`] = ` +"items: + - name: Visible Item 1 + - name: Visible Item 2 +" +`; + +exports[`toc-loader > should handle filter title 1`] = ` +"title: Title B +" +`; + +exports[`toc-loader > should handle simple title 1`] = ` +"title: Title +" +`; + +exports[`toc-loader > should interpolate conditions in title 1`] = ` +"title: 'Title IF ' +" +`; + +exports[`toc-loader > should interpolate filter title 1`] = ` +"title: Title C +" +`; + +exports[`toc-loader > should interpolate item href 1`] = ` +"items: + - href: ./file.md +" +`; + +exports[`toc-loader > should interpolate item name 1`] = ` +"items: + - name: Item C +" +`; + +exports[`toc-loader > should interpolate nested item 1`] = ` +"items: + - name: Parent + items: + - name: Item C + href: ./file.md +" +`; + +exports[`toc-loader > should interpolate title 1`] = ` +"title: Title C +" +`; + +exports[`toc-loader > should not filter item with accepted rule 1`] = ` +"items: + - name: Visible Item 1 + - name: Visible Item 2 + - name: Visible Item 3 +" +`; + +exports[`toc-loader > should not interpolate title if conditions is disabled 1`] = ` +"title: Title {% if var == "C"%} IF {% endif %} +" +`; + +exports[`toc-loader > should not interpolate title if substitutions is disabled 1`] = ` +"title: Title {{var}} +" +`; diff --git a/src/commands/build/core/toc/includers/generic.spec.ts b/src/commands/build/core/toc/includers/generic.spec.ts new file mode 100644 index 00000000..2792ad27 --- /dev/null +++ b/src/commands/build/core/toc/includers/generic.spec.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import {when} from 'vitest-when'; +import {join} from 'node:path'; + +import {setupBuild, setupRun} from '~/commands/build/__tests__'; + +import {GenericIncluderExtension} from './generic'; + +describe('Generic includer', () => { + it('should work', async () => { + const input = '/dev/null/input' as AbsolutePath; + const output = '/dev/null/output' as AbsolutePath; + const build = setupBuild(); + const run = setupRun({input, output}); + const extension = new GenericIncluderExtension(); + + when(run.glob).calledWith('**/*.md', { + cwd: join(run.input, 'test') + }).thenResolve([ + './index.md', + './test.md', + './sub/sub-1.md', + './sub/sub-2.md', + './sub/sub/sub-3.md', + './skip/sub/sub-1.md', + ] as RelativePath[]); + + extension.apply(build); + + await build.hooks.BeforeAnyRun.promise(run); + + const result = await run.toc.hooks.Includer.for('generic').promise({}, { + input: 'test' as RelativePath, + path: 'test/toc.yaml' as RelativePath, + }, './toc.yaml' as RelativePath); + + console.log(JSON.stringify(result, null, 2)); + }); +}); diff --git a/src/commands/build/core/toc/includers/generic.ts b/src/commands/build/core/toc/includers/generic.ts new file mode 100644 index 00000000..f3ad90b1 --- /dev/null +++ b/src/commands/build/core/toc/includers/generic.ts @@ -0,0 +1,88 @@ +import type { Build } from '~/commands'; +import type { IncluderOptions, RawToc, RawTocItem, Run, YfmString } from '~/commands/build'; + +import {dirname, extname, join, normalize} from 'node:path'; + +const AUTOTITLE = '{$T}'; + +type Options = IncluderOptions<{ + input?: RelativePath; + autotitle?: boolean; + leadingPage?: { + autotitle?: boolean; + name?: string; + }; +}>; + +type Graph = { + [prop: string]: string | Graph; +}; + +// TODO: implement autotitle after md refactoring +// TODO: implement sort +export class GenericIncluderExtension { + apply(program: Build) { + program.hooks.BeforeAnyRun.tap('GenericIncluder', (run: Run) => { + run.toc.hooks.Includer.for('generic').tapPromise('GenericIncluder', async (toc: RawToc, options: Options, path: RelativePath) => { + const input = options.input ? join(dirname(path), options.input) : path; + const files = await run.glob('**/*.md', { + cwd: join(run.input, input), + }); + + return fillToc(toc, graph(files), options); + }); + }); + } +} + +function graph(paths: RelativePath[]): Graph { + const graph: Graph = {}; + + for (const path of paths) { + const chunks: string[] = normalize(path) + .replace(/\\/g, '/') + .split('/'); + + let level: Hash = graph; + while (chunks.length) { + const field = chunks.shift() as string; + + if (!chunks.length) { + level[field.replace(extname(field), '')] = path; + } else { + level[field] = level[field] || {}; + level = level[field]; + } + } + } + + return graph; +} + +function pageName(key: string, options: Options) { + if (key === 'index') { + if (options?.leadingPage?.autotitle) { + return AUTOTITLE; + } + + return options?.leadingPage?.name ?? 'Overview'; + } + + return options.autotitle ? AUTOTITLE : key; +} + +function fillToc(toc: RawToc, graph: Graph, options: Options) { + function item([key, value]: [string, Graph | string]): RawTocItem { + const name: YfmString = pageName(key, options); + + if (typeof value === 'string') { + return {name, href: value}; + } + + return {name, items: Object.entries(value).map(item)} + } + + toc.items = Object.entries(graph).map(item); + + return toc; +} diff --git a/src/commands/build/core/toc/index.spec.ts b/src/commands/build/core/toc/index.spec.ts new file mode 100644 index 00000000..57fbf254 --- /dev/null +++ b/src/commands/build/core/toc/index.spec.ts @@ -0,0 +1,385 @@ +import {join} from 'node:path'; +import {describe, expect, it, vi} from 'vitest'; +import {when} from 'vitest-when'; +import {dedent} from 'ts-dedent'; + +import {TocService, TocServiceConfig} from './TocService'; +import { setupRun } from '~/commands/build/__tests__'; + +type Options = DeepPartial; + +vi.mock('../vars/VarsService'); + +function test(content: string, options: Options = {}, vars = {}, files = {}, copy = []) { + return async () => { + const input = '/dev/null/input' as AbsolutePath; + const output = '/dev/null/output' as AbsolutePath; + const run = setupRun({ + input, + output, + ignoreStage: [], + removeHiddenTocItems: false, + ...options, + template: { + enabled: true, + ...(options.template || {}), + features: { + conditions: true, + substitutions: true, + ...((options.template || {}).features || {}), + }, + }, + }); + const toc = new TocService(run); + + when(run.vars.load).calledWith('./toc.yaml' as RelativePath).thenResolve(vars); + + when(run.read) + .calledWith(join(run.input, './toc.yaml' as RelativePath)) + .thenResolve(content); + + for (const [path, content] of Object.entries(files)) { + when(run.read) + .calledWith(join(run.input, path)) + .thenResolve(content as string); + } + + for (const [from, to] of copy) { + when(run.copy) + .calledWith(join(run.input, from), join(run.input, to), expect.anything()) + .thenResolve(); + } + + const result = await toc.load('./toc.yaml' as RelativePath); + + expect(toc.dump(result)).toMatchSnapshot(); + }; +} + +describe('toc-loader', () => { + it( + 'should handle simple title', + test(dedent` + title: Title + `), + ); + + it( + 'should handle filter title', + test( + dedent` + title: + - text: Title A + when: var == "A" + - text: Title B + when: var == "B" + `, + {}, + {var: 'B'}, + ), + ); + + it('should interpolate title', test(`title: Title {{var}}`, {}, {var: 'C'})); + + it( + 'should interpolate conditions in title', + test( + dedent` + title: Title {% if var == "C"%} IF {% endif %} + `, + {}, + {var: 'C'}, + ), + ); + + it( + 'should interpolate filter title', + test( + dedent` + title: + - text: Title A + when: var == "A" + - text: Title B + when: var == "B" + - text: Title {{var}} + when: var == "C" + `, + {}, + {var: 'C'}, + ), + ); + + it( + 'should not interpolate title if substitutions is disabled', + test( + dedent` + title: Title {{var}} + `, + {template: {features: {substitutions: false}}}, + {var: 'C'}, + ), + ); + + it( + 'should not interpolate title if conditions is disabled', + test( + dedent` + title: Title {% if var == "C"%} IF {% endif %} + `, + {template: {features: {conditions: false}}}, + {var: 'C'}, + ), + ); + + it( + 'should not filter item with accepted rule', + test( + dedent` + items: + - name: Visible Item 1 + - name: Visible Item {{name}} + when: stage == 'test' + - name: Visible Item 3 + `, + {}, + {stage: 'test', name: 2}, + ), + ); + + it( + 'should filter item with declined rule', + test( + dedent` + items: + - name: Visible Item 1 + - name: Item {{name}} + when: stage == 'test' + - name: Visible Item 2 + `, + {}, + {stage: 'dev'}, + ), + ); + + it( + 'should filter hidden item', + test( + dedent` + items: + - name: Visible Item 1 + - name: Hidden Item {{name}} + hidden: true + when: stage == 'test' + - name: Visible Item 2 + when: stage == 'test' + `, + {removeHiddenTocItems: true}, + {stage: 'test'}, + ), + ); + + it( + 'should interpolate item name', + test( + dedent` + items: + - name: Item {{name}} + `, + {}, + {name: 'C'}, + ), + ); + + it( + 'should interpolate item href', + test( + dedent` + items: + - href: "{{file}}" + `, + {}, + {file: './file.md'}, + ), + ); + + it( + 'should interpolate nested item', + test( + dedent` + items: + - name: Parent + items: + - name: Item {{name}} + href: "{{file}}" + `, + {}, + {name: 'C', file: './file.md'}, + ), + ); + + describe('includes', () => { + it( + 'should rebase items href for includes in link mode', + test( + dedent` + items: + - name: Outer Item + include: + path: _includes/core/i-toc.yaml + mode: link + `, + {}, + {}, + { + '_includes/core/i-toc.yaml': dedent` + items: + - name: Inner Item 1 + href: item-1.md + items: + - name: Inner Sub Item 1 + href: sub-item-1.md + - name: Inner Item 2 + href: ./item-2.md + - name: Inner Item 3 + href: ./sub/item-3.md + - name: Inner Item 4 + href: sub/item-4.md + - name: Inner Item 5 + href: ../item-5.md + - name: Inner Item 6 + href: https://example.com + - name: Inner Item 7 + href: //example.com + `, + }, + ), + ); + + it( + 'should merge includes in link mode', + test( + dedent` + items: + - name: Outer Item + include: + path: _includes/core/i-toc.yaml + mode: link + `, + {}, + {}, + { + '_includes/core/i-toc.yaml': dedent` + items: + - name: Inner Item 1 + `, + }, + ), + ); + + it( + 'should merge includes in flat link mode', + test( + dedent` + items: + - include: + path: _includes/core/i-toc.yaml + mode: link + `, + {}, + {}, + { + '_includes/core/i-toc.yaml': dedent` + items: + - name: Inner Item 1 + `, + }, + ), + ); + + it( + 'should merge deep includes in link mode', + test( + dedent` + items: + - name: Outer Item + include: + path: _includes/core/i-toc.yaml + mode: link + `, + {}, + {}, + { + '_includes/core/i-toc.yaml': dedent` + items: + - name: Inner Item 1 + href: item-1.md + - name: Inner Item 2 + include: + path: ../lib/i-toc.yaml + mode: link + `, + '_includes/lib/i-toc.yaml': dedent` + items: + - name: Inner Lib Item 1 + href: item-1.md + `, + }, + ), + ); + + it( + 'should merge deep includes in merge mode', + test( + dedent` + items: + - name: Outer Item + include: + path: _includes/core/toc.yaml + mode: link + `, + {}, + {}, + { + '_includes/core/toc.yaml': dedent` + items: + - name: Inner Item 1 + href: item-1.md + - name: Inner Item 2 + include: + path: ../merge/i-toc.yaml + mode: merge + `, + '_includes/core/sub/toc.yaml': dedent` + items: + - name: Inner Sub Item 1 + href: sub-item-1.md + `, + '_includes/merge/i-toc.yaml': dedent` + items: + - name: Inner Merge Item 1 + href: merge-item-1.md + - include: + path: ../deep-merge/i-toc.yaml + mode: merge + - include: + path: ./sub/toc.yaml + mode: merge + `, + '_includes/deep-merge/i-toc.yaml': dedent` + items: + - name: Inner Deep Merge Item 1 + href: deep-merge-item-1.md + `, + }, + [ + ['_includes/merge', '_includes/core'], + ['_includes/deep-merge', '_includes/core'], + ['_includes/core/sub', '_includes/core'], + ] + ), + ); + }); + + // TODO: implement includers + describe.skip('includers', () => {}); +}); diff --git a/src/commands/build/core/toc/index.ts b/src/commands/build/core/toc/index.ts new file mode 100644 index 00000000..5b40b5c9 --- /dev/null +++ b/src/commands/build/core/toc/index.ts @@ -0,0 +1,6 @@ +export type {IncluderOptions, RawToc, RawTocItem} from './types'; +export type {LoaderContext} from './loader'; + +export {default as loader} from './loader'; +export {TocService} from './TocService'; +export {GenericIncluderExtension} from './includers/generic'; diff --git a/src/commands/build/core/toc/loader.ts b/src/commands/build/core/toc/loader.ts new file mode 100644 index 00000000..38415823 --- /dev/null +++ b/src/commands/build/core/toc/loader.ts @@ -0,0 +1,349 @@ +import type { TocService } from './TocService'; +import { RawToc, RawTocItem, TextFilter } from './types'; + +import { ok } from 'node:assert'; +import { dirname, join, relative } from 'node:path'; +import { omit } from 'lodash'; +import evalExp from '@diplodoc/transform/lib/liquid/evaluation'; +import { liquidSnippet } from '@diplodoc/transform/lib/liquid'; + +import { Stage } from '~/constants'; +import { own } from '~/utils'; +import { isRelative } from './utils'; + +type Toc = { + title?: string; + label?: string; + stage?: string; + navigation?: boolean | Navigation; + items?: TocItem[]; +}; + +type TocItem = { + href: RelativePath; +}; + +export type {RawToc}; + +export type LoaderContext = { + root: AbsolutePath; + path: RelativePath; + base?: RelativePath; + mergeBase?: RelativePath; + mode: TocIncludeMode; + vars: Hash; + options: { + ignoreStage: string[]; + resolveConditions: boolean; + resolveSubstitutions: boolean; + removeHiddenItems: boolean; + }; + toc: TocService; +}; + +export enum IncludeMode { + RootMerge = 'root_merge', + Merge = 'merge', + Link = 'link', +} + +export default async function (this: LoaderContext, toc: RawToc): Promise { + const {ignoreStage} = this.options; + + if (toc.stage && ignoreStage.length && ignoreStage.includes(toc.stage)) { + return toc; + } + + // Designed to be isolated loaders in future + toc = await resolveFields.call(this, toc); + toc = await resolveItems.call(this, toc); + toc = await templateFields.call(this, toc); + toc = await processItems.call(this, toc); + toc = await rebaseItems.call(this, toc); + + return toc; +} + +async function resolveFields(this: LoaderContext, toc: RawToc): Promise { + for (const field of ['title', 'label'] as const) { + const value = toc[field]; + if (value) { + toc[field] = getFirstValuable(value, this.vars); + } + } + + return toc; +} + +async function templateFields(this: LoaderContext, toc: RawToc): Promise { + const {resolveConditions, resolveSubstitutions} = this.options; + const interpolate = (box: Hash, field: string) => { + const value = box[field]; + if (typeof value !== 'string') { + return; + } + + box[field] = liquidSnippet(value, this.vars, this.path, { + substitutions: resolveSubstitutions, + conditions: resolveConditions, + keepNotVar: true, + withSourceMap: false, + }); + }; + + if (!resolveConditions && !resolveSubstitutions) { + return toc; + } + + for (const field of ['title', 'label', 'navigation'] as const) { + interpolate(toc, field); + } + + toc.items = await this.toc.walkItems(toc.items, (item: RawTocItem) => { + for (const field of ['name', 'href'] as const) { + interpolate(item, field); + } + + return item; + }); + + return toc; +} + +async function resolveItems(this: LoaderContext, toc: RawToc): Promise { + const {removeHiddenItems, resolveConditions} = this.options; + + if (!removeHiddenItems && !resolveConditions) { + return toc; + } + + toc.items = await this.toc.walkItems(toc.items, (item: RawTocItem) => { + let when = true; + + if (resolveConditions) { + when = + typeof item.when === 'string' ? evalExp(item.when, this.vars) : item.when !== false; + delete item.when; + } + + if (removeHiddenItems) { + when = when && !item.hidden; + delete item.hidden; + } + + return when ? item : undefined; + }); + + return toc; +} + +async function rebaseItems(this: LoaderContext, toc: RawToc): Promise { + if (this.mode !== IncludeMode.Link) { + return toc; + } + + const rebase = (item: RawTocItem) => { + if (own(item, 'href') && isRelative(item.href)) { + const absBase = join(this.root, dirname(this.base) as RelativePath); + const absPath = join(this.root, this.mergeBase || dirname(this.path), item.href); + + item.href = relative(absBase, absPath); + } + + return item; + }; + + if (own(toc, 'href')) { + rebase(toc as RawTocItem); + } + + toc.items = await this.toc.walkItems(toc.items, rebase); + + return toc; +} + +async function processItems(this: LoaderContext, toc: RawToc): Promise { + toc.items = await this.toc.walkItems(toc.items, async (item: RawTocItem) => { + item = await this.toc.hooks.Item.promise(item, this.path); + + if (!own(item, 'include')) { + return item; + } + + const {include} = item; + + ok(include.path, 'Invalid value for include path.'); + + if (own(item.include, 'includers')) { + ok( + include.mode === IncludeMode.Link || !include.mode, + 'Invalid mode value for include with includers.', + ); + ok(Array.isArray(include.includers), 'Includers should be an array.'); + + if (!include.path.endsWith('toc.yaml')) { + include.path = join(include.path, 'toc.yaml'); + } + + include.mode = IncludeMode.Link; + + const tocPath = join(dirname(this.path), include.path); + for (const includer of include.includers) { + ok(includer.name, 'Includer name should be a string.'); + + await this.toc.applyIncluder(this.path, includer.name, { + ...(omit(includer, 'name')), + path: tocPath, + }); + } + } + + const {mode = IncludeMode.RootMerge} = include; + const {mergeBase, path} = this; + + const includeInfo = { + from: this.path, + path: join(dirname(path), include.path), + mode, + mergeBase + }; + + if ([IncludeMode.RootMerge, IncludeMode.Merge].includes(includeInfo.mode)) { + includeInfo.mergeBase = includeInfo.mergeBase || dirname(path); + includeInfo.path = join(includeInfo.mergeBase, include.path); + } + + const toc = await this.toc.load(includeInfo.path, includeInfo); + delete (item as RawTocItem).include; + + // Should ignore included toc with tech-preview stage. + // TODO(major): remove this + if (toc.stage === Stage.TECH_PREVIEW) { + return item; + } + + if (item.name) { + item.items = (item.items || []).concat(toc.items || []); + + return item; + } else { + return toc.items; + } + }); + + return toc; +} + +function getFirstValuable( + items: TextFilter[] | string, + vars: Hash, + fallback?: T, +): T | undefined { + if (typeof items === 'string') { + items = [{text: items, when: true}]; + } + + if (!Array.isArray(items)) { + items = []; + } + + for (const item of items) { + let {when = true} = item; + delete item.when; + + if (typeof when === 'string') { + when = evalExp(when, vars); + } + + if (when) { + return item.text as T; + } + } + + return fallback; +} + +/** + * Replaces include fields in toc file by resolved toc. + * @param path + * @param items + * @param tocDir + * @param sourcesDir + * @param vars + * @return + * @private + */ +// async function _replaceIncludes( +// path: string, +// items: YfmToc[], +// tocDir: string, +// sourcesDir: string, +// vars: Record, +// ): Promise { +// const result: YfmToc[] = []; +// +// for (const item of items) { +// let includedInlineItems: YfmToc[] | null = null; +// +// if (item.name) { +// const tocPath = join(tocDir, 'toc.yaml'); +// +// item.name = _liquidSubstitutions(item.name, vars, tocPath); +// } +// +// try { +// await applyIncluders(path, item, vars); +// } catch (err) { +// if (err instanceof Error || err instanceof IncludersError) { +// const message = err.toString(); +// +// const file = err instanceof IncludersError ? err.path : path; +// +// logger.error(file, message); +// } +// } +// +// if (item.include) { +// const {mode = IncludeMode.ROOT_MERGE} = item.include; +// const includeTocPath = +// mode === IncludeMode.ROOT_MERGE +// ? resolve(sourcesDir, item.include.path) +// : resolve(tocDir, item.include.path); +// const includeTocDir = dirname(includeTocPath); +// +// try { +// const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc; +// +// /* Save the path to exclude toc from the output directory in the next step */ +// addIncludeTocPath(includeTocPath); +// +// let includedTocItems = (item.items || []).concat(includeToc.items); +// +// /* Resolve nested toc inclusions */ +// const baseTocDir = mode === IncludeMode.LINK ? includeTocDir : tocDir; +// includedTocItems = await processTocItems( +// path, +// includedTocItems, +// baseTocDir, +// sourcesDir, +// vars, +// ); +// } catch (err) { +// const message = `Error while including toc: ${bold(includeTocPath)} to ${bold( +// join(tocDir, 'toc.yaml'), +// )}`; +// +// log.error(message); +// +// continue; +// } finally { +// delete item.include; +// } +// } else if (item.items) { +// item.items = await processTocItems(path, item.items, tocDir, sourcesDir, vars); +// } +// } +// +// return result; +// } +// diff --git a/src/commands/build/core/toc/types.ts b/src/commands/build/core/toc/types.ts new file mode 100644 index 00000000..8aaa0bb4 --- /dev/null +++ b/src/commands/build/core/toc/types.ts @@ -0,0 +1,47 @@ +import type {IncludeMode} from './loader'; + +type YfmString = string & { + __interpolable: true; +}; + +export type Filter = { + when?: string | boolean; +}; + +export type TextFilter = { + text: string; +} & Filter; + +export type WithItems = { + items: RawTocItem[]; +}; + +export type RawToc = { + title?: YfmString | TextFilter[]; + label?: YfmString | TextFilter[]; + stage?: string; + navigation?: boolean | YfmString | Navigation; +} & Partial; + +export type RawTocItem = Filter & + Partial & {hidden?: boolean} & (RawNamedTocItem | RawIncludeTocItem); + +type RawNamedTocItem = { + name: YfmString; + href?: YfmString & (RelativePath | URIString); +}; + +type RawIncludeTocItem = { + name?: YfmString; + include: TocInclude; +}; + +type TocInclude = { + mode?: IncludeMode; + path: RelativePath; + includers?: Includer[]; +}; + +export type IncluderOptions = { + path: RelativePath; +} & T; diff --git a/src/commands/build/core/toc/utils.ts b/src/commands/build/core/toc/utils.ts new file mode 100644 index 00000000..b9e94a22 --- /dev/null +++ b/src/commands/build/core/toc/utils.ts @@ -0,0 +1,3 @@ +export function isRelative(path: AnyPath): path is RelativePath { + return /^\.{1,2}\//.test(path) || !/^(\w{0,7}:)?\/\//.test(path); +} diff --git a/src/commands/build/core/vars/VarsService.ts b/src/commands/build/core/vars/VarsService.ts index 5891ba8b..bf58d1d4 100644 --- a/src/commands/build/core/vars/VarsService.ts +++ b/src/commands/build/core/vars/VarsService.ts @@ -56,7 +56,7 @@ export class VarsService { return this.cache[file]; } - this.logger.proc(path); + this.logger.proc(file); const scopes = []; diff --git a/src/commands/build/features/html/index.ts b/src/commands/build/features/html/index.ts new file mode 100644 index 00000000..d847c2d3 --- /dev/null +++ b/src/commands/build/features/html/index.ts @@ -0,0 +1,32 @@ +import type {Build} from '../..'; + +import {basename, dirname, extname, join} from 'path'; +import {isExternalHref, own} from '~/utils'; + +export class Html { + apply(program: Build) { + program.hooks.BeforeRun.for('html').tap('Html', async (run) => { + run.toc.hooks.Resolved.tapPromise('Html', async (toc, path) => { + const copy = JSON.parse(JSON.stringify(toc)) as Toc; + await run.toc.walkItems([copy], (item) => { + if (own(item, 'href') && !isExternalHref(item.href)) { + if (item.href.endsWith('/')) { + item.href += 'index.yaml'; + } + + const fileExtension: string = extname(item.href); + const filename: string = basename(item.href, fileExtension) + '.html'; + + item.href = join(dirname(path), dirname(item.href), filename); + } + + return item; + }); + + const file = join(run.output, dirname(path), 'toc.js'); + + await run.write(file, `window.__DATA__.data.toc = ${JSON.stringify(copy)};`); + }); + }); + } +} diff --git a/src/commands/build/features/singlepage/index.ts b/src/commands/build/features/singlepage/index.ts index be266f63..7b3c2cf6 100644 --- a/src/commands/build/features/singlepage/index.ts +++ b/src/commands/build/features/singlepage/index.ts @@ -1,7 +1,11 @@ import type {Build} from '~/commands'; import type {Command} from '~/config'; + +import {dirname, join} from 'node:path'; +import {dedent} from 'ts-dedent'; import {defined} from '~/config'; import {options} from './config'; +import {isExternalHref, own} from '~/utils'; export type SinglePageArgs = { singlePage: boolean; @@ -22,5 +26,61 @@ export class SinglePage { return config; }); + + program.hooks.BeforeRun.for('html').tap('SinglePage', (run) => { + run.toc.hooks.Resolved.tapPromise('SinglePage', async (toc, path) => { + const copy = JSON.parse(JSON.stringify(toc)) as Toc; + await run.toc.walkItems([copy], (item) => { + if (own(item, 'href') && !isExternalHref(item.href)) { + item.href = getSinglePageUrl(dirname(path), item.href); + } + + return item; + }); + + const file = join(run.output, dirname(path), 'single-page-toc.js'); + + await run.write(file, `window.__DATA__.data.toc = ${JSON.stringify(copy)};`); + }); + }); } } + +function dropExt(path: string) { + return path.replace(/\.(md|ya?ml|html)$/i, ''); +} + +function toUrl(path: string) { + // replace windows backslashes + return path.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +function relativeTo(root: string, path: string) { + root = toUrl(root); + path = toUrl(path); + + if (root && path.startsWith(root + '/')) { + path = path.replace(root + '/', ''); + } + + return path; +} + +function getAnchorId(tocDir: string, path: string) { + const [pathname, hash] = path.split('#'); + const url = toUrl(dropExt(pathname)) + (hash ? '#' + hash : ''); + + // TODO: encodeURIComponent will be best option + return relativeTo(tocDir, url.replace(/\.\.\/|[/#]/g, '_')); +} + +export function getSinglePageUrl(tocDir: string, path: string) { + const prefix = toUrl(tocDir) || '.'; + const suffix = getAnchorId(tocDir, path); + + if (prefix === '.') { + return '#' + suffix; + } + + return prefix + '/single-page.html#' + suffix; +} diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index abe17161..67da5f06 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -4,10 +4,9 @@ import 'threads/register'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {ArgvService, Includers, PresetService, SearchService} from '~/services'; +import {ArgvService, PresetService, SearchService, TocService} from '~/services'; import { initLinterWorkers, - preparingTocFiles, processAssets, processChangelogs, processExcludedFiles, @@ -21,22 +20,17 @@ export async function handler(run: Run) { try { ArgvService.init(run.legacyConfig); SearchService.init(); - // TODO: Remove duplicated types from openapi-extension - // @ts-ignore - Includers.init([OpenapiIncluder]); + PresetService.init(run.vars); + TocService.init(run.toc); const {lintDisabled, buildDisabled, addMapFile} = ArgvService.getConfig(); - PresetService.init(run.vars); - await preparingTocFiles(run); processExcludedFiles(); if (addMapFile) { prepareMapFile(); } - const outputBundlePath = run.bundlePath; - if (!lintDisabled) { /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ await initLinterWorkers(); @@ -44,7 +38,7 @@ export async function handler(run: Run) { const processes = [ !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), + !buildDisabled && processPages(run), ].filter(Boolean) as Promise[]; await Promise.all(processes); @@ -58,6 +52,7 @@ export async function handler(run: Run) { await SearchService.release(); } } catch (error) { + console.log(error); run.logger.error(error); } finally { processLogs(run.input); diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index acfe3472..322225e4 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -2,6 +2,7 @@ import type {IProgram, ProgramArgs, ProgramConfig} from '~/program'; import type {DocAnalytics} from '@diplodoc/client'; import {ok} from 'node:assert'; +import {join} from 'node:path'; import {pick} from 'lodash'; import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; @@ -23,10 +24,16 @@ import {SinglePage, SinglePageArgs, SinglePageConfig} from './features/singlepag import {Redirects} from './features/redirects'; import {Lint, LintArgs, LintConfig, LintRawConfig} from './features/linter'; import {Changelogs, ChangelogsArgs, ChangelogsConfig} from './features/changelogs'; +import {Html} from './features/html'; import {Search, SearchArgs, SearchConfig, SearchRawConfig} from './features/search'; import {Legacy, LegacyArgs, LegacyConfig, LegacyRawConfig} from './features/legacy'; + +import {GenericIncluderExtension} from './core/toc'; + import shell from 'shelljs'; +export type * from './types'; + export enum ResourceType { style = 'style', script = 'script', @@ -189,6 +196,8 @@ export class Build readonly changelogs = new Changelogs(); + readonly html = new Html(); + readonly search = new Search(); readonly legacy = new Legacy(); @@ -249,7 +258,14 @@ export class Build return config; }); + this.hooks.BeforeRun.for('md').tap('Build', (run) => { + run.toc.hooks.Resolved.tapPromise('Build', async (toc, path) => { + await run.write(join(run.output, path), run.toc.dump(toc)); + }); + }); + this.hooks.AfterRun.for('md').tap('Build', async (run) => { + // TODO: save normalized config instead if (run.config[configPath]) { shell.cp(run.config[configPath], run.output); } @@ -262,8 +278,11 @@ export class Build this.linter.apply(this); this.changelogs.apply(this); this.search.apply(this); + this.html.apply(this); this.legacy.apply(this); + new GenericIncluderExtension().apply(this); + super.apply(program); } @@ -293,6 +312,14 @@ export class Build await run.vars.load(preset); } + const tocs = (await run.glob('**/toc.yaml', { + cwd: run.input, + ignore: run.config.ignore, + })) as RelativePath[]; + for (const toc of tocs) { + await run.toc.load(toc); + } + await Promise.all([handler(run), this.hooks.Run.promise(run)]); await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index 94d07ebc..9d9a5d17 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -19,6 +19,7 @@ import {LogLevel, Logger} from '~/logger'; import {legacyConfig} from './legacy-config'; import {InsecureAccessError} from './errors'; import {VarsService} from './core/vars'; +import {TocService} from './core/toc'; type FileSystem = { access: typeof access; @@ -33,6 +34,7 @@ type FileSystem = { class RunLogger extends Logger { proc = this.topic(LogLevel.INFO, 'PROC'); + copy = this.topic(LogLevel.INFO, 'COPY'); } /** @@ -58,6 +60,8 @@ export class Run { readonly vars: VarsService; + readonly toc: TocService; + get bundlePath() { return join(this.output, BUNDLE_FOLDER); } @@ -87,6 +91,8 @@ export class Run { ]); this.vars = new VarsService(this); + this.toc = new TocService(this); + this.legacyConfig = legacyConfig(this); } @@ -158,6 +164,8 @@ export class Run { dirs.add(dir); } + this.logger.copy(join(from, file), join(to, file)); + await hardlink(join(from, file), join(to, file)); } }; diff --git a/src/commands/build/types.ts b/src/commands/build/types.ts new file mode 100644 index 00000000..b540ba76 --- /dev/null +++ b/src/commands/build/types.ts @@ -0,0 +1,2 @@ +export type * from './core/vars'; +export type * from './core/toc'; diff --git a/src/commands/build/utils.ts b/src/commands/build/utils.ts new file mode 100644 index 00000000..0a6e548f --- /dev/null +++ b/src/commands/build/utils.ts @@ -0,0 +1,37 @@ +import { Hook } from 'tapable'; + +type HookMeta = { + service: string; + hook: string; + name: string; + type: string; +}; + +export function intercept>>(service: string, hooks: T): T { + for (const [hook, handler] of Object.entries(hooks)) { + handler.intercept({ + register: (info) => { + const {type, name, fn} = info; + const meta = {service, hook, name, type}; + + if (type === 'promise') { + info.fn = async (...args: any[]) => { + try { + return await fn(...args); + } catch (error) { + if (error instanceof Error) { + Object.assign(error, {hook: meta}); + } + + throw error; + } + }; + } + + return info; + } + }); + } + + return hooks; +} diff --git a/src/globals.d.ts b/src/globals.d.ts index 474aaea4..6f66fdc1 100644 --- a/src/globals.d.ts +++ b/src/globals.d.ts @@ -25,7 +25,7 @@ type NormalizedPath = string & { type AnyPath = string | UnresolvedPath | AbsolutePath | RelativePath | NormalizedPath; -declare module 'path' { +declare module 'node:path' { namespace path { interface PlatformPath extends PlatformPath { normalize(path: T): T; @@ -49,3 +49,22 @@ declare module 'path' { const path: path.PlatformPath; export = path; } + +declare module 'node:fs/promises' { + import {BufferEncoding, ObjectEncodingOptions} from 'node:fs'; + + export function readFile( + path: AbsolutePath, + options: + | ObjectEncodingOptions + | BufferEncoding, + ): Promise; + + export function realpath( + path: AbsolutePath, + options?: + | ObjectEncodingOptions + | BufferEncoding + | null + ): Promise; +} diff --git a/src/index.ts b/src/index.ts index 12fed307..eb450640 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import type {HookMeta} from './utils'; + import {MAIN_TIMER_ID} from '~/constants'; export type {ICallable, IProgram, ProgramConfig, ProgramArgs} from './program'; @@ -7,6 +9,7 @@ export type {Config, OptionInfo} from './config'; export {Command, option} from './config'; import {Program} from './program'; +import {own} from './utils'; if (require.main === module) { (async () => { @@ -21,8 +24,12 @@ if (require.main === module) { } catch (error: any) { exitCode = 1; - const message = error?.message || error; + if (own(error, 'hook')) { + const {service, hook, name} = error.hook as HookMeta; + console.error(`Intercept error for ${service}.${hook} hook from ${name} extension.`); + } + const message = error?.message || error; if (message) { // eslint-disable-next-line no-console console.error(error.message || error); diff --git a/src/models.ts b/src/models.ts index 16680fd5..e90c7138 100644 --- a/src/models.ts +++ b/src/models.ts @@ -108,8 +108,6 @@ export interface YfmArgv extends YfmConfig { export type DocPreset = { default: YfmPreset; [varsPreset: string]: YfmPreset; -} & { - __metadata: Record[]; }; export interface YfmTocLabel extends Filter { diff --git a/src/services/includers/batteries/common.ts b/src/services/includers/batteries/common.ts deleted file mode 100644 index 447a1a08..00000000 --- a/src/services/includers/batteries/common.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -const { - promises: {readdir}, -} = require('fs'); - -async function getDirs(path: string) { - const isDir = (i: any) => i.isDirectory(); - - return readdir(path, {withFileTypes: true}).then((list: any) => list.filter(isDir)); -} - -async function getFiles(path: string) { - const isFile = (i: any) => i.isFile(); - - return readdir(path, {withFileTypes: true}).then((list: any) => list.filter(isFile)); -} - -const complement = (fn: Function) => (x: any) => !fn(x); - -const isMdExtension = (str: string): boolean => /.md$/gmu.test(str); - -const isHidden = (str: string) => /^\./gmu.test(str); - -const allPass = (predicates: Function[]) => (arg: any) => - predicates.map((fn) => fn(arg)).reduce((p, c) => p && c, true); - -const compose = (fn1: (a: R) => R, ...fns: Array<(a: R) => R>) => - fns.reduce((prevFn, nextFn) => (value) => prevFn(nextFn(value)), fn1); - -const prop = (string: string) => (object: Object) => object[string as keyof typeof object]; - -function concatNewLine(prefix: string, suffix: string) { - return prefix.trim().length ? `${prefix}
${suffix}` : suffix; -} - -export { - complement, - isMdExtension, - isHidden, - allPass, - compose, - prop, - getDirs, - getFiles, - concatNewLine, -}; diff --git a/src/services/includers/batteries/generic.ts b/src/services/includers/batteries/generic.ts deleted file mode 100644 index e6f12226..00000000 --- a/src/services/includers/batteries/generic.ts +++ /dev/null @@ -1,150 +0,0 @@ -import {mkdir, readFile, writeFile} from 'fs/promises'; -import {dirname, join, parse} from 'path'; - -import {updateWith} from 'lodash'; -import {dump} from 'js-yaml'; - -import {getRealPath} from '@diplodoc/transform/lib/utilsFS'; - -import {glob} from 'glob'; - -import {IncluderFunctionParams} from '../../../models'; - -class GenericIncluderError extends Error { - path: string; - - constructor(message: string, path: string) { - super(message); - - this.name = 'GenericIncluderError'; - this.path = path; - } -} - -const name = 'generic'; - -const MD_GLOB = '**/*.md'; - -type Params = { - input: string; - leadingPage: { - name?: string; - }; -}; - -async function includerFunction(params: IncluderFunctionParams) { - const { - readBasePath, - writeBasePath, - tocPath, - item, - passedParams: {input, leadingPage}, - index, - } = params; - - if (!input?.length || !item.include?.path) { - throw new GenericIncluderError('provide includer with input parameter', tocPath); - } - - try { - const leadingPageName = leadingPage?.name ?? 'Overview'; - - const tocDirPath = dirname(tocPath); - - const contentPath = - index === 0 - ? join(writeBasePath, tocDirPath, input) - : join(readBasePath, tocDirPath, input); - - const found = await glob(MD_GLOB, { - cwd: contentPath, - nocase: true, - }); - - const writePath = getRealPath(join(writeBasePath, tocDirPath, item.include.path)); - - if (!writePath.startsWith(writeBasePath)) { - throw new GenericIncluderError( - `Expected the include path to be located inside project root, got: ${writePath}`, - writePath, - ); - } - - await mkdir(writePath, {recursive: true}); - - for (const filePath of found) { - const file = await readFile(join(contentPath, filePath)); - - await mkdir(dirname(join(writePath, filePath)), {recursive: true}); - await writeFile(join(writePath, filePath), file); - } - - const graph = createGraphFromPaths(found); - - const toc = createToc(leadingPageName, item.include.path)(graph, []); - - await writeFile(join(writePath, 'toc.yaml'), dump(toc)); - } catch (err) { - throw new GenericIncluderError(err.toString(), tocPath); - } -} - -function createGraphFromPaths(paths: string[]) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const graph: Record = {}; - - for (const path of paths) { - const chunks = path.split('/').filter(Boolean); - if (chunks.length < 2) { - if (chunks.length === 1) { - graph.files = graph.files ? graph.files.concat(chunks[0]) : chunks; - } - - continue; - } - - const file = chunks.pop(); - - updateWith( - graph, - chunks, - (old) => { - return old ? {files: [...old.files, file]} : {files: [file]}; - }, - Object, - ); - } - - return graph; -} - -function createToc(leadingPageName: string, tocName: string) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return function createTocRec( - graph: Record, - cursor: string[], - ): Record { - const handler = (file: string) => ({ - name: parse(file).name === 'index' ? leadingPageName : file, - href: join(...cursor, file), - }); - - const recurse = (key: string) => createTocRec(graph[key], [...cursor, key]); - - const current = { - name: cursor[cursor.length - 1] ?? tocName, - items: [ - ...(graph.files ?? []).map(handler), - ...Object.keys(graph) - .filter((key) => key !== 'files') - .map(recurse), - ], - }; - - return current; - }; -} - -export {name, includerFunction}; - -export default {name, includerFunction}; diff --git a/src/services/includers/batteries/index.ts b/src/services/includers/batteries/index.ts deleted file mode 100644 index 0170b0a3..00000000 --- a/src/services/includers/batteries/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * as generic from './generic'; -export * as sourcedocs from './sourcedocs'; -export * as unarchive from './unarchive'; diff --git a/src/services/includers/batteries/sourcedocs.ts b/src/services/includers/batteries/sourcedocs.ts deleted file mode 100644 index 388399bc..00000000 --- a/src/services/includers/batteries/sourcedocs.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {logger} from '../../../utils/logger'; - -import generic from './generic'; - -import {IncluderFunctionParams} from '../../../models'; - -const name = 'sourcedocs'; - -const usage = `include: - path: - includers: - - name: generic - input: - leadingPage: - name: -`; - -type Params = { - input: string; - leadingPage: { - name?: string; - }; -}; - -async function includerFunction(params: IncluderFunctionParams) { - logger.warn( - params.tocPath, - `sourcedocs inlcuder is getting depricated in favor of generic includer\n${usage}`, - ); - - await generic.includerFunction(params); -} - -export {name, includerFunction}; - -export default {name, includerFunction}; diff --git a/src/services/includers/batteries/unarchive.ts b/src/services/includers/batteries/unarchive.ts deleted file mode 100644 index a743da0a..00000000 --- a/src/services/includers/batteries/unarchive.ts +++ /dev/null @@ -1,110 +0,0 @@ -import {createReadStream, createWriteStream, mkdirSync} from 'fs'; -import {dirname, join} from 'path'; -import {Headers, extract} from 'tar-stream'; - -import type {PassThrough} from 'stream'; - -import {getRealPath} from '@diplodoc/transform/lib/utilsFS'; - -import {IncluderFunctionParams} from '../../../models'; - -const name = 'unarchive'; - -class UnarchiveIncluderError extends Error { - path: string; - - constructor(message: string, path: string) { - super(message); - - this.name = 'UnarchiveIncluderError'; - this.path = path; - } -} - -function pipeline(readPath: string, writeBasePath: string): Promise { - return new Promise((res, rej) => { - const reader = createReadStream(readPath); - - reader.on('error', (err: Error) => { - rej(err); - }); - - const extractor = extract(); - - extractor.on('error', (err: Error) => { - rej(err); - }); - - mkdirSync(writeBasePath, {recursive: true}); - - extractor.on('entry', (header: Headers, stream: PassThrough, next: Function) => { - const {type, name} = header; - - const writePath = join(writeBasePath, name); - - const writeDirPath = type === 'directory' ? writePath : dirname(writePath); - - mkdirSync(writeDirPath, {recursive: true}); - - if (type !== 'directory') { - const writer = createWriteStream(writePath, {flags: 'w'}); - - writer.on('error', (err) => { - rej(err); - }); - - stream.pipe(writer); - } - - stream.on('end', () => { - next(); - }); - - stream.resume(); - }); - - reader.pipe(extractor).on('finish', () => { - res(); - }); - }); -} - -type Params = { - input: string; - output: string; -}; - -async function includerFunction(params: IncluderFunctionParams) { - const { - readBasePath, - writeBasePath, - tocPath, - passedParams: {input, output}, - index, - } = params; - - if (!input?.length || !output?.length) { - throw new UnarchiveIncluderError('provide includer with input parameter', tocPath); - } - - const contentPath = index === 0 ? join(writeBasePath, input) : join(readBasePath, input); - - const writePath = getRealPath(join(writeBasePath, output)); - - if (!writePath.startsWith(writeBasePath)) { - throw new UnarchiveIncluderError( - `Expected the output parameter to be located inside project root, got: ${output}`, - output, - ); - } - - try { - await pipeline(contentPath, writePath); - } catch (err) { - throw new UnarchiveIncluderError(err.toString(), tocPath); - } -} - -export {name, includerFunction}; - -export default {name, includerFunction}; diff --git a/src/services/includers/index.ts b/src/services/includers/index.ts deleted file mode 100644 index 402c3f22..00000000 --- a/src/services/includers/index.ts +++ /dev/null @@ -1,159 +0,0 @@ -import {join} from 'path'; - -import {isObject} from 'lodash'; - -import {ArgvService} from '../index'; -import {IncludeMode} from '../../constants'; -import {generic, sourcedocs, unarchive} from './batteries'; - -import type { - Includer, - YfmPreset, - YfmToc, - YfmTocInclude, - YfmTocIncluder, - YfmTocIncluders, -} from '../../models'; - -const includersUsage = `include: - path: - includers: - - name: - : - - name: - : -`; - -type IncludersMap = Record; - -let includersMap!: IncludersMap; - -class IncludersError extends Error { - path: string; - - constructor(message: string, path: string) { - super(message); - - this.name = 'IncludersError'; - this.path = path; - } -} - -function init(custom: Includer[] = []) { - if (includersMap) { - return; - } - - includersMap = {generic, sourcedocs, unarchive}; - - for (const includer of custom) { - includersMap[includer.name] = includer; - } -} - -async function applyIncluders(path: string, item: YfmToc, vars: YfmPreset) { - if (!item.include?.includers) { - return; - } - - if (!includeValid(item.include)) { - throw new IncludersError("include doesn't comply with includers standard", path); - } - - // normalize include mode (includers support link mode only) - item.include.mode = IncludeMode.LINK; - - const {status, message} = includersValid(item.include.includers); - if (!status) { - throw new IncludersError(message ?? '', path); - } - - let index = 0; - for (const {name, ...rest} of item.include.includers) { - const includer = getIncluder(name); - const passedParams = {...rest}; - - await applyIncluder({path, item, includer, passedParams, index, vars}); - } - - // contract to be fullfilled by the includer: - // provide builder generated toc.yaml - item.include.path = join(item.include.path, 'toc.yaml'); - index++; -} - -function includeValid(include: YfmTocInclude) { - return (include.mode === IncludeMode.LINK || !include.mode) && include.path?.length; -} - -function includersValid(includers: YfmTocIncluders) { - for (const includer of includers) { - const {status, message} = includerValid(includer); - - if (!status) { - return {status, message}; - } - } - - return {status: true}; -} - -function includerValid(includer: YfmTocIncluder) { - if (isObject(includer)) { - if (typeof includer.name !== 'string') { - return { - status: false, - message: 'use string in the includer.name to specify includers name', - }; - } - - if (includerExists(includer)) { - return {status: true}; - } - - return {status: false, message: `includer ${includer.name} not implemented`}; - } - - return { - status: false, - message: `use appropriate includers format:\n${includersUsage}`, - }; -} - -function getIncluder(includerName: string) { - return includersMap[includerName]; -} - -function includerExists(includer: YfmTocIncluder) { - return includersMap[includer.name as keyof typeof includersMap]; -} - -export type ApplyIncluderParams = { - path: string; - item: YfmToc; - includer: Includer; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - passedParams: Record; - index: number; - vars: YfmPreset; -}; - -async function applyIncluder(args: ApplyIncluderParams) { - const {rootInput: readBasePath, input: writeBasePath} = ArgvService.getConfig(); - - const {path, item, includer, passedParams, index, vars} = args; - - const params = { - tocPath: path, - passedParams, - index, - item, - readBasePath, - writeBasePath, - vars, - }; - - return await includer.includerFunction(params); -} - -export {init, applyIncluders, IncludersError}; diff --git a/src/services/index.ts b/src/services/index.ts index 1a4e2fdf..46898249 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -4,4 +4,3 @@ export {default as ArgvService} from './argv'; export {default as LeadingService} from './leading'; export {default as SearchService} from './search'; export * as PluginService from './plugins'; -export * as Includers from './includers'; diff --git a/src/services/metadata/vcsMetadata.ts b/src/services/metadata/vcsMetadata.ts index 18fbbac6..24ed9309 100644 --- a/src/services/metadata/vcsMetadata.ts +++ b/src/services/metadata/vcsMetadata.ts @@ -35,13 +35,9 @@ const getModifiedTimeISOString = async (options: MetaDataOptions, fileContent: s ); includedFiles.push(relativeFilePath); - const tocCopyFileMap = TocService.getCopyFileMap(); - - const mtimeList = includedFiles - .map((path) => { - const mappedPath = tocCopyFileMap.get(path) || path; - return vcsConnector.getModifiedTimeByPath(mappedPath); - }) + const mappedIncludedFiles = await Promise.all(includedFiles.map(TocService.realpath)); + const mtimeList = mappedIncludedFiles + .map((path) => vcsConnector.getModifiedTimeByPath(path)) .filter((v) => typeof v === 'number') as number[]; if (mtimeList.length) { diff --git a/src/services/tocs.ts b/src/services/tocs.ts index c8da036f..a0e04de0 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -1,445 +1,28 @@ -import {dirname, extname, join, normalize, parse, relative, resolve, sep} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; -import {dump, load} from 'js-yaml'; -import shell from 'shelljs'; -import walkSync from 'walk-sync'; -import {liquidSnippet} from '@diplodoc/transform/lib/liquid'; -import log from '@diplodoc/transform/lib/log'; -import {bold} from 'chalk'; - -import {ArgvService, PresetService} from './index'; import {YfmToc} from '../models'; -import {IncludeMode, Stage} from '../constants'; -import {isExternalHref, logger} from '../utils'; -import {filterFiles, firstFilterItem, firstFilterTextItems, liquidField} from './utils'; -import {IncludersError, applyIncluders} from './includers'; -import {addSourcePath} from './metadata'; +import type {TocService} from '~/commands/build/core/toc'; export interface TocServiceData { - storage: Map; - tocs: Map; navigationPaths: string[]; - includedTocPaths: Set; } -const storage: TocServiceData['storage'] = new Map(); -const paths: Map = new Map(); -const tocs: TocServiceData['tocs'] = new Map(); let navigationPaths: TocServiceData['navigationPaths'] = []; -const includedTocPaths: TocServiceData['includedTocPaths'] = new Set(); const tocFileCopyMap = new Map(); -async function init(tocFilePaths: string[]) { - for (const path of tocFilePaths) { - logger.proc(path); - - await add(path); - await addNavigation(path); - } -} - -async function add(path: string) { - const { - input: inputFolderPath, - output: outputFolderPath, - outputFormat, - ignoreStage, - vars, - } = ArgvService.getConfig(); - - const pathToDir = dirname(path); - const content = readFileSync(resolve(inputFolderPath, path), 'utf8'); - const parsedToc = load(content) as YfmToc; - - // Should ignore toc with specified stage. - if (parsedToc.stage === ignoreStage) { - return; - } - - const combinedVars = { - ...PresetService.get(pathToDir), - ...vars, - }; - - if (parsedToc.title) { - parsedToc.title = firstFilterTextItems(parsedToc.title, combinedVars, { - resolveConditions: true, - }); - } - - if (typeof parsedToc.title === 'string') { - parsedToc.title = liquidField(parsedToc.title, combinedVars, path); - } - - if (typeof parsedToc.navigation === 'string') { - parsedToc.navigation = liquidField(parsedToc.navigation, combinedVars, path); - } - - if (parsedToc.label) { - parsedToc.label = firstFilterItem(parsedToc.label, combinedVars, { - resolveConditions: true, - }); - } - - parsedToc.items = await processTocItems( - path, - parsedToc.items, - join(inputFolderPath, pathToDir), - resolve(inputFolderPath), - combinedVars, - ); - - /* Store parsed toc for .md output format */ - storage.set(path, parsedToc); - - /* save toc in distinct set, storage includes .md files too */ - tocs.set(path, parsedToc); - - if (outputFormat === 'md') { - /* Should copy resolved and filtered toc to output folder */ - const outputPath = resolve(outputFolderPath, path); - const outputToc = dump(parsedToc); - shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputToc); - } -} - -// To collect root toc.yaml we need to move from root into deep -async function addNavigation(path: string) { - const parsedToc = getForPath(path)[1]; - - if (!parsedToc) { - return; - } - - const pathToDir = dirname(path); - prepareNavigationPaths(parsedToc, pathToDir); -} - -async function processTocItems( - path: string, - items: YfmToc[], - tocDir: string, - sourcesDir: string, - vars: Record, -) { - const {resolveConditions, removeHiddenTocItems} = ArgvService.getConfig(); - - let preparedItems = items; - - /* Should remove all links with false expressions */ - if (resolveConditions || removeHiddenTocItems) { - try { - preparedItems = filterFiles(items, 'items', vars, { - resolveConditions, - removeHiddenTocItems, - }); - } catch (error) { - log.error(`Error while filtering toc file: ${path}. Error message: ${error}`); - } - } - - /* Should resolve all includes */ - return _replaceIncludes(path, preparedItems, tocDir, sourcesDir, vars); +let toc: TocService; +async function init(service: TocService) { + toc = service; } function getForPath(path: string): [string | null, YfmToc | null] { - let tocPath = paths.get(path) || null; - let tocData = storage.get(path) || null; - - // TODO: normalize paths on add - if (!tocData && path.endsWith('.md')) { - path = path.replace('.md', ''); - tocPath = paths.get(path) || null; - tocData = storage.get(path) || null; - } - - if (!tocData && path.endsWith('index.yaml')) { - path = path.replace('index.yaml', ''); - tocPath = paths.get(path) || null; - tocData = storage.get(path) || null; - } - - return [tocPath, tocData]; + return toc.for(path); } function getNavigationPaths(): string[] { - return [...navigationPaths]; -} - -function getIncludedTocPaths(): string[] { - return [...includedTocPaths]; -} - -function prepareNavigationPaths(parsedToc: YfmToc, dirPath: string) { - function processItems(items: YfmToc[], pathToDir: string) { - items.forEach((item) => { - if (!parsedToc.singlePage && item.items) { - const preparedSubItems = item.items.map((yfmToc: YfmToc, index: number) => { - // Generate personal id for each navigation item - yfmToc.id = `${yfmToc.name}-${index}-${Math.random()}`; - return yfmToc; - }); - processItems(preparedSubItems, pathToDir); - } - - if (item.href && !isExternalHref(item.href)) { - const href = join(pathToDir, item.href); - storage.set(href, parsedToc); - paths.set(href, dirPath); - - const navigationPath = _normalizeHref(href); - navigationPaths.push(navigationPath); - } - }); - } - - processItems([parsedToc], dirPath); -} - -/** - * Should normalize hrefs. MD and YAML files will be ignored. - * @param href - * @return {string} - * @example instance-groups/create-with-coi/ -> instance-groups/create-with-coi/index.yaml - * @example instance-groups/create-with-coi -> instance-groups/create-with-coi.md - * @private - */ -function _normalizeHref(href: string): string { - const preparedHref = normalize(href); - - if (preparedHref.endsWith('.md') || preparedHref.endsWith('.yaml')) { - return preparedHref; - } - - if (preparedHref.endsWith(sep)) { - return `${preparedHref}index.yaml`; - } - - return `${preparedHref}.md`; -} - -/** - * Copies all files of include toc to original dir. - * @param tocPath - * @param destDir - * @return - * @private - */ -function _copyTocDir(tocPath: string, destDir: string) { - const {input: inputFolderPath} = ArgvService.getConfig(); - - const {dir: tocDir} = parse(tocPath); - const files: string[] = walkSync(tocDir, { - globs: ['**/*.*'], - ignore: ['**/toc.yaml'], - directories: false, - }); - - files.forEach((relPath) => { - const from = resolve(tocDir, relPath); - const to = resolve(destDir, relPath); - const fileExtension = extname(relPath); - const isMdFile = fileExtension === '.md'; - - shell.mkdir('-p', parse(to).dir); - - if (isMdFile) { - const fileContent = readFileSync(from, 'utf8'); - const sourcePath = relative(inputFolderPath, from); - const updatedFileContent = addSourcePath(fileContent, sourcePath); - - writeFileSync(to, updatedFileContent); - } else { - shell.cp(from, to); - } - - const relFrom = relative(inputFolderPath, from); - const relTo = relative(inputFolderPath, to); - tocFileCopyMap.set(relTo, relFrom); - }); -} - -/** - * Make hrefs relative to the main toc in the included toc. - * @param items - * @param includeTocDir - * @param tocDir - * @return - * @private - */ -function _replaceIncludesHrefs(items: YfmToc[], includeTocDir: string, tocDir: string): YfmToc[] { - return items.reduce((acc, tocItem) => { - if (tocItem.href) { - tocItem.href = relative(tocDir, resolve(includeTocDir, tocItem.href)); - } - - if (tocItem.items) { - tocItem.items = _replaceIncludesHrefs(tocItem.items, includeTocDir, tocDir); - } - - if (tocItem.include) { - const {path} = tocItem.include; - tocItem.include.path = relative(tocDir, resolve(includeTocDir, path)); - } - - return acc.concat(tocItem); - }, [] as YfmToc[]); -} - -/** - * Liquid substitutions in toc file. - * @param input - * @param vars - * @param path - * @return {string} - * @private - */ -function _liquidSubstitutions(input: string, vars: Record, path: string) { - const {outputFormat, applyPresets} = ArgvService.getConfig(); - if (outputFormat === 'md' && !applyPresets) { - return input; - } - - return liquidSnippet(input, vars, path, { - conditions: false, - substitutions: true, - }); -} - -function addIncludeTocPath(includeTocPath: string) { - includedTocPaths.add(includeTocPath); -} - -/** - * Replaces include fields in toc file by resolved toc. - * @param path - * @param items - * @param tocDir - * @param sourcesDir - * @param vars - * @return - * @private - */ -async function _replaceIncludes( - path: string, - items: YfmToc[], - tocDir: string, - sourcesDir: string, - vars: Record, -): Promise { - const result: YfmToc[] = []; - - for (const item of items) { - let includedInlineItems: YfmToc[] | null = null; - - if (item.name) { - const tocPath = join(tocDir, 'toc.yaml'); - - item.name = _liquidSubstitutions(item.name, vars, tocPath); - } - - try { - await applyIncluders(path, item, vars); - } catch (err) { - if (err instanceof Error || err instanceof IncludersError) { - const message = err.toString(); - - const file = err instanceof IncludersError ? err.path : path; - - logger.error(file, message); - } - } - - if (item.include) { - const {mode = IncludeMode.ROOT_MERGE} = item.include; - const includeTocPath = - mode === IncludeMode.ROOT_MERGE - ? resolve(sourcesDir, item.include.path) - : resolve(tocDir, item.include.path); - const includeTocDir = dirname(includeTocPath); - - try { - const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc; - - // Should ignore included toc with tech-preview stage. - if (includeToc.stage === Stage.TECH_PREVIEW) { - continue; - } - - if (mode === IncludeMode.MERGE || mode === IncludeMode.ROOT_MERGE) { - _copyTocDir(includeTocPath, tocDir); - } - - /* Save the path to exclude toc from the output directory in the next step */ - addIncludeTocPath(includeTocPath); - - let includedTocItems = (item.items || []).concat(includeToc.items); - - /* Resolve nested toc inclusions */ - const baseTocDir = mode === IncludeMode.LINK ? includeTocDir : tocDir; - includedTocItems = await processTocItems( - path, - includedTocItems, - baseTocDir, - sourcesDir, - vars, - ); - - /* Make hrefs relative to the main toc */ - if (mode === IncludeMode.LINK) { - includedTocItems = _replaceIncludesHrefs( - includedTocItems, - includeTocDir, - tocDir, - ); - } - - if (item.name) { - item.items = includedTocItems; - } else { - includedInlineItems = includedTocItems; - } - } catch (err) { - const message = `Error while including toc: ${bold(includeTocPath)} to ${bold( - join(tocDir, 'toc.yaml'), - )}`; - - log.error(message); - - continue; - } finally { - delete item.include; - } - } else if (item.items) { - item.items = await processTocItems(path, item.items, tocDir, sourcesDir, vars); - } - - if (includedInlineItems) { - result.push(...includedInlineItems); - } else { - result.push(item); - } - } - - return result; + return [...toc.entries]; } function getTocDir(pagePath: string): string { - const {input: inputFolderPath} = ArgvService.getConfig(); - - const tocDir = dirname(pagePath); - const tocPath = resolve(tocDir, 'toc.yaml'); - - if (!tocDir.includes(inputFolderPath)) { - throw new Error('Error while finding toc dir'); - } - - if (existsSync(tocPath)) { - return tocDir; - } - - return getTocDir(tocDir); + return toc.for(pagePath)[0]; } function setNavigationPaths(paths: TocServiceData['navigationPaths']) { @@ -450,19 +33,15 @@ function getCopyFileMap() { return tocFileCopyMap; } -function getAllTocs() { - return [...tocs.entries()]; +function realpath(path: string) { + return toc.realpath(path as RelativePath); } export default { init, - add, - process, getForPath, getNavigationPaths, getTocDir, - getIncludedTocPaths, setNavigationPaths, - getCopyFileMap, - getAllTocs, + realpath, }; diff --git a/src/services/utils.ts b/src/services/utils.ts index 16c5d30d..7c3397c2 100644 --- a/src/services/utils.ts +++ b/src/services/utils.ts @@ -93,26 +93,6 @@ export function firstFilterTextItems( return filteredItems[0] || ''; } -export function firstFilterItem( - itemOrItems: T | T[], - vars: Record, - options?: FilterFilesOptions, -) { - const items = Array.isArray(itemOrItems) ? itemOrItems : [itemOrItems]; - - const filteredItems = items.reduce((result: T[], item) => { - const useItem = shouldProcessItem(item, vars, options); - - if (useItem) { - result.push(item); - } - - return result; - }, []); - - return filteredItems[0]; -} - function shouldProcessItem( item: T, vars: Record, diff --git a/src/steps/index.ts b/src/steps/index.ts index dc7c243b..4e7442fb 100644 --- a/src/steps/index.ts +++ b/src/steps/index.ts @@ -3,5 +3,4 @@ export * from './processExcludedFiles'; export * from './processLogs'; export * from './processPages'; export * from './processLinter'; -export * from './processServiceFiles'; export * from './processChangelogs'; diff --git a/src/steps/processExcludedFiles.ts b/src/steps/processExcludedFiles.ts index 20fb99f2..e855f10f 100644 --- a/src/steps/processExcludedFiles.ts +++ b/src/steps/processExcludedFiles.ts @@ -1,4 +1,4 @@ -import {relative, resolve} from 'path'; +import {resolve} from 'path'; import walkSync from 'walk-sync'; import shell from 'shelljs'; @@ -10,7 +10,7 @@ import {convertBackSlashToSlash} from '../utils'; * @return {void} */ export function processExcludedFiles() { - const {input: inputFolderPath, output: outputFolderPath, ignore} = ArgvService.getConfig(); + const {input: inputFolderPath, ignore} = ArgvService.getConfig(); const allContentFiles: string[] = walkSync(inputFolderPath, { directories: false, @@ -28,15 +28,4 @@ export function processExcludedFiles() { if (excludedFiles.length) { shell.rm('-f', excludedFiles); } - - const includedTocPaths = TocService.getIncludedTocPaths().map((filePath) => { - const relativeTocPath = relative(inputFolderPath, filePath); - const destTocPath = resolve(outputFolderPath, relativeTocPath); - - return convertBackSlashToSlash(destTocPath); - }); - - if (includedTocPaths.length) { - shell.rm('-rf', includedTocPaths); - } } diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index 9ce664b6..ba32314f 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,12 +1,11 @@ import type {DocInnerProps} from '@diplodoc/client'; import {basename, dirname, extname, join, resolve} from 'path'; -import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'fs'; +import {existsSync, readFileSync, writeFileSync} from 'fs'; import log from '@diplodoc/transform/lib/log'; import {asyncify, mapLimit} from 'async'; import {bold} from 'chalk'; import {dump, load} from 'js-yaml'; import shell from 'shelljs'; -import dedent from 'ts-dedent'; import { Lang, @@ -27,21 +26,16 @@ import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '../services'; import {generateStaticMarkup} from '~/pages/document'; import {generateStaticRedirect} from '~/pages/redirect'; -import { - getDepth, - joinSinglePageResults, - logger, - transformToc, - transformTocForSinglePage, -} from '../utils'; +import {getDepth, joinSinglePageResults, logger} from '../utils'; import {getVCSConnector} from '../vcs-connector'; import {VCSConnector} from '../vcs-connector/connector-models'; +import {Run} from '~/commands/build'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { +export async function processPages(run: Run): Promise { const { input: inputFolderPath, output: outputFolderPath, @@ -65,10 +59,10 @@ export async function processPages(outputBundlePath: string): Promise { inputFolderPath, outputFolderPath, outputFormat, - outputBundlePath, + run.bundlePath, ); - logger.proc(pathToFile); + run.logger.proc(pathToFile); const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); @@ -83,15 +77,10 @@ export async function processPages(outputBundlePath: string): Promise { if (singlePage) { await saveSinglePages(); - - if (outputFormat === 'html') { - await saveTocData(transformTocForSinglePage, 'single-page-toc'); - } } if (outputFormat === 'html') { saveRedirectPage(outputFolderPath); - await saveTocData(transformToc, 'toc'); } } @@ -110,7 +99,7 @@ function getPathData( const outputFileName = `${fileBaseName}.${outputFormat}`; const outputPath = resolve(outputDir, outputFileName); const resolvedPathToFile = resolve(inputFolderPath, pathToFile); - const outputTocDir = TocService.getTocDir(resolvedPathToFile); + const outputTocDir = TocService.getTocDir(pathToFile); const pathData: PathData = { pathToFile, @@ -130,23 +119,6 @@ function getPathData( return pathData; } -async function saveTocData(transform: (toc: YfmToc, tocDir: string) => YfmToc, filename: string) { - const tocs = TocService.getAllTocs(); - const {output} = ArgvService.getConfig(); - - for (const [path, toc] of tocs) { - const outputPath = join(output, dirname(path), filename + '.js'); - mkdirSync(dirname(outputPath), {recursive: true}); - writeFileSync( - outputPath, - dedent` - window.__DATA__.data.toc = ${JSON.stringify(transform(toc, dirname(path)))}; - `, - 'utf8', - ); - } -} - async function saveSinglePages() { const { input: inputFolderPath, @@ -168,7 +140,6 @@ async function saveSinglePages() { .replace(/^\/?/, ''); const singlePageBody = joinSinglePageResults( singlePageResults[tocDir], - inputFolderPath, relativeTocDir, ); diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts deleted file mode 100644 index 9c1fafe1..00000000 --- a/src/steps/processServiceFiles.ts +++ /dev/null @@ -1,25 +0,0 @@ -import walkSync from 'walk-sync'; -import log from '@diplodoc/transform/lib/log'; - -import {ArgvService, TocService} from '../services'; - -const getFilePathsByGlobals = (globs: string[]): string[] => { - const {input, ignore = []} = ArgvService.getConfig(); - - return walkSync(input, { - directories: false, - includeBasePath: false, - globs, - ignore, - }); -}; - -export async function preparingTocFiles(): Promise { - try { - const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); - await TocService.init(tocFilePaths); - } catch (error) { - log.error(`Preparing toc.yaml files failed. Error: ${error}`); - throw error; - } -} diff --git a/src/utils/common.ts b/src/utils/common.ts index bae1fff7..a7b219f8 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,3 +1,4 @@ +import type {Hook, HookMap} from 'tapable'; import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash'; import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; @@ -75,3 +76,39 @@ export function freeze(target: T, visited = new Set()): T { return target; } + +export type HookMeta = { + service: string; + hook: string; + name: string; + type: string; +}; + +export function intercept | HookMap>>(service: string, hooks: T): T { + for (const [hook, handler] of Object.entries(hooks)) { + handler.intercept({ + register: (info) => { + const {type, name, fn} = info; + const meta = {service, hook, name, type}; + + if (type === 'promise') { + info.fn = async (...args: any[]) => { + try { + return await fn(...args); + } catch (error) { + if (error instanceof Error) { + Object.assign(error, {hook: meta}); + } + + throw error; + } + }; + } + + return info; + } + }); + } + + return hooks; +} diff --git a/src/utils/index.ts b/src/utils/index.ts index a495e0b1..fb2db113 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -3,6 +3,5 @@ export * from './logger'; export * from './singlePage'; export * from './url'; export * from './path'; -export * from './toc'; export * from './presets'; export * from './file'; diff --git a/src/utils/singlePage.ts b/src/utils/singlePage.ts index c9794b1a..dc211546 100644 --- a/src/utils/singlePage.ts +++ b/src/utils/singlePage.ts @@ -4,6 +4,8 @@ import HTMLElement from 'node-html-parser/dist/nodes/html'; import {parse} from 'node-html-parser'; import {dirname, join} from 'path'; +import {getSinglePageUrl} from '~/commands/build/features/singlepage'; + interface PreprocessSinglePageOptions { path: string; tocDir: string; @@ -21,17 +23,6 @@ function toUrl(path: string) { return path.replace(/\\/g, '/').replace(/^\.\//, ''); } -function relativeTo(root: string, path: string) { - root = toUrl(root); - path = toUrl(path); - - if (root && path.startsWith(root + '/')) { - path = path.replace(root + '/', ''); - } - - return path; -} - function all(root: HTMLElement, selector: string): HTMLElement[] { return Array.from(root.querySelectorAll(selector)); } @@ -122,28 +113,8 @@ export function addMainTitle(root: HTMLElement, options: PreprocessSinglePageOpt } } -function getAnchorId(tocDir: string, path: string) { - const [pathname, hash] = path.split('#'); - const url = toUrl(dropExt(pathname)) + (hash ? '#' + hash : ''); - - // TODO: encodeURIComponent will be best option - return relativeTo(tocDir, url.replace(/\.\.\/|[/#]/g, '_')); -} - -export function getSinglePageUrl(tocDir: string, path: string) { - const prefix = toUrl(tocDir) || '.'; - const suffix = getAnchorId(tocDir, path); - - if (prefix === '.') { - return '#' + suffix; - } - - return prefix + '/single-page.html#' + suffix; -} - export function joinSinglePageResults( singlePageResults: SinglePageResult[], - root: string, tocDir: string, ): string { const delimeter = `
`; diff --git a/src/utils/toc.ts b/src/utils/toc.ts deleted file mode 100644 index 9f903039..00000000 --- a/src/utils/toc.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type {YfmToc} from '~/models'; - -import {basename, dirname, extname, join} from 'node:path'; - -import {filterFiles} from '../services/utils'; -import {isExternalHref} from './url'; -import {getSinglePageUrl} from './singlePage'; - -function baseTransformToc(toc: YfmToc, transformItemHref: (href: string) => string): YfmToc { - const localToc: YfmToc = JSON.parse(JSON.stringify(toc)); - - if (localToc.items) { - localToc.items = filterFiles( - localToc.items, - 'items', - {}, - { - removeHiddenTocItems: true, - }, - ); - } - - const queue = [localToc]; - - while (queue.length) { - const item = queue.shift(); - - if (!item) { - continue; - } - - const {items, href} = item; - - if (items) { - queue.push(...items); - } - - if (href) { - item.href = transformItemHref(href); - } - } - - return localToc; -} - -export function transformToc(toc: YfmToc, tocDir: string) { - return baseTransformToc(toc, (href: string) => { - if (isExternalHref(href)) { - return href; - } - - if (href.endsWith('/')) { - href += 'index.yaml'; - } - - const fileExtension: string = extname(href); - const filename: string = basename(href, fileExtension) + '.html'; - - return join(tocDir, dirname(href), filename); - }); -} - -export function transformTocForSinglePage(toc: YfmToc, tocDir: string) { - return baseTransformToc(toc, (href: string) => { - if (isExternalHref(href)) { - return href; - } - - return getSinglePageUrl(tocDir, href); - }); -} diff --git a/tests/utils.ts b/tests/utils.ts index 4ef01437..66552301 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -39,7 +39,7 @@ export function compareDirectories(outputPath: string) { const filesFromOutput = walkSync(outputPath, { directories: false, includeBasePath: false, - }); + }).sort(); expect(bundleless(JSON.stringify(filesFromOutput, null, 2))).toMatchSnapshot('filelist');