From 4a3ee6dbffa9305dfb4c564133eb6ae72d7cb932 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Thu, 17 Oct 2024 14:05:08 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=20Support=20parts=20as=20external?= =?UTF-8?q?=20files=20and=20in=20the=20site=20config=20(#1586)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rowan Cockett --- .changeset/blue-rocks-pull.md | 7 + .changeset/silly-candles-smile.md | 6 + .changeset/small-tigers-kneel.md | 5 + .changeset/strong-colts-eat.md | 6 + .changeset/stupid-flowers-push.md | 7 + .changeset/thirty-carrots-guess.md | 6 + .changeset/young-months-peel.md | 5 + docs/document-parts.md | 25 ++ docs/website-templates.md | 2 +- package-lock.json | 1 + packages/myst-cli/src/build/cff.ts | 18 +- packages/myst-cli/src/build/jats/single.ts | 8 +- packages/myst-cli/src/build/site/manifest.ts | 5 + packages/myst-cli/src/build/site/watch.ts | 58 +++-- packages/myst-cli/src/build/tex/single.ts | 5 +- packages/myst-cli/src/build/typst.ts | 5 +- .../src/build/utils/getFileContent.ts | 17 +- packages/myst-cli/src/config.ts | 32 ++- packages/myst-cli/src/process/file.ts | 234 ++++++++++++----- packages/myst-cli/src/process/mdast.ts | 30 ++- packages/myst-cli/src/process/site.ts | 241 +++++++++++------- packages/myst-cli/src/project/fromPath.ts | 1 + packages/myst-cli/src/project/fromTOC.ts | 11 +- .../myst-cli/src/project/toc.pattern.spec.ts | 34 +-- packages/myst-cli/src/project/toc.spec.ts | 57 ++++- packages/myst-cli/src/project/types.ts | 2 + packages/myst-cli/src/session/session.spec.ts | 2 +- packages/myst-cli/src/store/reducers.ts | 20 ++ packages/myst-cli/src/store/selectors.ts | 17 ++ .../src/transforms/crossReferences.ts | 34 ++- packages/myst-cli/src/transforms/embed.ts | 34 ++- packages/myst-cli/src/transforms/parts.ts | 5 + packages/myst-cli/src/utils/index.ts | 1 + .../src/utils/resolveFrontmatterParts.ts | 22 ++ packages/myst-common/src/extractParts.spec.ts | 75 ++++++ packages/myst-common/src/extractParts.ts | 79 ++++-- packages/myst-common/src/index.ts | 2 + packages/myst-common/src/types.ts | 5 + packages/myst-config/package.json | 1 + packages/myst-config/src/site/types.ts | 7 +- .../myst-frontmatter/src/project/types.ts | 13 - .../src/project/validators.ts | 37 +-- packages/myst-frontmatter/src/site/types.ts | 13 + .../myst-frontmatter/src/site/validators.ts | 37 ++- packages/myst-spec-ext/src/types.ts | 1 + packages/myst-to-jats/src/frontmatter.ts | 31 ++- packages/myst-to-jats/src/index.ts | 13 +- packages/myst-to-jats/src/types.ts | 9 +- packages/myst-transforms/src/frontmatter.ts | 15 +- packages/mystmd/tests/endToEnd.spec.ts | 64 +++-- 50 files changed, 977 insertions(+), 388 deletions(-) create mode 100644 .changeset/blue-rocks-pull.md create mode 100644 .changeset/silly-candles-smile.md create mode 100644 .changeset/small-tigers-kneel.md create mode 100644 .changeset/strong-colts-eat.md create mode 100644 .changeset/stupid-flowers-push.md create mode 100644 .changeset/thirty-carrots-guess.md create mode 100644 .changeset/young-months-peel.md create mode 100644 packages/myst-cli/src/utils/resolveFrontmatterParts.ts diff --git a/.changeset/blue-rocks-pull.md b/.changeset/blue-rocks-pull.md new file mode 100644 index 000000000..77cdcba1a --- /dev/null +++ b/.changeset/blue-rocks-pull.md @@ -0,0 +1,7 @@ +--- +'myst-to-jats': patch +'myst-common': patch +'myst-cli': patch +--- + +Consume frontmatter parts in static exports diff --git a/.changeset/silly-candles-smile.md b/.changeset/silly-candles-smile.md new file mode 100644 index 000000000..861b3d554 --- /dev/null +++ b/.changeset/silly-candles-smile.md @@ -0,0 +1,6 @@ +--- +'myst-common': patch +'myst-cli': patch +--- + +Load frontmatter parts as separate files for processing diff --git a/.changeset/small-tigers-kneel.md b/.changeset/small-tigers-kneel.md new file mode 100644 index 000000000..0517cdd1f --- /dev/null +++ b/.changeset/small-tigers-kneel.md @@ -0,0 +1,5 @@ +--- +'myst-cli': patch +--- + +Fix local image paths for embedded nodes diff --git a/.changeset/strong-colts-eat.md b/.changeset/strong-colts-eat.md new file mode 100644 index 000000000..e0c5f35fa --- /dev/null +++ b/.changeset/strong-colts-eat.md @@ -0,0 +1,6 @@ +--- +'myst-spec-ext': patch +'myst-cli': patch +--- + +Keep track of implicit vs. explicit pages in project TOC diff --git a/.changeset/stupid-flowers-push.md b/.changeset/stupid-flowers-push.md new file mode 100644 index 000000000..a6c70c501 --- /dev/null +++ b/.changeset/stupid-flowers-push.md @@ -0,0 +1,7 @@ +--- +'myst-frontmatter': patch +'myst-config': patch +'myst-cli': patch +--- + +Support parts in site config diff --git a/.changeset/thirty-carrots-guess.md b/.changeset/thirty-carrots-guess.md new file mode 100644 index 000000000..4ca3fb2d9 --- /dev/null +++ b/.changeset/thirty-carrots-guess.md @@ -0,0 +1,6 @@ +--- +'myst-config': patch +'myst-cli': patch +--- + +Parse project-level parts to mdast diff --git a/.changeset/young-months-peel.md b/.changeset/young-months-peel.md new file mode 100644 index 000000000..f79ea062c --- /dev/null +++ b/.changeset/young-months-peel.md @@ -0,0 +1,5 @@ +--- +'myst-cli': patch +--- + +Update processing to handle parts files diff --git a/docs/document-parts.md b/docs/document-parts.md index abc6f4fa6..2ae15b6c6 100644 --- a/docs/document-parts.md +++ b/docs/document-parts.md @@ -22,6 +22,15 @@ abstract: | --- ``` +You may also write your part in a separate file, and point to that file from the frontmatter, for example: + +```yaml +--- +title: My document +abstract: ../abstract.md +--- +``` + ### Known Frontmatter Parts The known parts that are recognized as _top-level_ document frontmatter keys are: @@ -116,4 +125,20 @@ Project-level `parts` are useful, for example, if you have an abstract, acknowle ```{caution} Project-level `parts` are a new feature and may not yet be respected by your chosen MyST template or export format. If the project `part` is not behaving as you expect, try moving it to page frontmatter for now. +``` + +(parts:site)= + +## Parts in `myst.yml` Site configuration + +You may specify `parts` in the site configuration of your `myst.yml` file. These parts will only be used for MyST site builds, and they must correspond to `parts` declared in your [website theme's template](website-templates.md). + +```yaml +version: 1 +site: + template: ... + parts: + footer: | + (c) MyST Markdown + ... ``` \ No newline at end of file diff --git a/docs/website-templates.md b/docs/website-templates.md index 430f2d59f..3bbd6c61c 100644 --- a/docs/website-templates.md +++ b/docs/website-templates.md @@ -58,7 +58,7 @@ Site options allow you to configure a theme's behavior.[^opts] These should be placed in the `site.options` in your `myst.yml`. For example: -[^opts]: They are generally unique to the theme (and thus in a dediated `site.options` key rather than a top-level option in `site`). +[^opts]: They are generally unique to the theme (and thus in a dedicated `site.options` key rather than a top-level option in `site`). ```{code-block} yaml :filename: myst.yml diff --git a/package-lock.json b/package-lock.json index f0c8b633d..61052b65e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15632,6 +15632,7 @@ "version": "1.7.1", "license": "MIT", "dependencies": { + "myst-common": "^1.7.1", "myst-frontmatter": "^1.7.1", "simple-validators": "^1.1.0" }, diff --git a/packages/myst-cli/src/build/cff.ts b/packages/myst-cli/src/build/cff.ts index 3503a031c..351793ad7 100644 --- a/packages/myst-cli/src/build/cff.ts +++ b/packages/myst-cli/src/build/cff.ts @@ -14,6 +14,7 @@ import { KNOWN_IMAGE_EXTENSIONS } from '../utils/resolveExtension.js'; import type { ExportWithOutput, ExportFnOptions } from './types.js'; import { cleanOutput } from './utils/cleanOutput.js'; import { getFileContent } from './utils/getFileContent.js'; +import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js'; import { parseMyst } from '../process/myst.js'; function exportOptionsToCFF(exportOptions: ExportWithOutput): CFF { @@ -39,11 +40,16 @@ export async function runCffExport( let frontmatter: PageFrontmatter | undefined; let abstract: string | undefined; if (projectPath && selectors.selectLocalConfigFile(state, projectPath) === sourceFile) { + // Process the project only, without any files + await getFileContent(session, [], { + projectPath, + imageExtensions: KNOWN_IMAGE_EXTENSIONS, + extraLinkTransformers, + }); frontmatter = selectors.selectLocalProjectConfig(state, projectPath); - const rawAbstract = frontmatter?.parts?.abstract?.join('\n\n'); - if (rawAbstract) { - const abstractAst = parseMyst(session, rawAbstract, sourceFile); - abstract = toText(abstractAst); + const { abstract: frontmatterAbstract } = resolveFrontmatterParts(session, frontmatter) ?? {}; + if (frontmatterAbstract) { + abstract = toText(frontmatterAbstract.mdast); } } else if (article.file) { const [content] = await getFileContent(session, [article.file], { @@ -52,7 +58,9 @@ export async function runCffExport( extraLinkTransformers, }); frontmatter = content.frontmatter; - const abstractMdast = extractPart(content.mdast, 'abstract'); + const abstractMdast = extractPart(content.mdast, 'abstract', { + frontmatterParts: resolveFrontmatterParts(session, frontmatter), + }); if (abstractMdast) abstract = toText(abstractMdast); } if (!frontmatter) return { tempFolders: [] }; diff --git a/packages/myst-cli/src/build/jats/single.ts b/packages/myst-cli/src/build/jats/single.ts index c72949e45..170d4ba8a 100644 --- a/packages/myst-cli/src/build/jats/single.ts +++ b/packages/myst-cli/src/build/jats/single.ts @@ -10,6 +10,7 @@ import { castSession } from '../../session/cache.js'; import type { ISession } from '../../session/types.js'; import { logMessagesFromVFile } from '../../utils/logging.js'; import { KNOWN_IMAGE_EXTENSIONS } from '../../utils/resolveExtension.js'; +import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js'; import type { ExportWithOutput, ExportFnOptions } from '../types.js'; import { cleanOutput } from '../utils/cleanOutput.js'; import { getFileContent } from '../utils/getFileContent.js'; @@ -59,7 +60,12 @@ export async function runJatsExport( }); }), ); - const [processedArticle, ...processedSubArticles] = processedContents; + const [processedArticle, ...processedSubArticles] = processedContents.map( + ({ frontmatter, ...contents }) => { + const parts = resolveFrontmatterParts(session, frontmatter); + return { frontmatter: { ...frontmatter, parts }, ...contents }; + }, + ); const vfile = new VFile(); vfile.path = output; const jats = writeJats(vfile, processedArticle as any, { diff --git a/packages/myst-cli/src/build/site/manifest.ts b/packages/myst-cli/src/build/site/manifest.ts index fc8e526e6..0318951d2 100644 --- a/packages/myst-cli/src/build/site/manifest.ts +++ b/packages/myst-cli/src/build/site/manifest.ts @@ -18,6 +18,7 @@ import type { RootState } from '../../store/index.js'; import { selectors } from '../../store/index.js'; import { transformBanner, transformThumbnail } from '../../transforms/images.js'; import { addWarningForFile } from '../../utils/addWarningForFile.js'; +import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js'; import version from '../../version.js'; import { getSiteTemplate } from './template.js'; import { collectExportOptions } from '../utils/collectExportOptions.js'; @@ -173,6 +174,7 @@ export async function localToManifestProject( const downloads = projConfigFile ? await resolvePageDownloads(session, projConfigFile, projectPath) : undefined; + const parts = resolveFrontmatterParts(session, projFrontmatter); const banner = await transformBanner( session, path.join(projectPath, 'myst.yml'), @@ -204,6 +206,7 @@ export async function localToManifestProject( undefined, exports, downloads, + parts, bibliography: projFrontmatter.bibliography || [], title: projectTitle || 'Untitled', slug: projectSlug, @@ -405,8 +408,10 @@ export async function getSiteManifest( ); const resolvedOptions = await resolveTemplateFileOptions(session, mystTemplate, validatedOptions); validatedFrontmatter.options = resolvedOptions; + const parts = resolveFrontmatterParts(session, validatedFrontmatter); const manifest: SiteManifest = { ...validatedFrontmatter, + parts, myst: version, nav: nav || [], actions: actions || [], diff --git a/packages/myst-cli/src/build/site/watch.ts b/packages/myst-cli/src/build/site/watch.ts index c55a47a8f..5ce47d9ee 100644 --- a/packages/myst-cli/src/build/site/watch.ts +++ b/packages/myst-cli/src/build/site/watch.ts @@ -41,7 +41,7 @@ function triggerProjectReload( : selectors.selectCurrentProjectFile(state); if (selectors.selectConfigExtensions(state).includes(file)) return true; if (file === projectConfigFile || basename(file) === '_toc.yml') return true; - // Reload project if file is added or remvoed + // Reload project if file is added or removed if (['add', 'unlink'].includes(eventType)) return true; // Otherwise do not reload project return false; @@ -86,38 +86,46 @@ async function processorFn( return; } const projectPath = siteProject.path; - const pageSlug = selectors.selectPageSlug(session.store.getState(), siteProject.path, file); - const dependencies = selectors.selectDependentFiles(session.store.getState(), file); + const state = session.store.getState(); + const pageSlug = selectors.selectPageSlug(state, siteProject.path, file); + const dependencies = selectors.selectDependentFiles(state, file); if (!pageSlug && dependencies.length === 0) { session.log.warn(`⚠️ File is not in project: ${file}`); return; } - if (pageSlug) { - await fastProcessFile(session, { - file, - projectPath, - projectSlug: siteProject.slug, - pageSlug, - ...opts, - }); - } + await fastProcessFile(session, { + file, + projectPath, + projectSlug: siteProject.slug, + pageSlug, + ...opts, + }); if (dependencies.length) { session.log.info( `🔄 Updating dependent pages for ${file} ${chalk.dim(`[${dependencies.join(', ')}]`)}`, ); - await Promise.all([ - dependencies.map((dep) => { - const depSlug = selectors.selectPageSlug(session.store.getState(), projectPath, dep); - if (!depSlug) return undefined; - return fastProcessFile(session, { - file: dep, - projectPath, - projectSlug: siteProject.slug, - pageSlug: depSlug, - ...opts, - }); - }), - ]); + const siteConfig = selectors.selectCurrentSiteFile(state); + const projConfig = selectors.selectCurrentProjectFile(state); + if ( + (siteConfig && dependencies.includes(siteConfig)) || + (projConfig && dependencies.includes(projConfig)) + ) { + await processSite(session, { ...opts, reloadProject: true }); + } else { + await Promise.all([ + dependencies.map(async (dep) => { + const depSlug = selectors.selectPageSlug(state, projectPath, dep); + if (!depSlug) return undefined; + return fastProcessFile(session, { + file: dep, + projectPath, + projectSlug: siteProject.slug, + pageSlug: depSlug, + ...opts, + }); + }), + ]); + } } serverReload(); // TODO: process full site silently and update if there are any diff --git a/packages/myst-cli/src/build/tex/single.ts b/packages/myst-cli/src/build/tex/single.ts index 24d456eee..de1414fcc 100644 --- a/packages/myst-cli/src/build/tex/single.ts +++ b/packages/myst-cli/src/build/tex/single.ts @@ -31,6 +31,7 @@ import { getFileContent } from '../utils/getFileContent.js'; import { addWarningForFile } from '../../utils/addWarningForFile.js'; import { cleanOutput } from '../utils/cleanOutput.js'; import { createTempFolder } from '../../utils/createTempFolder.js'; +import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js'; import type { ExportWithOutput, ExportResults, ExportFnOptions } from '../types.js'; import { writeBibtexFromCitationRenderers } from '../utils/bibtex.js'; @@ -72,7 +73,9 @@ export function extractTexPart( frontmatter: PageFrontmatter, templateYml: TemplateYml, ): LatexResult | LatexResult[] | undefined { - const part = extractPart(mdast, partDefinition.id); + const part = extractPart(mdast, partDefinition.id, { + frontmatterParts: resolveFrontmatterParts(session, frontmatter), + }); if (!part) return undefined; if (!partDefinition.as_list) { // Do not build glossaries when extracting parts: references cannot be mapped to definitions diff --git a/packages/myst-cli/src/build/typst.ts b/packages/myst-cli/src/build/typst.ts index d23a05b50..eabbeb687 100644 --- a/packages/myst-cli/src/build/typst.ts +++ b/packages/myst-cli/src/build/typst.ts @@ -32,6 +32,7 @@ import { logMessagesFromVFile } from '../utils/logging.js'; import { getFileContent } from './utils/getFileContent.js'; import { addWarningForFile } from '../utils/addWarningForFile.js'; import { createTempFolder } from '../utils/createTempFolder.js'; +import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js'; import version from '../version.js'; import { cleanOutput } from './utils/cleanOutput.js'; import type { ExportWithOutput, ExportResults, ExportFnOptions } from './types.js'; @@ -90,7 +91,9 @@ export function extractTypstPart( frontmatter: PageFrontmatter, templateYml: TemplateYml, ): TypstResult | TypstResult[] | undefined { - const part = extractPart(mdast, partDefinition.id); + const part = extractPart(mdast, partDefinition.id, { + frontmatterParts: resolveFrontmatterParts(session, frontmatter), + }); if (!part) return undefined; if (!partDefinition.as_list) { // Do not build glossaries when extracting parts: references cannot be mapped to definitions diff --git a/packages/myst-cli/src/build/utils/getFileContent.ts b/packages/myst-cli/src/build/utils/getFileContent.ts index aa49f6af1..058b6d29d 100644 --- a/packages/myst-cli/src/build/utils/getFileContent.ts +++ b/packages/myst-cli/src/build/utils/getFileContent.ts @@ -9,6 +9,7 @@ import type { TransformFn } from '../../process/mdast.js'; import { postProcessMdast, transformMdast } from '../../process/mdast.js'; import { loadProject, selectPageReferenceStates } from '../../process/site.js'; import type { ISession } from '../../session/types.js'; +import { selectors } from '../../store/index.js'; import type { ImageExtensions } from '../../utils/resolveExtension.js'; export async function getFileContent( @@ -37,13 +38,11 @@ export async function getFileContent( projectPath = projectPath ?? resolve('.'); const { project, pages } = await loadProject(session, projectPath); const projectFiles = pages.map((page) => page.file).filter((file) => !files.includes(file)); - // Keep 'files' indices consistent in 'allFiles' as index is used for other fields. - const allFiles = [...files, ...projectFiles]; await Promise.all([ // Load all citations (.bib) ...project.bibliography.map((path) => loadFile(session, path, projectPath, '.bib')), // Load all content (.md, .tex, .myst.json, or .ipynb) - ...allFiles.map((file, ind) => { + ...[...files, ...projectFiles].map((file, ind) => { const preFrontmatter = Array.isArray(preFrontmatters) ? preFrontmatters?.[ind] : preFrontmatters; @@ -57,6 +56,10 @@ export async function getFileContent( // Consolidate all citations onto single project citation renderer combineProjectCitationRenderers(session, projectPath); + const projectParts = selectors.selectProjectParts(session.store.getState(), projectPath); + // Keep 'files' indices consistent in 'allFiles' as index is used for other fields. + const allFiles = [...files, ...projectFiles, ...projectParts]; + await Promise.all( allFiles.map(async (file, ind) => { const pageSlug = pages.find((page) => page.file === file)?.slug; @@ -80,13 +83,17 @@ export async function getFileContent( return { file }; }), ); - const selectedFiles = await Promise.all( - files.map(async (file) => { + await Promise.all( + [...files, ...projectParts].map(async (file) => { await postProcessMdast(session, { file, extraLinkTransformers, pageReferenceStates, }); + }), + ); + const selectedFiles = await Promise.all( + files.map(async (file) => { const selectedFile = selectFile(session, file); if (!selectedFile) throw new Error(`Could not load file information for ${file}`); return selectedFile; diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts index 9a9a31060..f9d9cc0b1 100644 --- a/packages/myst-cli/src/config.ts +++ b/packages/myst-cli/src/config.ts @@ -16,6 +16,7 @@ import { } from 'simple-validators'; import { VFile } from 'vfile'; import { prepareToWrite } from './frontmatter.js'; +import { loadFrontmatterParts } from './process/file.js'; import { cachePath, loadFromCache, writeToCache } from './session/cache.js'; import type { ISession } from './session/types.js'; import { selectors } from './store/index.js'; @@ -361,6 +362,7 @@ async function resolveSiteConfigPaths( allowRemote?: boolean; }, ) => string | Promise, + file: string, ) { const resolvedFields: SiteConfig = {}; if (siteConfig.projects) { @@ -376,6 +378,15 @@ async function resolveSiteConfigPaths( if (siteConfig.favicon) { resolvedFields.favicon = await resolutionFn(session, path, siteConfig.favicon); } + if (siteConfig.parts) { + resolvedFields.parts = await loadFrontmatterParts( + session, + file, + 'site.parts', + { parts: siteConfig.parts }, + path, + ); + } return { ...siteConfig, ...resolvedFields }; } @@ -392,12 +403,13 @@ async function resolveProjectConfigPaths( allowRemote?: boolean; }, ) => string | Promise, + file: string, ) { const resolvedFields: ProjectConfig = {}; if (projectConfig.bibliography) { resolvedFields.bibliography = await Promise.all( - projectConfig.bibliography.map(async (file) => { - const resolved = await resolutionFn(session, path, file); + projectConfig.bibliography.map(async (f) => { + const resolved = await resolutionFn(session, path, f); return resolved; }), ); @@ -419,6 +431,15 @@ async function resolveProjectConfigPaths( }), ); } + if (projectConfig.parts) { + resolvedFields.parts = await loadFrontmatterParts( + session, + file, + 'project.parts', + { parts: projectConfig.parts }, + path, + ); + } return { ...projectConfig, ...resolvedFields }; } @@ -437,7 +458,7 @@ async function validateSiteConfigAndThrow( const errorSuffix = vfile.path ? ` in ${vfile.path}` : ''; throw Error(`Please address invalid site config${errorSuffix}`); } - return resolveSiteConfigPaths(session, path, site, resolveToAbsolute); + return resolveSiteConfigPaths(session, path, site, resolveToAbsolute, vfile.path); } function saveSiteConfig(session: ISession, path: string, site: SiteConfig) { @@ -459,7 +480,7 @@ async function validateProjectConfigAndThrow( const errorSuffix = vfile.path ? ` in ${vfile.path}` : ''; throw Error(`Please address invalid project config${errorSuffix}`); } - return resolveProjectConfigPaths(session, path, project, resolveToAbsolute); + return resolveProjectConfigPaths(session, path, project, resolveToAbsolute, vfile.path); } function saveProjectConfig(session: ISession, path: string, project: ProjectConfig) { @@ -499,7 +520,7 @@ export async function writeConfigs( } siteConfig = selectors.selectLocalSiteConfig(session.store.getState(), path); if (siteConfig) { - siteConfig = await resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative); + siteConfig = await resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative, file); } // Get project config to save if (projectConfig) { @@ -517,6 +538,7 @@ export async function writeConfigs( path, projectConfig, resolveToRelative, + file, ); } // Return early if nothing new to save diff --git a/packages/myst-cli/src/process/file.ts b/packages/myst-cli/src/process/file.ts index 735ce8195..ba7dcf63a 100644 --- a/packages/myst-cli/src/process/file.ts +++ b/packages/myst-cli/src/process/file.ts @@ -13,10 +13,10 @@ import { SourceFileKind } from 'myst-spec-ext'; import { frontmatterValidationOpts, getPageFrontmatter } from '../frontmatter.js'; import type { ISession, ISessionWithCache } from '../session/types.js'; import { castSession } from '../session/cache.js'; -import { warnings, watch } from '../store/reducers.js'; +import { config, projects, warnings, watch } from '../store/reducers.js'; import type { PreRendererData, RendererData } from '../transforms/types.js'; import { logMessagesFromVFile } from '../utils/logging.js'; -import { parseFilePath } from '../utils/resolveExtension.js'; +import { isValidFile, parseFilePath } from '../utils/resolveExtension.js'; import { addWarningForFile } from '../utils/addWarningForFile.js'; import { loadBibTeXCitationRenderers } from './citations.js'; import { parseMyst } from './myst.js'; @@ -25,7 +25,11 @@ import { selectors } from '../store/index.js'; import { defined, incrementOptions, validateObjectKeys, validateEnum } from 'simple-validators'; import type { ValidationOptions } from 'simple-validators'; -type LoadFileOptions = { preFrontmatter?: Record; keepTitleNode?: boolean }; +type LoadFileOptions = { + preFrontmatter?: Record; + keepTitleNode?: boolean; + kind?: SourceFileKind; +}; export type LoadFileResult = { kind: SourceFileKind; @@ -59,7 +63,7 @@ export function loadMdFile( opts?.preFrontmatter, opts?.keepTitleNode, ); - return { kind: SourceFileKind.Article, mdast, frontmatter, identifiers }; + return { kind: opts?.kind ?? SourceFileKind.Article, mdast, frontmatter, identifiers }; } export async function loadNotebookFile( @@ -87,7 +91,7 @@ export async function loadNotebookFile( nbFrontmatter, frontmatterValidationOpts(vfile), ); - return { kind: SourceFileKind.Notebook, mdast, frontmatter, identifiers, widgets }; + return { kind: opts?.kind ?? SourceFileKind.Notebook, mdast, frontmatter, identifiers, widgets }; } export function loadTexFile( @@ -113,7 +117,11 @@ export function loadTexFile( frontmatterValidationOpts(vfile), ); logMessagesFromVFile(session, vfile); - return { kind: SourceFileKind.Article, mdast: tex.ast as GenericParent, frontmatter }; + return { + kind: opts?.kind ?? SourceFileKind.Article, + mdast: tex.ast as GenericParent, + frontmatter, + }; } export function mystJSONValidationOpts( @@ -191,10 +199,19 @@ export function loadMySTJSON( opts?.keepTitleNode, ); logMessagesFromVFile(session, vfile); - return { mdast, kind, frontmatter, identifiers }; + return { mdast, kind: opts?.kind ?? kind, frontmatter, identifiers }; } } +function getLocation(file: string, projectPath?: string) { + let location = file; + if (projectPath) { + location = `/${path.relative(projectPath, file)}`; + } + // ensure forward slashes and not windows backslashes + return location.replaceAll('\\', '/'); +} + /** * Attempt to load a file into the current session. Unsupported files with * issue a warning @@ -224,64 +241,60 @@ export async function loadFile( session.store.dispatch(warnings.actions.clearWarnings({ file })); const cache = castSession(session); let success = true; - - let location = file; - if (projectPath) { - location = `/${path.relative(projectPath, file)}`; - } - // ensure forward slashes and not windows backslashes - location = location.replaceAll('\\', '/'); - + let pre: PreRendererData | undefined; + let successMessage: string | undefined; try { const content = fs.readFileSync(file).toString(); const { sha256, useCache } = checkCache(cache, content, file); if (useCache) { - session.log.debug(toc(`loadFile: ${file} already loaded.`)); - return cache.$getMdast(file)?.pre; - } - const ext = extension || parseFilePath(file).ext.toLowerCase(); - let loadResult: LoadFileResult | undefined; - switch (ext) { - case '.md': { - loadResult = loadMdFile(session, content, file, opts); - break; - } - case '.ipynb': { - loadResult = await loadNotebookFile(session, content, file, opts); - break; + successMessage = `loadFile: ${file} already loaded.`; + pre = cache.$getMdast(file)?.pre; + } else { + const ext = extension || parseFilePath(file).ext.toLowerCase(); + let loadResult: LoadFileResult | undefined; + switch (ext) { + case '.md': { + loadResult = loadMdFile(session, content, file, opts); + break; + } + case '.ipynb': { + loadResult = await loadNotebookFile(session, content, file, opts); + break; + } + case '.tex': { + loadResult = loadTexFile(session, content, file, opts); + break; + } + case '.bib': { + const renderers = await loadBibTeXCitationRenderers(session, file); + cache.$citationRenderers[file] = renderers; + Object.entries(renderers).forEach(([id, renderer]) => { + const normalizedDOI = doi.normalize(renderer.getDOI())?.toLowerCase(); + if (!normalizedDOI || cache.$doiRenderers[normalizedDOI]) return; + cache.$doiRenderers[normalizedDOI] = { id, render: renderer }; + }); + break; + } + case '.myst.json': { + loadResult = loadMySTJSON(session, content, file); + break; + } + default: + addWarningForFile(session, file, 'Unrecognized extension', 'error', { + ruleId: RuleId.mystFileLoads, + }); + session.log.info( + `"${file}": Please rerun the build with "-c" to ensure the built files are cleared.`, + ); + success = false; } - case '.tex': { - loadResult = loadTexFile(session, content, file, opts); - break; - } - case '.bib': { - const renderers = await loadBibTeXCitationRenderers(session, file); - cache.$citationRenderers[file] = renderers; - Object.entries(renderers).forEach(([id, renderer]) => { - const normalizedDOI = doi.normalize(renderer.getDOI())?.toLowerCase(); - if (!normalizedDOI || cache.$doiRenderers[normalizedDOI]) return; - cache.$doiRenderers[normalizedDOI] = { id, render: renderer }; + if (loadResult) { + pre = { file, location: getLocation(file, projectPath), ...loadResult }; + cache.$setMdast(file, { + sha256, + pre, }); - break; - } - case '.myst.json': { - loadResult = loadMySTJSON(session, content, file); - break; } - default: - addWarningForFile(session, file, 'Unrecognized extension', 'error', { - ruleId: RuleId.mystFileLoads, - }); - session.log.info( - `"${file}": Please rerun the build with "-c" to ensure the built files are cleared.`, - ); - success = false; - } - if (loadResult) { - cache.$setMdast(file, { - sha256, - pre: { file, location, ...loadResult }, - }); } } catch (error) { session.log.debug(`\n\n${(error as Error)?.stack}\n\n`); @@ -290,8 +303,109 @@ export async function loadFile( }); success = false; } - if (success) session.log.debug(toc(`loadFile: loaded ${file} in %s.`)); - return cache.$getMdast(file)?.pre; + if (success) session.log.debug(successMessage ?? toc(`loadFile: loaded ${file} in %s.`)); + if (pre?.frontmatter) { + pre.frontmatter.parts = await loadFrontmatterParts( + session, + file, + 'parts', + pre.frontmatter, + projectPath, + ); + } + return pre; +} + +export async function loadFrontmatterParts( + session: ISession, + file: string, + property: string, + frontmatter: PageFrontmatter, + projectPath?: string, +) { + const { parts, ...pageFrontmatter } = frontmatter; + const vfile = new VFile(); + vfile.path = file; + const modifiedParts: [string, string[]][] = await Promise.all( + Object.entries(parts ?? {}).map(async ([part, contents]) => { + let partFile: string; + if (contents.length === 1 && isValidFile(contents[0])) { + partFile = path.resolve(path.dirname(file), contents[0]); + if (!fs.existsSync(partFile)) { + fileWarn(vfile, `Part file does not exist: ${partFile}`); + return [part, contents]; + } + await loadFile(session, partFile, projectPath, undefined, { + kind: SourceFileKind.Part, + /** Frontmatter from the source page is prioritized over frontmatter from the part file itself */ + preFrontmatter: pageFrontmatter, + }); + session.store.dispatch( + watch.actions.addLocalDependency({ + path: file, + dependency: partFile, + }), + ); + const proj = selectors.selectLocalProject(session.store.getState(), projectPath ?? '.'); + if (proj?.index === partFile) { + fileWarn(vfile, `index file is also used as a part: ${partFile}`); + } else if (proj) { + const filteredPages = proj.pages.filter((page) => { + const { file: pageFile, implicit } = page as any; + if (!pageFile) return true; + if (pageFile !== partFile) return true; + if (!implicit) { + fileWarn(vfile, `project file is also used as a part: ${partFile}`); + } + return !implicit; + }); + const newProj = { ...proj, pages: filteredPages }; + session.store.dispatch(projects.actions.receive(newProj)); + } + } else { + const cache = castSession(session); + partFile = `${path.resolve(file)}#${property}.${part}`; + if (contents.length !== 1 || contents[0] !== partFile || !cache.$getMdast(contents[0])) { + const mdast = { + type: 'root', + children: contents.map((content) => { + const root = parseMyst(session, content, file); + return { + type: 'block', + data: { part }, + children: root.children, + }; + }), + }; + cache.$setMdast(partFile, { + pre: { + kind: SourceFileKind.Part, + file: partFile, + mdast, + // Same frontmatter as the containing page + frontmatter: { ...pageFrontmatter }, + location: getLocation(file, projectPath), + }, + }); + } + session.store.dispatch( + config.actions.receiveFilePart({ + partFile, + file, + }), + ); + } + session.store.dispatch( + config.actions.receiveProjectPart({ + partFile, + path: projectPath ?? '.', + }), + ); + return [part, [partFile]]; + }), + ); + logMessagesFromVFile(session, vfile); + return Object.fromEntries(modifiedParts); } /** diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 9c4bce996..0ad708b83 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -74,7 +74,6 @@ import type { ImageExtensions } from '../utils/resolveExtension.js'; import { logMessagesFromVFile } from '../utils/logging.js'; import { combineCitationRenderers } from './citations.js'; import { bibFilesInDir, selectFile } from './file.js'; -import { frontmatterPartsTransform } from '../transforms/parts.js'; import { parseMyst } from './myst.js'; import { kernelExecutionTransform, LocalDiskCache } from 'myst-execute'; import type { IOutput } from '@jupyterlab/nbformat'; @@ -99,6 +98,14 @@ export type TransformFn = ( opts: Parameters[1], ) => Promise; +function referenceFileFromPartFile(session: ISession, partFile: string) { + const state = session.store.getState(); + const partDeps = selectors.selectDependentFiles(state, partFile); + if (partDeps.length > 0) return partDeps[0]; + const file = selectors.selectFileFromPart(state, partFile); + return file ?? partFile; +} + export async function transformMdast( session: ISession, opts: { @@ -164,10 +171,14 @@ export async function transformMdast( const references: References = { cite: { order: [], data: {} }, }; - const state = new ReferenceState(file, { numbering: frontmatter.numbering, identifiers, vfile }); + const refFile = kind === SourceFileKind.Part ? referenceFileFromPartFile(session, file) : file; + const state = new ReferenceState(refFile, { + numbering: frontmatter.numbering, + identifiers, + vfile, + }); cache.$internalReferences[file] = state; // Import additional content from mdast or other files - frontmatterPartsTransform(session, file, mdast, frontmatter); importMdastFromJson(session, file, mdast); await includeFilesTransform(session, file, mdast, frontmatter, vfile); rawDirectiveTransform(mdast, vfile); @@ -242,10 +253,15 @@ export async function transformMdast( if (isJupytext) transformLiftCodeBlocksInJupytext(mdast); const sha256 = selectors.selectFileInfo(store.getState(), file).sha256 as string; const useSlug = pageSlug !== index; - const url = projectSlug - ? `/${projectSlug}/${useSlug ? pageSlug : ''}` - : `/${useSlug ? pageSlug : ''}`; - const dataUrl = projectSlug ? `/${projectSlug}/${pageSlug}.json` : `/${pageSlug}.json`; + let url: string | undefined; + let dataUrl: string | undefined; + if (pageSlug && projectSlug) { + url = `/${projectSlug}/${useSlug ? pageSlug : ''}`; + dataUrl = `/${projectSlug}/${pageSlug}.json`; + } else if (pageSlug) { + url = `/${useSlug ? pageSlug : ''}`; + dataUrl = `/${pageSlug}.json`; + } updateFileInfoFromFrontmatter(session, file, frontmatter, url, dataUrl); const data: RendererData = { kind: isJupytext ? SourceFileKind.Notebook : kind, diff --git a/packages/myst-cli/src/process/site.ts b/packages/myst-cli/src/process/site.ts index 3331942ce..c6f1003ca 100644 --- a/packages/myst-cli/src/process/site.ts +++ b/packages/myst-cli/src/process/site.ts @@ -32,9 +32,11 @@ import { castSession } from '../session/cache.js'; import type { ISession } from '../session/types.js'; import { selectors } from '../store/index.js'; import { watch } from '../store/reducers.js'; +import type { MystData } from '../transforms/crossReferences.js'; import { addWarningForFile } from '../utils/addWarningForFile.js'; import { logMessagesFromVFile } from '../utils/logging.js'; import { ImageExtensions } from '../utils/resolveExtension.js'; +import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js'; import version from '../version.js'; import { combineProjectCitationRenderers } from './citations.js'; import { loadFile, selectFile } from './file.js'; @@ -118,34 +120,37 @@ function getReferenceTitleAsText(targetNode: Node): string | undefined { * @param states page reference states */ export async function writeMystXRefJson(session: ISession, states: ReferenceState[]) { + const references = states + .filter((state): state is ReferenceState & { url: string; dataUrl: string } => { + return !!state.url && !!state.dataUrl; + }) + .map((state) => { + const { url, dataUrl } = state; + const data = `/content${dataUrl}`; + const pageRef = { kind: 'page', data, url }; + const pageIdRefs = state.identifiers.map((identifier) => { + return { identifier, kind: 'page', data, url }; + }); + const targetRefs = Object.values(state.targets).map((target) => { + const { identifier, html_id } = target.node ?? {}; + return { + identifier, + html_id: html_id !== identifier ? html_id : undefined, + kind: target.kind, + data, + url, + implicit: (target.node as any).implicit, + }; + }); + return [pageRef, ...pageIdRefs, ...targetRefs]; + }) + .flat(); const mystXRefs: MystXRefs = { version: '1', myst: version, - references: states - .filter((state): state is ReferenceState & { url: string; dataUrl: string } => { - return !!state.url && !!state.dataUrl; - }) - .map((state) => { - const { url, dataUrl } = state; - const data = `/content${dataUrl}`; - const pageRef = { kind: 'page', data, url }; - const pageIdRefs = state.identifiers.map((identifier) => { - return { identifier, kind: 'page', data, url }; - }); - const targetRefs = Object.values(state.targets).map((target) => { - const { identifier, html_id } = target.node ?? {}; - return { - identifier, - html_id: html_id !== identifier ? html_id : undefined, - kind: target.kind, - data, - url, - implicit: (target.node as any).implicit, - }; - }); - return [pageRef, ...pageIdRefs, ...targetRefs]; - }) - .flat(), + references: [...new Set(references.map((ref) => JSON.stringify(ref)))].map((ref) => { + return JSON.parse(ref); + }), }; const filename = join(session.sitePath(), 'myst.xref.json'); session.log.debug(`Writing myst.xref.json file: ${filename}`); @@ -229,26 +234,28 @@ export async function writeObjectsInv( // TODO: allow a version on the project?! version: String((siteConfig as any)?.version), }); - states.forEach((state) => { - inv.setEntry({ - type: Domains.stdDoc, - name: (state.url as string).replace(/^\//, ''), - location: state.url as string, - display: state.title ?? '', - }); - Object.entries(state.targets).forEach(([name, target]) => { - if ((target.node as any).implicit) { - // Don't include implicit references - return; - } + states + .filter((state): state is ReferenceState & { url: string } => !!state.url) + .forEach((state) => { inv.setEntry({ - type: Domains.stdLabel, - name, - location: `${state.url}#${(target.node as any).html_id ?? target.node.identifier}`, - display: getReferenceTitleAsText(target.node), + type: Domains.stdDoc, + name: state.url.replace(/^\//, ''), + location: state.url, + display: state.title ?? '', + }); + Object.entries(state.targets).forEach(([name, target]) => { + if ((target.node as any).implicit) { + // Don't include implicit references + return; + } + inv.setEntry({ + type: Domains.stdLabel, + name, + location: `${state.url}#${(target.node as any).html_id ?? target.node.identifier}`, + display: getReferenceTitleAsText(target.node), + }); }); }); - }); const filename = join(session.sitePath(), 'objects.inv'); session.log.debug(`Writing objects.inv file: ${filename}`); inv.write(filename); @@ -322,7 +329,7 @@ export function selectPageReferenceStates( .map((page) => { const state = cache.$internalReferences[page.file]; if (state) { - const selectedFile = selectors.selectFileInfo(session.store.getState(), page.file); + const selectedFile = selectors.selectFileInfo(session.store.getState(), state.filePath); if (selectedFile?.url) state.url = selectedFile.url; if (selectedFile?.title) state.title = selectedFile.title; if (selectedFile?.dataUrl) state.dataUrl = selectedFile.dataUrl; @@ -332,11 +339,13 @@ export function selectPageReferenceStates( }) .filter((state): state is ReferenceState => !!state); if (!opts?.suppressWarnings) warnOnDuplicateIdentifiers(session, pageReferenceStates); - pageReferenceStates.forEach((state) => { - const { mdast } = cache.$getMdast(state.filePath)?.post ?? {}; + pages.forEach((page) => { + const state = cache.$internalReferences[page.file]; + if (!state) return; + const { mdast } = cache.$getMdast(page.file)?.post ?? {}; if (!mdast) return; const vfile = new VFile(); - vfile.path = state.filePath; + vfile.path = page.file; buildIndexTransform( mdast, vfile, @@ -376,24 +385,23 @@ export async function writeFile( ...(await resolvePageExports(session, file)), ]); const downloads = await resolvePageDownloads(session, file, projectPath); - const frontmatterWithExports = { ...frontmatter, exports, downloads }; + const parts = resolveFrontmatterParts(session, frontmatter); + const frontmatterWithExports = { ...frontmatter, exports, downloads, parts }; + const mystData: MystData = { + kind, + sha256, + slug, + location, + dependencies, + frontmatter: frontmatterWithExports, + widgets, + mdast, + references, + }; const jsonFilenameParts = [session.contentPath()]; if (projectSlug) jsonFilenameParts.push(projectSlug); jsonFilenameParts.push(`${pageSlug}.json`); - writeFileToFolder( - join(...jsonFilenameParts), - JSON.stringify({ - kind, - sha256, - slug, - location, - dependencies, - frontmatter: frontmatterWithExports, - widgets, - mdast, - references, - }), - ); + writeFileToFolder(join(...jsonFilenameParts), JSON.stringify(mystData)); session.log.debug(toc(`Wrote "${file}" in %s`)); } @@ -414,7 +422,7 @@ export async function fastProcessFile( maxSizeWebp, }: { file: string; - pageSlug: string; + pageSlug?: string; projectPath: string; projectSlug?: string; } & ProcessFileOptions & @@ -423,35 +431,57 @@ export async function fastProcessFile( const toc = tic(); await loadFile(session, file, projectPath); const { project, pages } = await loadProject(session, projectPath); - await transformMdast(session, { - file, - imageExtensions: imageExtensions ?? WEB_IMAGE_EXTENSIONS, - projectPath, - projectSlug, - pageSlug, - watchMode: true, - extraTransforms, - index: project.index, - execute, - }); - const pageReferenceStates = selectPageReferenceStates(session, pages); - await postProcessMdast(session, { - file, - pageReferenceStates, - extraLinkTransformers, - }); - const { mdast, frontmatter } = castSession(session).$getMdast(file)?.post ?? {}; - if (mdast && frontmatter) { - await finalizeMdast(session, mdast, frontmatter, file, { - imageWriteFolder: imageWriteFolder ?? session.publicPath(), - imageAltOutputFolder: imageAltOutputFolder ?? '/', - imageExtensions: imageExtensions ?? WEB_IMAGE_EXTENSIONS, - optimizeWebp: true, - processThumbnail: true, - maxSizeWebp, - }); + const state = session.store.getState(); + const fileParts = selectors.selectFileParts(state, file); + const projectParts = selectors.selectProjectParts(state, projectPath); + await Promise.all( + [file, ...fileParts].map(async (f) => { + return transformMdast(session, { + file: f, + imageExtensions: imageExtensions ?? WEB_IMAGE_EXTENSIONS, + projectPath, + projectSlug, + pageSlug, + watchMode: true, + extraTransforms, + index: project.index, + execute, + }); + }), + ); + const pageReferenceStates = selectPageReferenceStates(session, [ + ...pages, + ...projectParts.map((part) => { + return { file: part }; + }), + ]); + await Promise.all( + [file, ...fileParts].map(async (f) => { + return postProcessMdast(session, { + file: f, + pageReferenceStates, + extraLinkTransformers, + }); + }), + ); + await Promise.all( + [file, ...fileParts].map(async (f) => { + const { mdast, frontmatter } = castSession(session).$getMdast(f)?.post ?? {}; + if (mdast) { + await finalizeMdast(session, mdast, frontmatter ?? {}, f, { + imageWriteFolder: imageWriteFolder ?? session.publicPath(), + imageAltOutputFolder: imageAltOutputFolder ?? '/', + imageExtensions: imageExtensions ?? WEB_IMAGE_EXTENSIONS, + optimizeWebp: true, + processThumbnail: true, + maxSizeWebp, + }); + } + }), + ); + if (pageSlug) { + await writeFile(session, { file, pageSlug, projectSlug, projectPath }); } - await writeFile(session, { file, pageSlug, projectSlug, projectPath }); session.log.info(toc(`📖 Built ${file} in %s.`)); await writeSiteManifest(session, { defaultTemplate }); } @@ -502,10 +532,16 @@ export async function processProject( // Consolidate all citations onto single project citation renderer combineProjectCitationRenderers(session, siteProject.path); + const projectParts = selectors + .selectProjectParts(session.store.getState(), siteProject.path) + .map((part) => { + return { file: part }; + }); + const pagesToTransform: { file: string; slug?: string }[] = [...pages, ...projectParts]; const usedImageExtensions = imageExtensions ?? WEB_IMAGE_EXTENSIONS; // Transform all pages await Promise.all( - pages.map((page) => + pagesToTransform.map((page) => transformMdast(session, { file: page.file, projectPath: project.path, @@ -519,10 +555,10 @@ export async function processProject( }), ), ); - const pageReferenceStates = selectPageReferenceStates(session, pages); + const pageReferenceStates = selectPageReferenceStates(session, pagesToTransform); // Handle all cross references await Promise.all( - pages.map((page) => + pagesToTransform.map((page) => postProcessMdast(session, { file: page.file, checkLinks: checkLinks || strict, @@ -534,10 +570,10 @@ export async function processProject( // Write all pages if (writeFiles) { await Promise.all( - pages.map(async (page) => { + pagesToTransform.map(async (page) => { const { mdast, frontmatter } = castSession(session).$getMdast(page.file)?.post ?? {}; - if (mdast && frontmatter) { - await finalizeMdast(session, mdast, frontmatter, page.file, { + if (mdast) { + await finalizeMdast(session, mdast, frontmatter ?? {}, page.file, { imageWriteFolder: imageWriteFolder ?? session.publicPath(), imageAltOutputFolder, imageExtensions: usedImageExtensions, @@ -546,6 +582,10 @@ export async function processProject( maxSizeWebp, }); } + }), + ); + await Promise.all( + pages.map(async (page) => { return writeFile(session, { file: page.file, projectSlug: siteProject.slug as string, @@ -622,13 +662,19 @@ export async function processSite(session: ISession, opts?: ProcessSiteOptions): await writeSiteManifest(session, opts); const states: ReferenceState[] = []; const allPages: LocalProjectPage[] = []; + const sessionState = session.store.getState(); await Promise.all( siteConfig.projects.map(async (project) => { if (!project.path) return; const { pages } = await loadProject(session, project.path); allPages.push(...pages); + const projectParts = selectors + .selectProjectParts(sessionState, project.path) + .map((part) => { + return { file: part }; + }); states.push( - ...selectPageReferenceStates(session, pages, { + ...selectPageReferenceStates(session, [...pages, ...projectParts], { suppressWarnings: true, }), ); @@ -636,6 +682,7 @@ export async function processSite(session: ISession, opts?: ProcessSiteOptions): ); await writeObjectsInv(session, states, siteConfig); await writeMystXRefJson(session, states); + // Search does not include parts await writeMystSearchJson(session, allPages); } return true; diff --git a/packages/myst-cli/src/project/fromPath.ts b/packages/myst-cli/src/project/fromPath.ts index 71ab0ea6d..4bab3a13b 100644 --- a/packages/myst-cli/src/project/fromPath.ts +++ b/packages/myst-cli/src/project/fromPath.ts @@ -74,6 +74,7 @@ function projectPagesFromPath( file, level, slug: fileInfo(file, pageSlugs).slug, + implicit: true, } as LocalProjectPage; }); const folders = contents diff --git a/packages/myst-cli/src/project/fromTOC.ts b/packages/myst-cli/src/project/fromTOC.ts index 0667a34c9..b4ba50219 100644 --- a/packages/myst-cli/src/project/fromTOC.ts +++ b/packages/myst-cli/src/project/fromTOC.ts @@ -36,7 +36,13 @@ const DEFAULT_INDEX_WITH_EXT = ['.md', '.ipynb', '.myst.json'] .map((ext) => DEFAULT_INDEX_FILENAMES.map((file) => `${file}${ext}`)) .flat(); -type EntryWithoutPattern = FileEntry | URLEntry | FileParentEntry | URLParentEntry | ParentEntry; +type EntryWithoutPattern = ( + | FileEntry + | URLEntry + | FileParentEntry + | URLParentEntry + | ParentEntry +) & { implicit?: boolean }; export function comparePaths(a: string, b: string): number { const aDirName = dirname(a); @@ -118,6 +124,7 @@ export function patternsToFileEntries( const newEntries = matches.map((item) => { return { file: item, + implicit: true, }; }); if (newEntries.length === 0) { @@ -168,7 +175,7 @@ function pagesFromEntries( }); if (file && fs.existsSync(file) && !isDirectory(file)) { const { slug } = fileInfo(file, pageSlugs); - pages.push({ file, level: entryLevel, slug }); + pages.push({ file, level: entryLevel, slug, implicit: entry.implicit }); } } else if (isURL(entry)) { addWarningForFile( diff --git a/packages/myst-cli/src/project/toc.pattern.spec.ts b/packages/myst-cli/src/project/toc.pattern.spec.ts index 03dc9ba33..b39013f7a 100644 --- a/packages/myst-cli/src/project/toc.pattern.spec.ts +++ b/packages/myst-cli/src/project/toc.pattern.spec.ts @@ -38,11 +38,11 @@ describe('patternsToFileEntries', () => { fs: memfs, }), ).toEqual([ - { file: 'bar/file-2.md' }, - { file: 'bar/file-9.md' }, - { file: 'foo/file-1.md' }, - { file: 'foo/file-3.md' }, - { file: 'foo/file-10.md' }, + { file: 'bar/file-2.md', implicit: true }, + { file: 'bar/file-9.md', implicit: true }, + { file: 'foo/file-1.md', implicit: true }, + { file: 'foo/file-3.md', implicit: true }, + { file: 'foo/file-10.md', implicit: true }, ]); }); it('files containing natural numbers are sorted correctly', () => { @@ -57,10 +57,10 @@ describe('patternsToFileEntries', () => { fs: memfs, }), ).toEqual([ - { file: 'foo/file-01.md' }, - { file: 'foo/file-2.md' }, - { file: 'foo/file-3.md' }, - { file: 'foo/file-10.md' }, + { file: 'foo/file-01.md', implicit: true }, + { file: 'foo/file-2.md', implicit: true }, + { file: 'foo/file-3.md', implicit: true }, + { file: 'foo/file-10.md', implicit: true }, ]); }); it('directories with index files are sorted correctly', () => { @@ -79,14 +79,14 @@ describe('patternsToFileEntries', () => { fs: memfs, }), ).toEqual([ - { file: 'bar/index.md' }, - { file: 'bar/file-1.md' }, - { file: 'bar/file-8.md' }, - { file: 'bar/file-10.md' }, - { file: 'foo/index.ipynb' }, - { file: 'foo/file-2.md' }, - { file: 'foo/file-3.md' }, - { file: 'foo/file-10.md' }, + { file: 'bar/index.md', implicit: true }, + { file: 'bar/file-1.md', implicit: true }, + { file: 'bar/file-8.md', implicit: true }, + { file: 'bar/file-10.md', implicit: true }, + { file: 'foo/index.ipynb', implicit: true }, + { file: 'foo/file-2.md', implicit: true }, + { file: 'foo/file-3.md', implicit: true }, + { file: 'foo/file-10.md', implicit: true }, ]); }); }); diff --git a/packages/myst-cli/src/project/toc.spec.ts b/packages/myst-cli/src/project/toc.spec.ts index c7a8b87dd..50d39c6e8 100644 --- a/packages/myst-cli/src/project/toc.spec.ts +++ b/packages/myst-cli/src/project/toc.spec.ts @@ -48,7 +48,7 @@ describe('site section generation', () => { path: '.', index: 'index', implicitIndex: true, - pages: [{ file: 'README.md', level: 1, slug: 'readme' }], + pages: [{ file: 'README.md', level: 1, slug: 'readme', implicit: true }], }); }); it('index.md only', async () => { @@ -83,11 +83,13 @@ describe('site section generation', () => { file: 'notebook.ipynb', slug: 'notebook', level: 1, + implicit: true, }, { file: 'page.md', slug: 'page', level: 1, + implicit: true, }, ], }); @@ -108,11 +110,13 @@ describe('site section generation', () => { file: 'folder/notebook.ipynb', slug: 'notebook', level: 2, + implicit: true, }, { file: 'folder/page.md', slug: 'page', level: 2, + implicit: true, }, ], }); @@ -145,11 +149,13 @@ describe('site section generation', () => { file: 'folder1/01_MySecond_folder-ok/folder3/01_notebook.ipynb', slug: 'notebook', level: 4, + implicit: true, }, { file: 'folder1/01_MySecond_folder-ok/folder3/02_page.md', slug: 'page', level: 4, + implicit: true, }, ], }); @@ -166,6 +172,7 @@ describe('site section generation', () => { file: 'zfile.md', slug: 'zfile', level: 1, + implicit: true, }, { title: 'Afolder', @@ -175,6 +182,7 @@ describe('site section generation', () => { file: 'afolder/page.md', slug: 'page', level: 2, + implicit: true, }, ], }); @@ -197,6 +205,7 @@ describe('site section generation', () => { file: 'folder1/page1.md', slug: 'page1', level: 1, + implicit: true, }, { title: 'Folder2', @@ -206,6 +215,7 @@ describe('site section generation', () => { file: 'folder1/folder2/page2.md', slug: 'page2', level: 2, + implicit: true, }, { title: 'Folder3', @@ -215,6 +225,7 @@ describe('site section generation', () => { file: 'folder1/folder2/folder3/page3.md', slug: 'page3', level: 3, + implicit: true, }, ], }); @@ -235,6 +246,7 @@ describe('site section generation', () => { file: 'folder/notebook.ipynb', slug: 'notebook', level: 2, + implicit: true, }, ], }); @@ -251,6 +263,7 @@ describe('site section generation', () => { file: 'page.md', slug: 'page', level: 1, + implicit: true, }, ], }); @@ -267,6 +280,7 @@ describe('site section generation', () => { file: 'aaa.ipynb', slug: 'aaa', level: 1, + implicit: true, }, ], }); @@ -297,11 +311,13 @@ describe('site section generation', () => { file: 'index.ipynb', slug: 'index', level: 1, + implicit: true, }, { file: 'page.md', slug: 'page', level: 1, + implicit: true, }, ], }); @@ -326,11 +342,13 @@ describe('site section generation', () => { file: 'folder/index.ipynb', slug: 'index', level: 2, + implicit: true, }, { file: 'folder/main.tex', slug: 'main', level: 2, + implicit: true, }, ], }); @@ -355,11 +373,13 @@ describe('site section generation', () => { file: 'folder/main.tex', slug: 'main', level: 2, + implicit: true, }, { file: 'folder/page.md', slug: 'page', level: 2, + implicit: true, }, ], }); @@ -380,6 +400,7 @@ describe('site section generation', () => { file: 'notebook.ipynb', slug: 'notebook', level: 1, + implicit: true, }, { title: 'Folder', @@ -389,6 +410,7 @@ describe('site section generation', () => { file: 'folder/main.tex', slug: 'main', level: 2, + implicit: true, }, ], }); @@ -415,11 +437,13 @@ describe('site section generation', () => { file: 'folder/notebook.ipynb', slug: 'notebook', level: 2, + implicit: true, }, { file: 'folder/page.md', slug: 'page', level: 2, + implicit: true, }, ], }); @@ -441,16 +465,19 @@ describe('site section generation', () => { file: 'chapter1.md', slug: 'chapter1', level: 1, + implicit: true, }, { file: 'chapter2.ipynb', slug: 'chapter2', level: 1, + implicit: true, }, { file: 'chapter10.ipynb', slug: 'chapter10', level: 1, + implicit: true, }, ], }); @@ -478,11 +505,13 @@ describe('site section generation', () => { file: 'folder/notebook.ipynb', slug: 'notebook', level: 2, + implicit: true, }, { file: 'folder/page.md', slug: 'page', level: 2, + implicit: true, }, ], }); @@ -822,21 +851,21 @@ describe('pagesFromSphinxTOC', () => { 'project/c.md': '', 'project/d.md': '', }); - expect(await projectFromPath(session, '.')).toEqual({ + expect(projectFromPath(session, '.')).toEqual({ path: '.', index: 'readme', file: 'readme.md', implicitIndex: true, pages: [ - { slug: 'x', file: 'x.md', level: 1 }, + { slug: 'x', file: 'x.md', level: 1, implicit: true }, { slug: 'index', file: 'project/index.md', level: 1 }, { slug: 'a', file: 'project/a.md', level: 2 }, { title: 'Sections', level: 2 }, { slug: 'b', file: 'project/b.md', level: 3 }, { slug: 'c', file: 'project/c.md', level: 3 }, { title: 'Section', level: 1 }, - { slug: 'y', file: 'section/y.md', level: 2 }, - { slug: 'z', file: 'section/z.md', level: 2 }, + { slug: 'y', file: 'section/y.md', level: 2, implicit: true }, + { slug: 'z', file: 'section/z.md', level: 2, implicit: true }, ], }); }); @@ -853,22 +882,22 @@ describe('pagesFromSphinxTOC', () => { 'project/c.md': '', 'project/d.md': '', }); - expect(await projectFromPath(session, '.')).toEqual({ + expect(projectFromPath(session, '.')).toEqual({ path: '.', index: 'readme', file: 'readme.md', implicitIndex: true, pages: [ - { slug: 'x', file: 'x.md', level: 1 }, + { slug: 'x', file: 'x.md', level: 1, implicit: true }, { title: 'Project', level: 1 }, - { slug: 'index', file: 'project/index.md', level: 2 }, - { slug: 'a', file: 'project/a.md', level: 2 }, - { slug: 'b', file: 'project/b.md', level: 2 }, - { slug: 'c', file: 'project/c.md', level: 2 }, - { slug: 'd', file: 'project/d.md', level: 2 }, + { slug: 'index', file: 'project/index.md', level: 2, implicit: true }, + { slug: 'a', file: 'project/a.md', level: 2, implicit: true }, + { slug: 'b', file: 'project/b.md', level: 2, implicit: true }, + { slug: 'c', file: 'project/c.md', level: 2, implicit: true }, + { slug: 'd', file: 'project/d.md', level: 2, implicit: true }, { title: 'Section', level: 1 }, - { slug: 'y', file: 'section/y.md', level: 2 }, - { slug: 'z', file: 'section/z.md', level: 2 }, + { slug: 'y', file: 'section/y.md', level: 2, implicit: true }, + { slug: 'z', file: 'section/z.md', level: 2, implicit: true }, ], }); }); diff --git a/packages/myst-cli/src/project/types.ts b/packages/myst-cli/src/project/types.ts index 36155a7aa..20847d978 100644 --- a/packages/myst-cli/src/project/types.ts +++ b/packages/myst-cli/src/project/types.ts @@ -20,6 +20,8 @@ export type LocalProjectPage = { file: string; slug: string; level: PageLevels; + /** Flag to mark if the page is implied from a TOC pattern or folder structure */ + implicit?: boolean; }; export type LocalProject = { diff --git a/packages/myst-cli/src/session/session.spec.ts b/packages/myst-cli/src/session/session.spec.ts index a4a5b94ae..4de8c97ab 100644 --- a/packages/myst-cli/src/session/session.spec.ts +++ b/packages/myst-cli/src/session/session.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { addWarningForFile } from 'myst-cli'; import { Session } from '../session'; import { RuleId } from 'myst-common'; +import { addWarningForFile } from '../utils/addWarningForFile'; describe('session warnings', () => { it('getAllWarnings returns for single session', async () => { diff --git a/packages/myst-cli/src/store/reducers.ts b/packages/myst-cli/src/store/reducers.ts index ba76a9f07..a13e30d61 100644 --- a/packages/myst-cli/src/store/reducers.ts +++ b/packages/myst-cli/src/store/reducers.ts @@ -33,6 +33,8 @@ export const config = createSlice({ initialState: { rawConfigs: {}, projects: {}, + projectParts: {}, + fileParts: {}, sites: {}, filenames: {}, } as { @@ -40,6 +42,8 @@ export const config = createSlice({ currentSitePath: string | undefined; rawConfigs: Record; validated: ValidatedRawConfig }>; projects: Record>; + projectParts: Record; + fileParts: Record; sites: Record>; filenames: Record; configExtensions?: string[]; @@ -76,6 +80,20 @@ export const config = createSlice({ state.configExtensions ??= []; state.configExtensions.push(action.payload.file); }, + receiveProjectPart(state, action: PayloadAction<{ partFile: string; path: string }>) { + const { path, partFile } = action.payload; + const partFiles = state.projectParts[resolve(path)] ?? []; + if (!partFiles.includes(partFile)) { + state.projectParts[resolve(path)] = [...partFiles, partFile]; + } + }, + receiveFilePart(state, action: PayloadAction<{ partFile: string; file: string }>) { + const { file, partFile } = action.payload; + const partFiles = state.fileParts[resolve(file)] ?? []; + if (!partFiles.includes(partFile)) { + state.fileParts[resolve(file)] = [...partFiles, partFile]; + } + }, }, }); @@ -115,6 +133,7 @@ export const watch = createSlice({ }, addLocalDependency(state, action: PayloadAction<{ path: string; dependency: string }>) { const { path, dependency } = action.payload; + if (!state.files[resolve(path)]) state.files[resolve(path)] = {}; const existingDeps = [...(state.files[resolve(path)].localDependencies ?? [])]; if (!existingDeps.includes(dependency)) { state.files[resolve(path)].localDependencies = [...existingDeps, dependency]; @@ -154,6 +173,7 @@ export const watch = createSlice({ dataUrl, } = action.payload; const resolvedPath = resolve(path); + if (!state.files[resolvedPath]) state.files[resolvedPath] = {}; if (title) state.files[resolvedPath].title = title; if (short_title) state.files[resolvedPath].short_title = short_title; if (description) state.files[resolvedPath].description = description; diff --git a/packages/myst-cli/src/store/selectors.ts b/packages/myst-cli/src/store/selectors.ts index 34f2ba053..7b68be8fd 100644 --- a/packages/myst-cli/src/store/selectors.ts +++ b/packages/myst-cli/src/store/selectors.ts @@ -50,6 +50,23 @@ export function selectCurrentProjectConfig(state: RootState): ProjectConfig | un return mutableCopy(state.local.config.projects[resolve(state.local.config.currentProjectPath)]); } +export function selectProjectParts(state: RootState, path: string): string[] { + return [...(state.local.config.projectParts[resolve(path)] ?? [])]; +} + +export function selectFileParts(state: RootState, file: string): string[] { + return [...(state.local.config.fileParts[resolve(file)] ?? [])]; +} + +export function selectFileFromPart(state: RootState, partFile: string): string | undefined { + let file: string | undefined; + Object.entries(state.local.config.fileParts).forEach(([key, value]) => { + if (file) return; + if (value.includes(partFile)) file = key; + }); + return file; +} + export function selectCurrentProjectPath(state: RootState): string | undefined { return state.local.config.currentProjectPath; } diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index d274948b9..470c1a956 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -1,14 +1,14 @@ import type { VFile } from 'vfile'; import { selectAll } from 'unist-util-select'; -import type { GenericNode, GenericParent } from 'myst-common'; +import type { FrontmatterParts, GenericNode, GenericParent, References } from 'myst-common'; import { RuleId, fileWarn, plural, selectMdastNodes } from 'myst-common'; import { computeHash, tic } from 'myst-cli-utils'; import { addChildrenFromTargetNode } from 'myst-transforms'; import type { PageFrontmatter } from 'myst-frontmatter'; -import type { CrossReference, Link } from 'myst-spec-ext'; +import type { CrossReference, Dependency, Link, SourceFileKind } from 'myst-spec-ext'; import type { ISession } from '../session/types.js'; import { loadFromCache, writeToCache } from '../session/cache.js'; -import type { RendererData } from './types.js'; +import type { SiteAction, SiteExport } from 'myst-config'; export const XREF_MAX_AGE = 1; // in days @@ -16,6 +16,22 @@ function mystDataFilename(dataUrl: string) { return `myst-${computeHash(dataUrl)}.json`; } +export type MystData = { + kind?: SourceFileKind; + sha256?: string; + slug?: string; + location?: string; + dependencies?: Dependency[]; + frontmatter?: Omit & { + downloads?: SiteAction[]; + exports?: [{ format: string; filename: string; url: string }, ...SiteExport[]]; + parts?: FrontmatterParts; + }; + widgets?: Record; + mdast?: GenericParent; + references?: References; +}; + async function fetchMystData( session: ISession, dataUrl: string | undefined, @@ -27,12 +43,12 @@ async function fetchMystData( const filename = mystDataFilename(dataUrl); const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE }); if (cacheData) { - return JSON.parse(cacheData) as RendererData; + return JSON.parse(cacheData) as MystData; } try { const resp = await session.fetch(dataUrl); if (resp.ok) { - const data = (await resp.json()) as RendererData; + const data = (await resp.json()) as MystData; writeToCache(session, filename, JSON.stringify(data)); return data; } @@ -67,7 +83,7 @@ export async function fetchMystXRefData(session: ISession, node: CrossReference, } export function nodesFromMystXRefData( - data: RendererData, + data: MystData, identifier: string, vfile: VFile, opts?: { @@ -75,7 +91,11 @@ export function nodesFromMystXRefData( urlSource?: string; }, ) { - const targetNodes = selectMdastNodes(data.mdast, identifier, opts?.maxNodes).nodes; + let targetNodes: GenericNode[] | undefined; + [data, ...Object.values(data.frontmatter?.parts ?? {})].forEach(({ mdast }) => { + if (!mdast || targetNodes?.length) return; + targetNodes = selectMdastNodes(mdast, identifier, opts?.maxNodes).nodes; + }); if (!targetNodes?.length) { fileWarn(vfile, `Unable to resolve content from external MyST reference: ${opts?.urlSource}`, { ruleId: RuleId.mystLinkValid, diff --git a/packages/myst-cli/src/transforms/embed.ts b/packages/myst-cli/src/transforms/embed.ts index cc7c60f52..a2653cc8e 100644 --- a/packages/myst-cli/src/transforms/embed.ts +++ b/packages/myst-cli/src/transforms/embed.ts @@ -1,3 +1,5 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { filter } from 'unist-util-filter'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; @@ -21,15 +23,16 @@ import { selectFile } from '../process/file.js'; import type { ISession } from '../session/types.js'; import { watch } from '../store/reducers.js'; import { castSession } from '../session/cache.js'; +import type { MystData } from './crossReferences.js'; import { fetchMystLinkData, fetchMystXRefData, nodesFromMystXRefData } from './crossReferences.js'; import { fileFromRelativePath } from './links.js'; -import type { RendererData } from './types.js'; function mutateEmbedNode( node: Embed, targetNode?: GenericNode | null, - opts?: { url?: string; dataUrl?: string }, + opts?: { url?: string; dataUrl?: string; targetFile?: string; sourceFile?: string }, ) { + const { url, dataUrl, targetFile, sourceFile } = opts ?? {}; if (targetNode && node['remove-output']) { targetNode = filter(targetNode, (n: GenericNode) => { return n.type !== 'output' && n.data?.type !== 'output'; @@ -49,11 +52,28 @@ function mutateEmbedNode( }); (selectAll('crossReference', targetNode) as CrossReference[]).forEach((targetXRef) => { if (!targetXRef.remote) { - targetXRef.url = opts?.url; - targetXRef.dataUrl = opts?.dataUrl; + targetXRef.url = url; + targetXRef.dataUrl = dataUrl; targetXRef.remote = true; } }); + if (targetFile && sourceFile) { + // TODO: Other node types that point to a file? + selectAll('image', targetNode).forEach((imageNode: GenericNode) => { + if (imageNode.url) { + const absoluteUrl = path.resolve(path.dirname(sourceFile), imageNode.url); + if (fs.existsSync(absoluteUrl)) { + imageNode.url = path.relative(path.dirname(targetFile), absoluteUrl); + } + } + if (imageNode.urlSource) { + const absoluteUrlSource = path.resolve(path.dirname(sourceFile), imageNode.urlSource); + if (fs.existsSync(absoluteUrlSource)) { + imageNode.urlSource = path.relative(path.dirname(targetFile), absoluteUrlSource); + } + } + }); + } if (!targetNode) { node.children = []; } else if (targetNode.type === 'block') { @@ -111,7 +131,7 @@ export async function embedTransform( const transformed = mystTransformer.transform(referenceLink, vfile); const referenceXRef = referenceLink as any as CrossReference; if (transformed) { - let data: RendererData | undefined; + let data: MystData | undefined; let targetNodes: GenericNode[] | undefined; if (referenceXRef.identifier) { data = await fetchMystXRefData(session, referenceXRef, vfile); @@ -122,7 +142,7 @@ export async function embedTransform( }); } else { data = await fetchMystLinkData(session, referenceLink, vfile); - if (!data) return; + if (!data?.mdast) return; targetNodes = data.mdast.children; } if (!targetNodes?.length) return; @@ -196,7 +216,7 @@ export async function embedTransform( return; } const { url, dataUrl, filePath } = stateProvider; - mutateEmbedNode(node, target, { url, dataUrl }); + mutateEmbedNode(node, target, { url, dataUrl, targetFile: file, sourceFile: filePath }); if (!url) return; const source: Dependency = { url, label }; if (filePath) { diff --git a/packages/myst-cli/src/transforms/parts.ts b/packages/myst-cli/src/transforms/parts.ts index 1625fbc5e..146bee9f5 100644 --- a/packages/myst-cli/src/transforms/parts.ts +++ b/packages/myst-cli/src/transforms/parts.ts @@ -4,6 +4,11 @@ import { parseMyst } from '../process/myst.js'; import type { GenericParent } from 'myst-common'; import type { PageFrontmatter } from 'myst-frontmatter'; +/** + * Parse frontmatter parts and prepend them as blocks to mdast children + * + * @deprecated frontmatter parts are now processed separately by MyST + */ export function frontmatterPartsTransform( session: ISession, file: string, diff --git a/packages/myst-cli/src/utils/index.ts b/packages/myst-cli/src/utils/index.ts index ce46030e8..0894edd6b 100644 --- a/packages/myst-cli/src/utils/index.ts +++ b/packages/myst-cli/src/utils/index.ts @@ -10,6 +10,7 @@ export * from './logging.js'; export * from './nextLevel.js'; export * from './removeExtension.js'; export * from './resolveExtension.js'; +export * from './resolveFrontmatterParts.js'; export * from './shouldIgnoreFile.js'; export * from './toc.js'; export * from './uniqueArray.js'; diff --git a/packages/myst-cli/src/utils/resolveFrontmatterParts.ts b/packages/myst-cli/src/utils/resolveFrontmatterParts.ts new file mode 100644 index 000000000..535754177 --- /dev/null +++ b/packages/myst-cli/src/utils/resolveFrontmatterParts.ts @@ -0,0 +1,22 @@ +import type { FrontmatterParts } from 'myst-common'; +import type { PageFrontmatter } from 'myst-frontmatter'; +import { castSession } from '../session/cache.js'; +import type { ISession } from '../session/types.js'; + +/** + * Load frontmatter parts from session and return part:node lookup + */ +export function resolveFrontmatterParts( + session: ISession, + pageFrontmatter?: PageFrontmatter, +): FrontmatterParts | undefined { + const { parts } = pageFrontmatter ?? {}; + if (!parts || Object.keys(parts).length === 0) return undefined; + const partsMdast: FrontmatterParts = {}; + Object.entries(parts).forEach(([part, content]) => { + if (content.length !== 1) return; + const { mdast, frontmatter } = castSession(session).$getMdast(content[0])?.post ?? {}; + if (mdast) partsMdast[part] = { mdast, frontmatter }; + }); + return partsMdast; +} diff --git a/packages/myst-common/src/extractParts.spec.ts b/packages/myst-common/src/extractParts.spec.ts index 6bea14e63..0afe1ea74 100644 --- a/packages/myst-common/src/extractParts.spec.ts +++ b/packages/myst-common/src/extractParts.spec.ts @@ -244,6 +244,81 @@ describe('extractPart', () => { ], }); }); + it('frontmatter part prioritized, tagged block removed, implicit part unchanged', async () => { + const tree: GenericParent = { + type: 'root', + children: [ + { + type: 'block' as any, + data: { part: 'other_tag' }, + children: [{ type: 'text', value: 'untagged content' }], + }, + { + type: 'block' as any, + data: { part: 'test_part' }, + children: [{ type: 'text', value: 'block part' }], + }, + { + type: 'heading', + children: [{ type: 'text', value: 'test_part' }], + }, + { + type: 'paragraph', + children: [{ type: 'text', value: 'implicit part' }], + }, + ], + }; + expect( + extractPart(tree, 'test_part', { + frontmatterParts: { + test_part: { + mdast: { + type: 'root', + children: [ + { + type: 'block', + children: [ + { type: 'paragraph', children: [{ type: 'text', value: 'frontmatter part' }] }, + ], + }, + ], + }, + }, + }, + }), + ).toEqual({ + type: 'root', + children: [ + { + type: 'block', + data: { + part: 'test_part', + }, + children: [ + { type: 'paragraph', children: [{ type: 'text', value: 'frontmatter part' }] }, + ], + }, + ], + }); + expect(tree).toEqual({ + type: 'root', + children: [ + { + type: 'block' as any, + data: { part: 'other_tag' }, + children: [{ type: 'text', value: 'untagged content' }], + }, + { + type: 'heading', + children: [{ type: 'text', value: 'test_part' }], + }, + { + type: 'paragraph', + children: [{ type: 'text', value: 'implicit part' }], + }, + ], + }); + }); }); describe('extractImplicitPart', () => { diff --git a/packages/myst-common/src/extractParts.ts b/packages/myst-common/src/extractParts.ts index 457ad9a85..1bf468c57 100644 --- a/packages/myst-common/src/extractParts.ts +++ b/packages/myst-common/src/extractParts.ts @@ -1,5 +1,5 @@ import type { Block } from 'myst-spec-ext'; -import type { GenericNode, GenericParent } from './types.js'; +import type { FrontmatterParts, GenericNode, GenericParent } from './types.js'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; import { copyNode, toText } from './utils.js'; @@ -45,6 +45,30 @@ export function selectBlockParts(tree: GenericParent, part?: string | string[]): return blockParts as Block[]; } +/** + * Selects the frontmatterParts entries by `part` + * + * If `part` is a string array, any matching part from the frontmatter will be + * returned. + * + * Returns array of blocks. + */ +export function selectFrontmatterParts( + frontmatterParts?: FrontmatterParts, + part?: string | string[], +): Block[] { + if (!frontmatterParts) return []; + const parts = coercePart(part); + if (parts.length === 0) return []; + const blockParts: Block[] = []; + parts.forEach((p) => { + Object.entries(frontmatterParts).forEach(([key, value]) => { + if (p === key.toLowerCase()) blockParts.push(...(value.mdast.children as Block[])); + }); + }); + return blockParts; +} + function createPartBlock( children: GenericNode[], part: string, @@ -131,7 +155,9 @@ export function extractImplicitPart( } /** - * Returns a copy of the block parts and removes them from the tree. + * Returns a copy of block parts, if defined in the tree, and removes them from the tree. + * + * This does not look at parts defined in frontmatter. */ export function extractPart( tree: GenericParent, @@ -143,38 +169,43 @@ export function extractPart( keepVisibility?: boolean; /** Provide an option so implicit section-to-part behavior can be disabled */ requireExplicitPart?: boolean; + /** Dictionary of part trees, processed from frontmatter */ + frontmatterParts?: FrontmatterParts; }, ): GenericParent | undefined { const partStrings = coercePart(part); if (partStrings.length === 0) return; + const frontmatterParts = selectFrontmatterParts(opts?.frontmatterParts, part); const blockParts = selectBlockParts(tree, part); - if (blockParts.length === 0) { + if (frontmatterParts.length === 0 && blockParts.length === 0) { if (opts?.requireExplicitPart) return; return extractImplicitPart(tree, partStrings); } - const children = copyNode(blockParts).map((block) => { - // Ensure the block always has the `part` defined, as it might be in the tags - block.data ??= {}; - block.data.part = partStrings[0]; - if ( - block.data.tags && - Array.isArray(block.data.tags) && - block.data.tags.reduce((a, t) => a || partStrings.includes(t.toLowerCase()), false) - ) { - block.data.tags = block.data.tags.filter( - (tag) => !partStrings.includes(tag.toLowerCase()), - ) as string[]; - if ((block.data.tags as string[]).length === 0) { - delete block.data.tags; + const children = copyNode(frontmatterParts.length > 0 ? frontmatterParts : blockParts).map( + (block) => { + // Ensure the block always has the `part` defined, as it might be in the tags + block.data ??= {}; + block.data.part = partStrings[0]; + if ( + block.data.tags && + Array.isArray(block.data.tags) && + block.data.tags.reduce((a, t) => a || partStrings.includes(t.toLowerCase()), false) + ) { + block.data.tags = block.data.tags.filter( + (tag) => !partStrings.includes(tag.toLowerCase()), + ) as string[]; + if ((block.data.tags as string[]).length === 0) { + delete block.data.tags; + } } - } - if (opts?.removePartData) delete block.data.part; - // The default is to remove the visibility on the parts - if (!opts?.keepVisibility) delete block.visibility; - return block; - }); + if (opts?.removePartData) delete block.data.part; + // The default is to remove the visibility on the parts + if (!opts?.keepVisibility) delete block.visibility; + return block; + }, + ); const partsTree = { type: 'root', children } as GenericParent; - // Remove the block parts from the main document + // Remove the block parts from the main document, even if frontmatter parts are returned blockParts.forEach((block) => { (block as any).type = '__delete__'; }); diff --git a/packages/myst-common/src/index.ts b/packages/myst-common/src/index.ts index d1401b8bb..0eaf0283f 100644 --- a/packages/myst-common/src/index.ts +++ b/packages/myst-common/src/index.ts @@ -53,4 +53,6 @@ export type { PluginOptions, PluginUtils, TransformSpec, + FrontmatterPart, + FrontmatterParts, } from './types.js'; diff --git a/packages/myst-common/src/types.ts b/packages/myst-common/src/types.ts index fee408c22..0231d1927 100644 --- a/packages/myst-common/src/types.ts +++ b/packages/myst-common/src/types.ts @@ -3,6 +3,7 @@ import type { Directive, Node, Role } from 'myst-spec'; import type { VFile } from 'vfile'; import type * as nbformat from '@jupyterlab/nbformat'; import type { PartialJSONObject } from '@lumino/coreutils'; +import type { PageFrontmatter } from 'myst-frontmatter'; export type GenericNode = Record> = { type: string; @@ -187,3 +188,7 @@ export interface IExpressionError { } export type IExpressionResult = IExpressionError | IExpressionOutput; + +export type FrontmatterPart = { mdast: GenericParent; frontmatter?: PageFrontmatter }; + +export type FrontmatterParts = Record; diff --git a/packages/myst-config/package.json b/packages/myst-config/package.json index 018ab1828..875d50eb6 100644 --- a/packages/myst-config/package.json +++ b/packages/myst-config/package.json @@ -31,6 +31,7 @@ "url": "https://github.com/jupyter-book/mystmd/issues" }, "dependencies": { + "myst-common": "^1.7.1", "myst-frontmatter": "^1.7.1", "simple-validators": "^1.1.0" }, diff --git a/packages/myst-config/src/site/types.ts b/packages/myst-config/src/site/types.ts index 0d9b2f987..e57cb5a17 100644 --- a/packages/myst-config/src/site/types.ts +++ b/packages/myst-config/src/site/types.ts @@ -1,3 +1,4 @@ +import type { FrontmatterParts } from 'myst-common'; import type { ExportFormats, ProjectFrontmatter, SiteFrontmatter } from 'myst-frontmatter'; export interface SiteProject { @@ -64,9 +65,10 @@ type ManifestProject = { tags?: string[]; downloads?: SiteAction[]; exports?: SiteExport[]; -} & Omit; + parts?: FrontmatterParts; +} & Omit; -export type SiteManifest = SiteFrontmatter & { +export type SiteManifest = Omit & { myst: string; id?: string; projects?: ManifestProject[]; @@ -75,4 +77,5 @@ export type SiteManifest = SiteFrontmatter & { domains?: string[]; favicon?: string; template?: string; + parts?: FrontmatterParts; }; diff --git a/packages/myst-frontmatter/src/project/types.ts b/packages/myst-frontmatter/src/project/types.ts index e837661c4..015cf283d 100644 --- a/packages/myst-frontmatter/src/project/types.ts +++ b/packages/myst-frontmatter/src/project/types.ts @@ -10,16 +10,6 @@ import type { SiteFrontmatter } from '../site/types.js'; import { SITE_FRONTMATTER_KEYS } from '../site/types.js'; import type { ExpandedThebeFrontmatter } from '../thebe/types.js'; -export const PAGE_KNOWN_PARTS = [ - 'abstract', - 'summary', - 'keypoints', - 'dedication', - 'epigraph', - 'data_availability', - 'acknowledgments', -]; - export const PROJECT_AND_PAGE_FRONTMATTER_KEYS = [ 'date', 'doi', @@ -43,8 +33,6 @@ export const PROJECT_AND_PAGE_FRONTMATTER_KEYS = [ 'exports', 'downloads', 'settings', // We maybe want to move this into site frontmatter in the future - 'parts', - ...PAGE_KNOWN_PARTS, // Do not add any project specific keys here! ...SITE_FRONTMATTER_KEYS, ]; @@ -86,7 +74,6 @@ export type ProjectAndPageFrontmatter = SiteFrontmatter & { exports?: Export[]; downloads?: Download[]; settings?: ProjectSettings; - parts?: Record; }; export type ProjectFrontmatter = ProjectAndPageFrontmatter & { diff --git a/packages/myst-frontmatter/src/project/validators.ts b/packages/myst-frontmatter/src/project/validators.ts index 79284eab6..829b5f7c2 100644 --- a/packages/myst-frontmatter/src/project/validators.ts +++ b/packages/myst-frontmatter/src/project/validators.ts @@ -10,7 +10,6 @@ import { validateObjectKeys, validateString, validateUrl, - validationError, } from 'simple-validators'; import { validateTOC } from 'myst-toc'; import { validatePublicationMeta } from '../biblio/validators.js'; @@ -22,7 +21,7 @@ import { validateExternalReferences } from '../references/validators.js'; import { validateSiteFrontmatterKeys } from '../site/validators.js'; import { validateThebe } from '../thebe/validators.js'; import { validateDoi, validateStringOrNumber } from '../utils/validators.js'; -import { PAGE_KNOWN_PARTS, PROJECT_FRONTMATTER_KEYS } from './types.js'; +import { PROJECT_FRONTMATTER_KEYS } from './types.js'; import type { ProjectAndPageFrontmatter, ProjectFrontmatter } from './types.js'; import { validateProjectAndPageSettings } from '../settings/validators.js'; import { FRONTMATTER_ALIASES } from '../site/types.js'; @@ -170,40 +169,6 @@ export function validateProjectAndPageFrontmatterKeys( ); if (settings) output.settings = settings; } - const partsOptions = incrementOptions('parts', opts); - let parts: Record | undefined; - if (defined(value.parts)) { - parts = validateObjectKeys( - value.parts, - { optional: PAGE_KNOWN_PARTS, alias: FRONTMATTER_ALIASES }, - { keepExtraKeys: true, suppressWarnings: true, ...partsOptions }, - ); - } - PAGE_KNOWN_PARTS.forEach((partKey) => { - if (defined(value[partKey])) { - parts ??= {}; - if (parts[partKey]) { - validationError(`duplicate value for part ${partKey}`, partsOptions); - } else { - parts[partKey] = value[partKey]; - } - } - }); - if (parts) { - const partsEntries = Object.entries(parts) - .map(([k, v]) => { - return [ - k, - validateList(v, { coerce: true, ...incrementOptions(k, partsOptions) }, (item, index) => { - return validateString(item, incrementOptions(`${k}.${index}`, partsOptions)); - }), - ]; - }) - .filter((entry): entry is [string, string[]] => !!entry[1]?.length); - if (partsEntries.length > 0) { - output.parts = Object.fromEntries(partsEntries); - } - } return output; } diff --git a/packages/myst-frontmatter/src/site/types.ts b/packages/myst-frontmatter/src/site/types.ts index c0f211d37..a38b1bb79 100644 --- a/packages/myst-frontmatter/src/site/types.ts +++ b/packages/myst-frontmatter/src/site/types.ts @@ -3,6 +3,16 @@ import type { Contributor } from '../contributors/types.js'; import type { Funding } from '../funding/types.js'; import type { Venue } from '../venues/types.js'; +export const PAGE_KNOWN_PARTS = [ + 'abstract', + 'summary', + 'keypoints', + 'dedication', + 'epigraph', + 'data_availability', + 'acknowledgments', +]; + export const SITE_FRONTMATTER_KEYS = [ 'title', 'subtitle', @@ -24,6 +34,8 @@ export const SITE_FRONTMATTER_KEYS = [ 'funding', 'copyright', 'options', + 'parts', + ...PAGE_KNOWN_PARTS, ]; export const FRONTMATTER_ALIASES = { @@ -82,4 +94,5 @@ export type SiteFrontmatter = { copyright?: string; contributors?: Contributor[]; options?: Record; + parts?: Record; }; diff --git a/packages/myst-frontmatter/src/site/validators.ts b/packages/myst-frontmatter/src/site/validators.ts index 3fb8fa4ac..ebebe888c 100644 --- a/packages/myst-frontmatter/src/site/validators.ts +++ b/packages/myst-frontmatter/src/site/validators.ts @@ -4,6 +4,7 @@ import { incrementOptions, validateList, validateObject, + validateObjectKeys, validateString, validationError, } from 'simple-validators'; @@ -14,7 +15,7 @@ import type { ReferenceStash } from '../utils/referenceStash.js'; import { validateAndStashObject } from '../utils/referenceStash.js'; import { validateGithubUrl } from '../utils/validators.js'; import { validateVenue } from '../venues/validators.js'; -import type { SiteFrontmatter } from './types.js'; +import { FRONTMATTER_ALIASES, PAGE_KNOWN_PARTS, type SiteFrontmatter } from './types.js'; import { RESERVED_EXPORT_KEYS } from '../exports/validators.js'; export function validateSiteFrontmatterKeys(value: Record, opts: ValidationOptions) { @@ -171,6 +172,40 @@ export function validateSiteFrontmatterKeys(value: Record, opts: Va }); } } + const partsOptions = incrementOptions('parts', opts); + let parts: Record | undefined; + if (defined(value.parts)) { + parts = validateObjectKeys( + value.parts, + { optional: PAGE_KNOWN_PARTS, alias: FRONTMATTER_ALIASES }, + { keepExtraKeys: true, suppressWarnings: true, ...partsOptions }, + ); + } + PAGE_KNOWN_PARTS.forEach((partKey) => { + if (defined(value[partKey])) { + parts ??= {}; + if (parts[partKey]) { + validationError(`duplicate value for part ${partKey}`, partsOptions); + } else { + parts[partKey] = value[partKey]; + } + } + }); + if (parts) { + const partsEntries = Object.entries(parts) + .map(([k, v]) => { + return [ + k, + validateList(v, { coerce: true, ...incrementOptions(k, partsOptions) }, (item, index) => { + return validateString(item, incrementOptions(`${k}.${index}`, partsOptions)); + }), + ]; + }) + .filter((entry): entry is [string, string[]] => !!entry[1]?.length); + if (partsEntries.length > 0) { + output.parts = Object.fromEntries(partsEntries); + } + } // Author/Contributor/Affiliation resolution should happen last const stashContribAuthors = stash.contributors?.filter((contrib) => diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index 800dd3512..d50007c59 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -184,6 +184,7 @@ export type InlineExpression = { export enum SourceFileKind { Article = 'Article', Notebook = 'Notebook', + Part = 'Part', } export type Dependency = { diff --git a/packages/myst-to-jats/src/frontmatter.ts b/packages/myst-to-jats/src/frontmatter.ts index 134bda771..ad7cb7379 100644 --- a/packages/myst-to-jats/src/frontmatter.ts +++ b/packages/myst-to-jats/src/frontmatter.ts @@ -1,8 +1,8 @@ -import type { Contributor, ProjectFrontmatter, Affiliation } from 'myst-frontmatter'; +import type { Contributor, Affiliation } from 'myst-frontmatter'; import * as credit from 'credit-roles'; import { doi } from 'doi-utils'; import { orcid } from 'orcid'; -import type { Element, IJatsSerializer } from './types.js'; +import type { Element, FrontmatterWithParts, IJatsSerializer } from './types.js'; export function getJournalIds(): Element[] { // [{ type: 'element', name: 'journal-id', attributes: {'journal-id-type': ...}, text: ...}] @@ -59,7 +59,7 @@ export function getJournalMeta(): Element | null { * * See: https://jats.nlm.nih.gov/archiving/tag-library/1.3/element/article-title.html */ -export function getArticleTitle(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleTitle(frontmatter: FrontmatterWithParts): Element[] { const title = frontmatter?.title; const subtitle = frontmatter?.subtitle; const short_title = frontmatter?.short_title; @@ -156,7 +156,7 @@ function nameElementFromContributor(contrib: Contributor): Element | undefined { * * Authors are tagged as contrib-type="author" */ -export function getArticleAuthors(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleAuthors(frontmatter: FrontmatterWithParts): Element[] { const generateContrib = (author: Contributor, type?: string): Element => { const attributes: Record = {}; const elements: Element[] = []; @@ -302,7 +302,7 @@ function instWrapElementsFromAffiliation(affiliation: Affiliation, includeDept = return elements; } -export function getArticleAffiliations(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleAffiliations(frontmatter: FrontmatterWithParts): Element[] { if (!frontmatter.affiliations?.length) return []; // Only add affiliations from authors, not contributors const affIds = [ @@ -393,7 +393,7 @@ export function getArticleAffiliations(frontmatter: ProjectFrontmatter): Element return affs ? affs : []; } -export function getArticlePermissions(frontmatter: ProjectFrontmatter): Element[] { +export function getArticlePermissions(frontmatter: FrontmatterWithParts): Element[] { // copyright-statement: '© 2023, Authors et al' // copyright-year: '2023' // copyright-holder: 'Authors et al' @@ -460,14 +460,14 @@ export function getArticlePermissions(frontmatter: ProjectFrontmatter): Element[ : []; } -export function getKwdGroup(frontmatter: ProjectFrontmatter): Element[] { +export function getKwdGroup(frontmatter: FrontmatterWithParts): Element[] { const kwds = frontmatter.keywords?.map((keyword): Element => { return { type: 'element', name: 'kwd', elements: [{ type: 'text', text: keyword }] }; }); return kwds?.length ? [{ type: 'element', name: 'kwd-group', elements: kwds }] : []; } -export function getFundingGroup(frontmatter: ProjectFrontmatter): Element[] { +export function getFundingGroup(frontmatter: FrontmatterWithParts): Element[] { const fundingGroups = frontmatter.funding?.map((fund): Element => { const elements: Element[] = []; if (fund.awards?.length) { @@ -588,21 +588,21 @@ export function getFundingGroup(frontmatter: ProjectFrontmatter): Element[] { return fundingGroups ? fundingGroups : []; } -export function getArticleVolume(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleVolume(frontmatter: FrontmatterWithParts): Element[] { const text = frontmatter.volume?.number; return text ? [{ type: 'element', name: 'volume', elements: [{ type: 'text', text: `${text}` }] }] : []; } -export function getArticleIssue(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleIssue(frontmatter: FrontmatterWithParts): Element[] { const text = frontmatter.issue?.number; return text ? [{ type: 'element', name: 'issue', elements: [{ type: 'text', text: `${text}` }] }] : []; } -export function getArticlePages(frontmatter: ProjectFrontmatter): Element[] { +export function getArticlePages(frontmatter: FrontmatterWithParts): Element[] { // fpage/lpage, page-range, or elocation-id const { first_page, last_page } = frontmatter ?? {}; const pages: Element[] = []; @@ -623,7 +623,7 @@ export function getArticlePages(frontmatter: ProjectFrontmatter): Element[] { return pages; } -export function getArticleIds(frontmatter: ProjectFrontmatter): Element[] { +export function getArticleIds(frontmatter: FrontmatterWithParts): Element[] { const ids: Element[] = []; if (doi.validate(frontmatter.doi)) { ids.push({ @@ -636,7 +636,10 @@ export function getArticleIds(frontmatter: ProjectFrontmatter): Element[] { return ids; } -export function getArticleMeta(frontmatter?: ProjectFrontmatter, state?: IJatsSerializer): Element { +export function getArticleMeta( + frontmatter?: FrontmatterWithParts, + state?: IJatsSerializer, +): Element { const elements = []; if (frontmatter) { elements.push( @@ -690,7 +693,7 @@ export function getArticleMeta(frontmatter?: ProjectFrontmatter, state?: IJatsSe * * This element must be defined in a JATS article and must include */ -export function getFront(frontmatter?: ProjectFrontmatter, state?: IJatsSerializer): Element[] { +export function getFront(frontmatter?: FrontmatterWithParts, state?: IJatsSerializer): Element[] { const elements: Element[] = []; const journalMeta = getJournalMeta(); if (journalMeta) elements.push(journalMeta); diff --git a/packages/myst-to-jats/src/index.ts b/packages/myst-to-jats/src/index.ts index 3796760f5..26c4e55ff 100644 --- a/packages/myst-to-jats/src/index.ts +++ b/packages/myst-to-jats/src/index.ts @@ -70,6 +70,7 @@ import type { ArticleContent, DocumentOptions, JatsPart, + FrontmatterWithParts, } from './types.js'; import { ACKNOWLEDGMENT_PARTS, ABSTRACT_PARTS } from './types.js'; import { @@ -713,11 +714,13 @@ function createText(text: string): Element { } function renderPart(vfile: VFile, mdast: GenericParent, part: string | string[], opts?: Options) { + const { frontmatterParts, ...otherOpts } = opts ?? {}; const partMdast = extractPart(mdast, part, { removePartData: true, + frontmatterParts, }); if (!partMdast) return undefined; - const serializer = new JatsSerializer(vfile, partMdast as Root, opts); + const serializer = new JatsSerializer(vfile, partMdast as Root, otherOpts); return serializer.render(true).elements(); } @@ -930,6 +933,7 @@ export class JatsDocument { ...this.options, isNotebookArticleRep, extractAbstract: true, + frontmatterParts: this.content.frontmatter?.parts, }); const inventory: IdInventory = {}; referenceTargetTransform(articleState.mdast as any, inventory, this.content.citations); @@ -945,7 +949,7 @@ export class JatsDocument { } affiliationIdTransform( [this.content.frontmatter, ...subArticles.map((a) => a.frontmatter)].filter( - (fm): fm is PageFrontmatter => !!fm, + (fm): fm is Omit => !!fm, ), 'aff', ); @@ -981,7 +985,7 @@ export class JatsDocument { } frontStub( - frontmatter?: PageFrontmatter, + frontmatter?: FrontmatterWithParts, state?: IJatsSerializer, notebookRep?: boolean, ): Element[] { @@ -1021,6 +1025,7 @@ export class JatsDocument { isSubArticle: true, slug: content.slug, extractAbstract: !notebookRep, + frontmatterParts: content.frontmatter?.parts, }); } @@ -1075,7 +1080,7 @@ export function writeJats(file: VFile, content: ArticleContent, opts?: DocumentO } const plugin: Plugin< - [SourceFileKind, PageFrontmatter?, CitationRenderer?, string?, DocumentOptions?], + [SourceFileKind, FrontmatterWithParts?, CitationRenderer?, string?, DocumentOptions?], Root, VFile > = function (kind, frontmatter, citations, slug, opts) { diff --git a/packages/myst-to-jats/src/types.ts b/packages/myst-to-jats/src/types.ts index 4055ad228..2a2ae9317 100644 --- a/packages/myst-to-jats/src/types.ts +++ b/packages/myst-to-jats/src/types.ts @@ -1,4 +1,4 @@ -import type { GenericNode, MessageInfo } from 'myst-common'; +import type { FrontmatterParts, GenericNode, MessageInfo } from 'myst-common'; import type { PageFrontmatter } from 'myst-frontmatter'; import type { Root } from 'myst-spec'; import type { SourceFileKind } from 'myst-spec-ext'; @@ -34,6 +34,7 @@ export type Options = { extractAbstract?: boolean; abstractParts?: JatsPart[]; backSections?: JatsPart[]; + frontmatterParts?: FrontmatterParts; }; export type DocumentOptions = Options & @@ -51,10 +52,14 @@ export type StateData = { acknowledgments?: Element; }; +export type FrontmatterWithParts = Omit & { + parts?: FrontmatterParts; +}; + export type ArticleContent = { mdast: Root; kind: SourceFileKind; - frontmatter?: PageFrontmatter; + frontmatter?: FrontmatterWithParts; citations?: CitationRenderer; slug?: string; }; diff --git a/packages/myst-transforms/src/frontmatter.ts b/packages/myst-transforms/src/frontmatter.ts index 962ee3976..539fd11a6 100644 --- a/packages/myst-transforms/src/frontmatter.ts +++ b/packages/myst-transforms/src/frontmatter.ts @@ -4,6 +4,7 @@ import { select } from 'unist-util-select'; import type { Block, Code, Heading } from 'myst-spec'; import type { GenericParent } from 'myst-common'; import { RuleId, fileError, toText, fileWarn, normalizeLabel } from 'myst-common'; +import { fillProjectFrontmatter } from 'myst-frontmatter'; import type { VFile } from 'vfile'; import { mystTargetsTransform } from './targets.js'; import { liftMystDirectivesAndRolesTransform } from './liftMystDirectivesAndRoles.js'; @@ -17,7 +18,7 @@ type Options = { propagateTargets?: boolean; /** * `preFrontmatter` overrides frontmatter from the file. It must be taken - * into account this early so tile is not removed if preFrontmatter.title + * into account this early so title is not removed if preFrontmatter.title * is defined. */ preFrontmatter?: Record; @@ -68,7 +69,17 @@ export function getFrontmatter( } } if (opts.preFrontmatter) { - frontmatter = { ...frontmatter, ...opts.preFrontmatter }; + frontmatter = fillProjectFrontmatter(opts.preFrontmatter, frontmatter, { + property: 'frontmatter', + file: file.path, + messages: {}, + errorLogFn: (message: string) => { + fileError(file, message, { ruleId: RuleId.validPageFrontmatter }); + }, + warningLogFn: (message: string) => { + fileWarn(file, message, { ruleId: RuleId.validPageFrontmatter }); + }, + }); } if (frontmatter.content_includes_title != null) { fileWarn(file, `'frontmatter' cannot explicitly set: content_includes_title`, { diff --git a/packages/mystmd/tests/endToEnd.spec.ts b/packages/mystmd/tests/endToEnd.spec.ts index a9cb4262e..134811a9f 100644 --- a/packages/mystmd/tests/endToEnd.spec.ts +++ b/packages/mystmd/tests/endToEnd.spec.ts @@ -28,39 +28,35 @@ function resolve(relative: string) { const only = ''; -describe.concurrent( - 'End-to-end cli export tests', - () => { - const cases = loadCases('exports.yml'); - test.each( - cases.filter((c) => !only || c.title === only).map((c): [string, TestCase] => [c.title, c]), - )('%s', async (_, { cwd, command, outputs }) => { - // Clean expected outputs if they already exist - await Promise.all( - outputs.map(async (output) => { - if (fs.existsSync(resolve(output.path))) { - await exec(`rm ${resolve(output.path)}`, { cwd: resolve(cwd) }); - } - }), - ); - // Run CLI command - await exec(command, { cwd: resolve(cwd) }); - // Expect correct output - outputs.forEach((output) => { - expect(fs.existsSync(resolve(output.path))).toBeTruthy(); - if (path.extname(output.content) === '.json') { - expect( - JSON.parse(fs.readFileSync(resolve(output.path), { encoding: 'utf-8' })), - ).toMatchObject( - JSON.parse(fs.readFileSync(resolve(output.content), { encoding: 'utf-8' })), - ); - } else { - expect(fs.readFileSync(resolve(output.path), { encoding: 'utf-8' })).toEqual( - fs.readFileSync(resolve(output.content), { encoding: 'utf-8' }), - ); +describe.concurrent('End-to-end cli export tests', { timeout: 15000 }, () => { + const cases = loadCases('exports.yml'); + test.each( + cases.filter((c) => !only || c.title === only).map((c): [string, TestCase] => [c.title, c]), + )('%s', async (_, { cwd, command, outputs }) => { + // Clean expected outputs if they already exist + await Promise.all( + outputs.map(async (output) => { + if (fs.existsSync(resolve(output.path))) { + await exec(`rm ${resolve(output.path)}`, { cwd: resolve(cwd) }); } - }); + }), + ); + // Run CLI command + await exec(command, { cwd: resolve(cwd) }); + // Expect correct output + outputs.forEach((output) => { + expect(fs.existsSync(resolve(output.path))).toBeTruthy(); + if (path.extname(output.content) === '.json') { + expect( + JSON.parse(fs.readFileSync(resolve(output.path), { encoding: 'utf-8' })), + ).toMatchObject( + JSON.parse(fs.readFileSync(resolve(output.content), { encoding: 'utf-8' })), + ); + } else { + expect(fs.readFileSync(resolve(output.path), { encoding: 'utf-8' })).toEqual( + fs.readFileSync(resolve(output.content), { encoding: 'utf-8' }), + ); + } }); - }, - { timeout: 15000 }, -); + }); +});