🏗 Support parts as external files and in the site config (#1586)

Co-authored-by: Rowan Cockett <[email protected]>

.changeset/blue-rocks-pull.md

@@ -0,0 +1,7 @@
+---
+'myst-to-jats': patch
+'myst-common': patch
+'myst-cli': patch
+---
+
+Consume frontmatter parts in static exports

.changeset/silly-candles-smile.md

@@ -0,0 +1,6 @@
+---
+'myst-common': patch
+'myst-cli': patch
+---
+
+Load frontmatter parts as separate files for processing

.changeset/small-tigers-kneel.md

@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Fix local image paths for embedded nodes

.changeset/strong-colts-eat.md

@@ -0,0 +1,6 @@
+---
+'myst-spec-ext': patch
+'myst-cli': patch
+---
+
+Keep track of implicit vs. explicit pages in project TOC

.changeset/stupid-flowers-push.md

@@ -0,0 +1,7 @@
+---
+'myst-frontmatter': patch
+'myst-config': patch
+'myst-cli': patch
+---
+
+Support parts in site config

.changeset/thirty-carrots-guess.md

@@ -0,0 +1,6 @@
+---
+'myst-config': patch
+'myst-cli': patch
+---
+
+Parse project-level parts to mdast

.changeset/young-months-peel.md

@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Update processing to handle parts files

docs/document-parts.md

@@ -22,6 +22,15 @@ abstract: |
 ---
 ```
 
+You may also write your part in a separate file, and point to that file from the frontmatter, for example:
+
+```yaml
+---
+title: My document
+abstract: ../abstract.md
+---
+```
+
 ### Known Frontmatter Parts
 
 The known parts that are recognized as _top-level_ document frontmatter keys are:
@@ -116,4 +125,20 @@ Project-level `parts` are useful, for example, if you have an abstract, acknowle
 
 ```{caution}
 Project-level `parts` are a new feature and may not yet be respected by your chosen MyST template or export format. If the project `part` is not behaving as you expect, try moving it to page frontmatter for now.
+```
+
+(parts:site)=
+
+## Parts in `myst.yml` Site configuration
+
+You may specify `parts` in the site configuration of your `myst.yml` file. These parts will only be used for MyST site builds, and they must correspond to `parts` declared in your [website theme's template](website-templates.md).
+
+```yaml
+version: 1
+site:
+  template: ...
+  parts:
+    footer: |
+      (c) MyST Markdown
+  ...
 ```

docs/website-templates.md

@@ -58,7 +58,7 @@ Site options allow you to configure a theme's behavior.[^opts]
 These should be placed in the `site.options` in your `myst.yml`.
 For example:
 
-[^opts]: They are generally unique to the theme (and thus in a dediated `site.options` key rather than a top-level option in `site`).
+[^opts]: They are generally unique to the theme (and thus in a dedicated `site.options` key rather than a top-level option in `site`).
 
 ```{code-block} yaml
 :filename: myst.yml

packages/myst-cli/src/build/cff.ts

@@ -14,6 +14,7 @@ import { KNOWN_IMAGE_EXTENSIONS } from '../utils/resolveExtension.js';
 import type { ExportWithOutput, ExportFnOptions } from './types.js';
 import { cleanOutput } from './utils/cleanOutput.js';
 import { getFileContent } from './utils/getFileContent.js';
+import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js';
 import { parseMyst } from '../process/myst.js';
 
 function exportOptionsToCFF(exportOptions: ExportWithOutput): CFF {
@@ -39,11 +40,16 @@ export async function runCffExport(
   let frontmatter: PageFrontmatter | undefined;
   let abstract: string | undefined;
   if (projectPath && selectors.selectLocalConfigFile(state, projectPath) === sourceFile) {
+    // Process the project only, without any files
+    await getFileContent(session, [], {
+      projectPath,
+      imageExtensions: KNOWN_IMAGE_EXTENSIONS,
+      extraLinkTransformers,
+    });
     frontmatter = selectors.selectLocalProjectConfig(state, projectPath);
-    const rawAbstract = frontmatter?.parts?.abstract?.join('\n\n');
-    if (rawAbstract) {
-      const abstractAst = parseMyst(session, rawAbstract, sourceFile);
-      abstract = toText(abstractAst);
+    const { abstract: frontmatterAbstract } = resolveFrontmatterParts(session, frontmatter) ?? {};
+    if (frontmatterAbstract) {
+      abstract = toText(frontmatterAbstract.mdast);
     }
   } else if (article.file) {
     const [content] = await getFileContent(session, [article.file], {
@@ -52,7 +58,9 @@ export async function runCffExport(
       extraLinkTransformers,
     });
     frontmatter = content.frontmatter;
-    const abstractMdast = extractPart(content.mdast, 'abstract');
+    const abstractMdast = extractPart(content.mdast, 'abstract', {
+      frontmatterParts: resolveFrontmatterParts(session, frontmatter),
+    });
     if (abstractMdast) abstract = toText(abstractMdast);
   }
   if (!frontmatter) return { tempFolders: [] };

packages/myst-cli/src/build/jats/single.ts

@@ -10,6 +10,7 @@ import { castSession } from '../../session/cache.js';
 import type { ISession } from '../../session/types.js';
 import { logMessagesFromVFile } from '../../utils/logging.js';
 import { KNOWN_IMAGE_EXTENSIONS } from '../../utils/resolveExtension.js';
+import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js';
 import type { ExportWithOutput, ExportFnOptions } from '../types.js';
 import { cleanOutput } from '../utils/cleanOutput.js';
 import { getFileContent } from '../utils/getFileContent.js';
@@ -59,7 +60,12 @@ export async function runJatsExport(
       });
     }),
   );
-  const [processedArticle, ...processedSubArticles] = processedContents;
+  const [processedArticle, ...processedSubArticles] = processedContents.map(
+    ({ frontmatter, ...contents }) => {
+      const parts = resolveFrontmatterParts(session, frontmatter);
+      return { frontmatter: { ...frontmatter, parts }, ...contents };
+    },
+  );
   const vfile = new VFile();
   vfile.path = output;
   const jats = writeJats(vfile, processedArticle as any, {

packages/myst-cli/src/build/site/manifest.ts

@@ -18,6 +18,7 @@ import type { RootState } from '../../store/index.js';
 import { selectors } from '../../store/index.js';
 import { transformBanner, transformThumbnail } from '../../transforms/images.js';
 import { addWarningForFile } from '../../utils/addWarningForFile.js';
+import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js';
 import version from '../../version.js';
 import { getSiteTemplate } from './template.js';
 import { collectExportOptions } from '../utils/collectExportOptions.js';
@@ -173,6 +174,7 @@ export async function localToManifestProject(
   const downloads = projConfigFile
     ? await resolvePageDownloads(session, projConfigFile, projectPath)
     : undefined;
+  const parts = resolveFrontmatterParts(session, projFrontmatter);
   const banner = await transformBanner(
     session,
     path.join(projectPath, 'myst.yml'),
@@ -204,6 +206,7 @@ export async function localToManifestProject(
       undefined,
     exports,
     downloads,
+    parts,
     bibliography: projFrontmatter.bibliography || [],
     title: projectTitle || 'Untitled',
     slug: projectSlug,
@@ -405,8 +408,10 @@ export async function getSiteManifest(
   );
   const resolvedOptions = await resolveTemplateFileOptions(session, mystTemplate, validatedOptions);
   validatedFrontmatter.options = resolvedOptions;
+  const parts = resolveFrontmatterParts(session, validatedFrontmatter);
   const manifest: SiteManifest = {
     ...validatedFrontmatter,
+    parts,
     myst: version,
     nav: nav || [],
     actions: actions || [],

packages/myst-cli/src/build/site/watch.ts

@@ -41,7 +41,7 @@ function triggerProjectReload(
     : selectors.selectCurrentProjectFile(state);
   if (selectors.selectConfigExtensions(state).includes(file)) return true;
   if (file === projectConfigFile || basename(file) === '_toc.yml') return true;
-  // Reload project if file is added or remvoed
+  // Reload project if file is added or removed
   if (['add', 'unlink'].includes(eventType)) return true;
   // Otherwise do not reload project
   return false;
@@ -86,38 +86,46 @@ async function processorFn(
     return;
   }
   const projectPath = siteProject.path;
-  const pageSlug = selectors.selectPageSlug(session.store.getState(), siteProject.path, file);
-  const dependencies = selectors.selectDependentFiles(session.store.getState(), file);
+  const state = session.store.getState();
+  const pageSlug = selectors.selectPageSlug(state, siteProject.path, file);
+  const dependencies = selectors.selectDependentFiles(state, file);
   if (!pageSlug && dependencies.length === 0) {
     session.log.warn(`⚠️ File is not in project: ${file}`);
     return;
   }
-  if (pageSlug) {
-    await fastProcessFile(session, {
-      file,
-      projectPath,
-      projectSlug: siteProject.slug,
-      pageSlug,
-      ...opts,
-    });
-  }
+  await fastProcessFile(session, {
+    file,
+    projectPath,
+    projectSlug: siteProject.slug,
+    pageSlug,
+    ...opts,
+  });
   if (dependencies.length) {
     session.log.info(
       `🔄 Updating dependent pages for ${file} ${chalk.dim(`[${dependencies.join(', ')}]`)}`,
     );
-    await Promise.all([
-      dependencies.map((dep) => {
-        const depSlug = selectors.selectPageSlug(session.store.getState(), projectPath, dep);
-        if (!depSlug) return undefined;
-        return fastProcessFile(session, {
-          file: dep,
-          projectPath,
-          projectSlug: siteProject.slug,
-          pageSlug: depSlug,
-          ...opts,
-        });
-      }),
-    ]);
+    const siteConfig = selectors.selectCurrentSiteFile(state);
+    const projConfig = selectors.selectCurrentProjectFile(state);
+    if (
+      (siteConfig && dependencies.includes(siteConfig)) ||
+      (projConfig && dependencies.includes(projConfig))
+    ) {
+      await processSite(session, { ...opts, reloadProject: true });
+    } else {
+      await Promise.all([
+        dependencies.map(async (dep) => {
+          const depSlug = selectors.selectPageSlug(state, projectPath, dep);
+          if (!depSlug) return undefined;
+          return fastProcessFile(session, {
+            file: dep,
+            projectPath,
+            projectSlug: siteProject.slug,
+            pageSlug: depSlug,
+            ...opts,
+          });
+        }),
+      ]);
+    }
   }
   serverReload();
   // TODO: process full site silently and update if there are any

packages/myst-cli/src/build/tex/single.ts

@@ -31,6 +31,7 @@ import { getFileContent } from '../utils/getFileContent.js';
 import { addWarningForFile } from '../../utils/addWarningForFile.js';
 import { cleanOutput } from '../utils/cleanOutput.js';
 import { createTempFolder } from '../../utils/createTempFolder.js';
+import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js';
 import type { ExportWithOutput, ExportResults, ExportFnOptions } from '../types.js';
 import { writeBibtexFromCitationRenderers } from '../utils/bibtex.js';
 
@@ -72,7 +73,9 @@ export function extractTexPart(
   frontmatter: PageFrontmatter,
   templateYml: TemplateYml,
 ): LatexResult | LatexResult[] | undefined {
-  const part = extractPart(mdast, partDefinition.id);
+  const part = extractPart(mdast, partDefinition.id, {
+    frontmatterParts: resolveFrontmatterParts(session, frontmatter),
+  });
   if (!part) return undefined;
   if (!partDefinition.as_list) {
     // Do not build glossaries when extracting parts: references cannot be mapped to definitions

packages/myst-cli/src/build/typst.ts

@@ -32,6 +32,7 @@ import { logMessagesFromVFile } from '../utils/logging.js';
 import { getFileContent } from './utils/getFileContent.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
 import { createTempFolder } from '../utils/createTempFolder.js';
+import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js';
 import version from '../version.js';
 import { cleanOutput } from './utils/cleanOutput.js';
 import type { ExportWithOutput, ExportResults, ExportFnOptions } from './types.js';
@@ -90,7 +91,9 @@ export function extractTypstPart(
   frontmatter: PageFrontmatter,
   templateYml: TemplateYml,
 ): TypstResult | TypstResult[] | undefined {
-  const part = extractPart(mdast, partDefinition.id);
+  const part = extractPart(mdast, partDefinition.id, {
+    frontmatterParts: resolveFrontmatterParts(session, frontmatter),
+  });
   if (!part) return undefined;
   if (!partDefinition.as_list) {
     // Do not build glossaries when extracting parts: references cannot be mapped to definitions

packages/myst-cli/src/build/utils/getFileContent.ts

@@ -9,6 +9,7 @@ import type { TransformFn } from '../../process/mdast.js';
 import { postProcessMdast, transformMdast } from '../../process/mdast.js';
 import { loadProject, selectPageReferenceStates } from '../../process/site.js';
 import type { ISession } from '../../session/types.js';
+import { selectors } from '../../store/index.js';
 import type { ImageExtensions } from '../../utils/resolveExtension.js';
 
 export async function getFileContent(
@@ -37,13 +38,11 @@ export async function getFileContent(
   projectPath = projectPath ?? resolve('.');
   const { project, pages } = await loadProject(session, projectPath);
   const projectFiles = pages.map((page) => page.file).filter((file) => !files.includes(file));
-  // Keep 'files' indices consistent in 'allFiles' as index is used for other fields.
-  const allFiles = [...files, ...projectFiles];
   await Promise.all([
     // Load all citations (.bib)
     ...project.bibliography.map((path) => loadFile(session, path, projectPath, '.bib')),
     // Load all content (.md, .tex, .myst.json, or .ipynb)
-    ...allFiles.map((file, ind) => {
+    ...[...files, ...projectFiles].map((file, ind) => {
       const preFrontmatter = Array.isArray(preFrontmatters)
         ? preFrontmatters?.[ind]
         : preFrontmatters;
@@ -57,6 +56,10 @@ export async function getFileContent(
   // Consolidate all citations onto single project citation renderer
   combineProjectCitationRenderers(session, projectPath);
 
+  const projectParts = selectors.selectProjectParts(session.store.getState(), projectPath);
+  // Keep 'files' indices consistent in 'allFiles' as index is used for other fields.
+  const allFiles = [...files, ...projectFiles, ...projectParts];
+
   await Promise.all(
     allFiles.map(async (file, ind) => {
       const pageSlug = pages.find((page) => page.file === file)?.slug;
@@ -80,13 +83,17 @@ export async function getFileContent(
       return { file };
     }),
   );
-  const selectedFiles = await Promise.all(
-    files.map(async (file) => {
+  await Promise.all(
+    [...files, ...projectParts].map(async (file) => {
       await postProcessMdast(session, {
         file,
         extraLinkTransformers,
         pageReferenceStates,
       });
+    }),
+  );
+  const selectedFiles = await Promise.all(
+    files.map(async (file) => {
       const selectedFile = selectFile(session, file);
       if (!selectedFile) throw new Error(`Could not load file information for ${file}`);
       return selectedFile;