feat(vscode): share body parsing and external ref logic with core

.github/workflows/test.yaml

@@ -17,7 +17,7 @@ jobs:
 
  # https://github.com/pnpm/action-setup
  - name: Install pnpm
- uses: pnpm/action-setup@v2
+ uses: pnpm/action-setup@v4
  with:
  version: 8.x.x
 
@@ -27,13 +27,13 @@ jobs:
  run: |
  echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
 
- - uses: actions/cache@v3
+ - uses: actions/cache@v4
  name: Setup pnpm cache
  with:
  path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
- key: ${{ runner.os }}-pnpm-store-v1-${{ hashFiles('**/pnpm-lock.yaml') }}
+ key: ${{ runner.os }}-pnpm-store-v2-${{ hashFiles('**/pnpm-lock.yaml') }}
  restore-keys: |
- ${{ runner.os }}-pnpm-store-v1-
+ ${{ runner.os }}-pnpm-store-v2-
 
  - name: Install dependencies
  run: pnpm install

.gitignore

@@ -2,3 +2,6 @@ coverage/
 dist/
 node_modules/
 data/
+out
+.vscode-test/
+*.vsix

.prettierignore

@@ -1,4 +1,6 @@
 coverage
 dist
+out
 pnpm-lock.yaml
 .svelte-kit
+.vscode-test

.tool-versions

@@ -1,3 +1,3 @@
 # Pinned here and in devcontainer to avoid OOM issues stemming from
 # https://github.com/TypeStrong/ts-node/issues/1995
-nodejs 20.11.0
+nodejs 20.14.0

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2023 James Dabbs
+Copyright (c) 2014-2024 James Dabbs
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package.json

@@ -9,15 +9,15 @@
  "test:cov": "pnpm run --recursive test:cov"
  },
  "devDependencies": {
- "@types/node": "^20.11.3",
- "@vitest/coverage-v8": "^0.34.6",
+ "@types/node": "^20.14.0",
+ "@vitest/coverage-v8": "^1.3.0",
  "nodemon": "^2.0.22",
  "npm-check-updates": "^16.14.12",
  "prettier": "^2.8.8",
  "prettier-plugin-svelte": "^2.10.1",
  "ts-node": "^10.9.2",
- "typescript": "^5.3.3",
- "vite": "^4.5.1",
- "vitest": "^0.34.6"
+ "typescript": "^5.4.5",
+ "vite": "^5.0.0",
+ "vitest": "^1.3.0"
  }
 }

packages/compile/package.json

@@ -37,7 +37,7 @@
  "glob": "^8.1.0",
  "js-yaml": "^4.1.0",
  "yaml-front-matter": "^4.1.1",
- "zod": "^3.22.4"
+ "zod": "^3.23.8"
  },
  "devDependencies": {
  "@types/cors": "^2.8.17",

packages/compile/tsconfig.json

@@ -4,7 +4,7 @@
  "declaration": true,
  "declarationDir": "dist/types",
  "module": "es2022",
- "moduleResolution": "node",
+ "moduleResolution": "bundler",
  "noImplicitAny": true,
  "outDir": "dist/esm",
  "preserveConstEnums": true,

packages/core/bin/build

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -exo pipefail
+
+# VSCode extensions can only use CommonJS modules (for now), but we want to 
+# continue using our standard Vite/ESM build process elsewhere. This builds
+# both versions, corresponding to `package.json`'s `exports` field.
+
+tsc --module es2022 --outDir dist/esm/
+echo '{"type": "module"}' > dist/esm/package.json
+
+tsc --module commonjs --outDir dist/cjs/
+echo '{"type": "commonjs"}' > dist/cjs/package.json

packages/core/package.json

@@ -14,23 +14,27 @@
  },
  "license": "MIT",
  "author": "James Dabbs <[email protected]> (https://jdabbs.com)",
- "main": "./dist/esm/index.js",
  "types": "./dist/types/index.d.ts",
+ "exports": {
+ "types": "./dist/types/index.d.ts",
+ "require": "./dist/cjs/index.js",
+ "import": "./dist/esm/index.js"
+ },
  "repository": {
  "type": "git",
  "url": "git+https://github.com/pi-base/web.git"
  },
  "scripts": {
  "build:peg": "peggy --plugin ./node_modules/ts-pegjs/dist/tspegjs -o src/Formula/Grammar.ts --cache src/Formula/Grammar.pegjs",
- "build": "pnpm build:peg && tsc",
+ "build": "pnpm build:peg && ./bin/build",
  "dev": "pnpm build:peg && tsc --watch",
  "test": "vitest run",
  "test:cov": "vitest run --coverage",
  "test:watch": "vitest"
  },
  "dependencies": {
  "debug": "^4.3.4",
- "micromark-util-types": "1.0.2",
+ "js-yaml": "^4.1.0",
  "rehype-katex": "^6.0.3",
  "rehype-stringify": "^9.0.4",
  "remark": "^14.0.3",
@@ -40,15 +44,17 @@
  "unified": "^10.1.2",
  "unist-util-is": "^5.2.1",
  "unist-util-visit": "^4.1.2",
- "zod": "^3.22.4"
+ "zod": "^3.23.8"
  },
  "devDependencies": {
  "@types/debug": "^4.1.12",
  "@types/hast": "^2.3.9",
+ "@types/js-yaml": "^4.0.9",
  "@types/mdast": "^3.0.15",
  "@types/unist": "^2.0.10",
  "hast-util-to-html": "^8.0.4",
  "mdast-util-from-markdown": "1.3.0",
+ "micromark-util-types": "1.0.2",
  "peggy": "^3.0.2",
  "ts-pegjs": "^4.2.1"
  }

packages/core/src/Bundle.ts

@@ -1,16 +1,11 @@
-/// <reference types="vite/client" />
 import { z } from 'zod'
 import { Id, traitId } from './Id.js'
 import { Property, propertySchema } from './Property.js'
 import { Space, spaceSchema } from './Space.js'
 import { Theorem, theoremSchema } from './Theorem.js'
 import { Trait, traitSchema } from './Trait.js'
 
-export const defaultHost = import.meta.env?.VITE_PUBLIC_DATA_URL
- ? import.meta.env.VITE_PUBLIC_DATA_URL
- : import.meta.env?.DEV
- ? 'http://localhost:3141'
- : 'https://pi-base-bundles.s3.us-east-2.amazonaws.com'
+export const defaultHost = 'https://pi-base-bundles.s3.us-east-2.amazonaws.com'
 
 export type Version = {
  ref: string

packages/core/src/Document.ts

@@ -0,0 +1,19 @@
+import * as z from 'zod'
+import * as yaml from 'js-yaml'
+
+// TODO: unify with logic in packages/compiler
+export function parseDocument<T>(schema: z.ZodSchema<T>, contents: string) {
+ const match = contents.match(
+ /^(---)?\s*(?<frontmatter>[\s\S]*?)\s*---(?<body>[\s\S]*)/,
+ )
+ if (!match?.groups) {
+ return
+ }
+
+ const { frontmatter, body } = match.groups
+ const meta = yaml.load(frontmatter)
+ const data = { description: body.trim() }
+ const raw = typeof meta === 'object' ? { ...meta, ...data } : data
+
+ return schema.safeParse(raw)
+}

packages/core/src/Id.ts

@@ -54,3 +54,39 @@ export function toInt(id: string): number {
 
  return tagged.id
 }
+
+// TODO: these were extracted from other parallel-but-divergent implementations
+// and should be unified.
+
+type Pad = '' | '0' | '00' | '000' | '0000' | '00000' | '00000'
+type XId<Prefix extends string> = `${Prefix}${Pad}${number}`
+
+export type SId = XId<'S'>
+export type PId = XId<'P'>
+export type TId = XId<'T'>
+export type SPId = [SId, PId]
+export type EntityId = SId | PId | TId | SPId
+
+export function isSpaceId(token: string): token is SId {
+ return token.match(/^S\d{1,6}$/) !== null
+}
+
+export function isPropertyId(token: string): token is PId {
+ return token.match(/^P\d{1,6}$/) !== null
+}
+
+export function isTheoremId(token: string): token is SId {
+ return token.match(/^T\d{1,6}$/) !== null
+}
+
+export function isTraitId(pair: [string, string]): pair is SPId {
+ return isSpaceId(pair[0]) && isPropertyId(pair[1])
+}
+
+export const idExp = /[PST]\d{1,6}/g
+
+export function normalizeId(id: SId): SId
+export function normalizeId(id: PId): PId
+export function normalizeId(id: string) {
+ return `${id[0]}${id.slice(1).padStart(6, '0')}`
+}

packages/core/src/Parser.ts

@@ -13,7 +13,7 @@ import { truncate as truncator } from './Parser/truncate.js'
 import { unnest } from './Parser/unnest.js'
 
 export type Options = {
- link: Linkers
+ link: Partial<Linkers>
  truncate?: boolean
 }
 

packages/core/src/Parser/references.ts

@@ -10,12 +10,30 @@ import { ExternalLinkNode, InternalLinkNode, Linkers } from './types'
  *
  * See https://github.com/syntax-tree/mdast-util-to-hast#fields-on-nodes
  */
-export function references({ internal, external }: Linkers) {
+export function references({ internal, external }: Partial<Linkers>) {
  return (): Transformer<Root, Root> => {
  return function transformer(tree: Root) {
  visit(tree, 'internalLink', (node: InternalLinkNode) => {
  const { kind, id } = node
 
+ if (!internal) {
+ Object.assign(node, {
+ data: {
+ hName: 'span',
+ hProperties: {
+ className: 'internal-link',
+ },
+ hChildren: [
+ {
+ type: 'text',
+ value: `${kind}${id}`,
+ },
+ ],
+ },
+ })
+ return
+ }
+
  const { href, title } = internal([kind, id])
 
  Object.assign(node, {
@@ -36,26 +54,27 @@ export function references({ internal, external }: Linkers) {
  })
  })
 
- visit(tree, 'externalLink', (node: ExternalLinkNode) => {
- const { href, title } = external([node.kind, node.id])
+ external &&
+ visit(tree, 'externalLink', (node: ExternalLinkNode) => {
+ const { href, title } = external([node.kind, node.id])
 
- Object.assign(node, {
- data: {
- hName: 'a',
- hProperties: {
- href,
- title,
- className: 'external-link',
- },
- hChildren: [
- {
- type: 'text',
- value: title,
+ Object.assign(node, {
+ data: {
+ hName: 'a',
+ hProperties: {
+ href,
+ title,
+ className: 'external-link',
  },
- ],
- },
+ hChildren: [
+ {
+ type: 'text',
+ value: title,
+ },
+ ],
+ },
+ })
  })
- })
  }
  }
 }

packages/core/src/Property.ts

@@ -1,5 +1,15 @@
 import { z } from 'zod'
 import { recordSchema } from './Record.js'
+import { refSchema } from './Ref.js'
+
+export const propertyPageSchema = z.object({
+ uid: z.string(),
+ name: z.string(),
+ aliases: z.array(z.string()).optional(),
+ counterexamples_id: z.number().nullable().optional(),
+ refs: z.array(refSchema).optional(),
+ description: z.string(),
+})
 
 export const propertySchema = z.intersection(
  z.object({
@@ -10,3 +20,4 @@ export const propertySchema = z.intersection(
 )
 
 export type Property = z.infer<typeof propertySchema>
+export type PropertyPage = z.infer<typeof propertyPageSchema>

packages/core/src/Ref.ts

@@ -24,6 +24,8 @@ export type TaggedRef =
  | { kind: 'mo'; id: string; name?: string }
  | { kind: 'zb'; id: string; name?: string }
 
+export type Kind = TaggedRef['kind']
+
 export function tag(ref: Ref): TaggedRef {
  const { name } = ref
 
@@ -41,3 +43,42 @@ export function tag(ref: Ref): TaggedRef {
  return { kind: 'zb', id: String(ref.zb), name }
  }
 }
+
+type FormatInput = { kind: Kind; id: string; name?: string } | Ref
+
+export function format(input: FormatInput) {
+ if ('kind' in input) {
+ const { kind, id, name } = input
+ switch (kind) {
+ case 'doi':
+ return { href: `https://doi.org/${id}`, title: name || `DOI ${id}` }
+ case 'mr':
+ return {
+ href: `https://mathscinet.ams.org/mathscinet-getitem?mr=${id}`,
+ title: name || `MR ${id}`,
+ }
+ case 'wikipedia':
+ return {
+ href: `https://en.wikipedia.org/wiki/${id}`,
+ title: name || `Wikipedia ${id}`,
+ }
+ case 'mathse':
+ return {
+ href: `https://math.stackexchange.com/questions/${id}`,
+ title: name || `Math StackExchange ${id}`,
+ }
+ case 'mo':
+ return {
+ href: `https://mathoverflow.net/questions/${id}`,
+ title: name || `MathOverflow ${id}`,
+ }
+ case 'zb':
+ return {
+ href: `https://zbmath.org/${id}`,
+ title: name || `zbMATH ${id}`,
+ }
+ }
+ } else {
+ return format(tag(input))
+ }
+}