remove old Ast, simplify codegen build sequence

app/gui2/env.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="vite/client" />
 
 declare const PROJECT_MANAGER_URL: string
-declare const RUNNING_VTEST: boolean
+declare const RUNNING_VITEST: boolean
 
 // This is an augmentation to the built-in `ImportMeta` interface.
 // This file MUST NOT contain any top-level imports.

app/gui2/eslint.config.js

@@ -9,7 +9,7 @@ const DIR_NAME = path.dirname(url.fileURLToPath(import.meta.url))
 
 const conf = [
   {
-    ignores: ['rust-ffi/pkg', 'dist', 'parser-codegen/dist', 'src/generated'],
+    ignores: ['rust-ffi/pkg', 'dist', 'src/generated'],
   },
   ...compat.extends('plugin:vue/vue3-recommended'),
   eslintJs.configs.recommended,

app/gui2/package.json

@@ -19,9 +19,10 @@
     "typecheck": "vue-tsc --noEmit -p tsconfig.app.json --composite false",
     "lint": "eslint .",
     "format": "prettier --write src/ && eslint . --fix",
-    "build-rust-ffi": "cd rust-ffi && wasm-pack build --release --target web",
-    "generate-parser-bindings": "cd parser-codegen && npm install && npm run generate-bindings ../src/generated/ast.ts",
-    "preinstall": "npm run build-rust-ffi && npm run generate-parser-bindings"
+    "build-rust-ffi": "wasm-pack build ./rust-ffi --release --target web",
+    "generate-ast-schema": "cargo run -p enso-parser-schema > src/generated/ast-schema.json",
+    "generate-ast-types": "tsx ./parser-codegen/index.ts src/generated/ast-schema.json src/generated/ast.ts",
+    "preinstall": "npm run build-rust-ffi && npm run generate-ast-schema && npm run generate-ast-types"
   },
   "dependencies": {
     "@babel/parser": "^7.22.16",
@@ -76,6 +77,7 @@
     "@vue/tsconfig": "^0.4.0",
     "ag-grid-community": "^30.1.0",
     "ag-grid-enterprise": "^30.1.0",
+    "change-case": "^5.0.2",
     "d3": "^7.4.0",
     "esbuild": "^0.19.3",
     "eslint": "^8.49.0",
@@ -87,6 +89,7 @@
     "shuffle-seed": "^1.1.6",
     "sql-formatter": "^13.0.0",
     "tailwindcss": "^3.2.7",
+    "tsx": "^3.12.6",
     "typescript": "~5.2.2",
     "vite": "^4.4.9",
     "vite-plugin-inspect": "^0.7.38",

app/gui2/parser-codegen/src/codegen.ts → app/gui2/parser-codegen/codegen.ts

@@ -27,7 +27,7 @@ import {
   toCamel,
   toPascal,
 } from './util.js'
-const { factory: tsf } = ts
+const tsf = ts.factory
 
 // === Public API ===
 
@@ -59,8 +59,8 @@ export function implement(schema: Schema.Schema): string {
   )
   for (const id in schema.types) {
     const ty = schema.types[id]
-    if (ty.parent == null) {
-      const discriminants = schema.serialization[id].discriminants
+    if (ty?.parent == null) {
+      const discriminants = schema.serialization[id]?.discriminants
       if (discriminants == null) {
         emit(makeConcreteType(id, schema))
       } else {
@@ -86,9 +86,11 @@ function makeType(ref: Schema.TypeRef, schema: Schema.Schema): Type {
   switch (c) {
     case 'type': {
       const ty = schema.types[ref.id]
+      if (!ty) throw new Error(`Invalid type ref: ${ref.id}`)
       const parent = ty.parent != null ? schema.types[ty.parent] : undefined
       const typeName = namespacedName(ty.name, parent?.name)
       const layout = schema.serialization[ref.id]
+      if (!layout) throw new Error(`Invalid serialization ref: ${ref.id}`)
       if (layout.discriminants != null) {
         return Type.Abstract(typeName)
       } else {
@@ -114,7 +116,7 @@ function makeType(ref: Schema.TypeRef, schema: Schema.Schema): Type {
           return Type.String
         default: {
           const _ = p satisfies never
-          throw new Error("unreachable: PrimitiveType.type='" + p + "'")
+          throw new Error(`unreachable: PrimitiveType.type='${p}'`)
         }
       }
     }
@@ -126,7 +128,7 @@ function makeType(ref: Schema.TypeRef, schema: Schema.Schema): Type {
       return Type.Result(makeType(ref.type0, schema), makeType(ref.type1, schema))
     default: {
       const _ = c satisfies never
-      throw new Error("unreachable: TypeRef.class='" + c + "' in " + JSON.stringify(ref))
+      throw new Error(`unreachable: TypeRef.class='${c}' in ${JSON.stringify(ref)}`)
     }
   }
 }
@@ -155,7 +157,7 @@ function makeGetter(field: Field): ts.GetAccessorDeclaration {
 }
 
 function makeConcreteType(id: string, schema: Schema.Schema): ts.ClassDeclaration {
-  const ident = tsf.createIdentifier(toPascal(schema.types[id].name))
+  const ident = tsf.createIdentifier(toPascal(schema.types[id]!.name))
   const paramIdent = tsf.createIdentifier('cursor')
   const cursorParam = tsf.createParameterDeclaration(
     [],
@@ -233,9 +235,14 @@ function makeDebugFunction(fields: Field[], typeName?: string): ts.MethodDeclara
 }
 
 function makeGetters(id: string, schema: Schema.Schema, typeName?: string): ts.ClassElement[] {
-  const fields = schema.serialization[id].fields.map(([name, offset]: [string, number]) =>
-    makeField(name, schema.types[id].fields[name], offset, schema),
-  )
+  const serialization = schema.serialization[id]
+  const type = schema.types[id]
+  if (serialization == null || type == null) throw new Error(`Invalid type id: ${id}`)
+  const fields = serialization.fields.map(([name, offset]: [string, number]) => {
+    const field = type.fields[name]
+    if (field == null) throw new Error(`Invalid field name '${name}' for type '${type.name}'`)
+    return makeField(name, field, offset, schema)
+  })
   return [...fields.map(makeGetter), makeDebugFunction(fields, typeName)]
 }
 
@@ -269,10 +276,11 @@ type ChildType = {
 function makeChildType(
   base: ts.Identifier,
   id: string,
-  discrim: string,
+  discriminant: string,
   schema: Schema.Schema,
 ): ChildType {
-  const ty: Schema.Type = schema.types[id]
+  const ty = schema.types[id]
+  if (ty == null) throw new Error(`Invalid type id: ${id}`)
   const name = toPascal(ty.name)
   const ident = tsf.createIdentifier(name)
   const cursorIdent = tsf.createIdentifier('cursor')
@@ -284,7 +292,7 @@ function makeChildType(
     support.Cursor,
     undefined,
   )
-  const discrimInt = tsf.createNumericLiteral(parseInt(discrim, 10))
+  const discriminantInt = tsf.createNumericLiteral(parseInt(discriminant, 10))
   return {
     definition: tsf.createClassDeclaration(
       [modifiers.export],
@@ -341,8 +349,8 @@ function makeChildType(
       ],
     ),
     reference: tsf.createTypeReferenceNode(name),
-    enumMember: tsf.createEnumMember(toPascal(schema.types[id].name), discrimInt),
-    case: tsf.createCaseClause(discrimInt, [
+    enumMember: tsf.createEnumMember(toPascal(ty.name), discriminantInt),
+    case: tsf.createCaseClause(discriminantInt, [
       tsf.createReturnStatement(tsf.createNewExpression(ident, [], [seekCursor(cursorIdent, 4)])),
     ]),
   }
@@ -358,7 +366,7 @@ function makeAbstractType(
   discriminants: Schema.DiscriminantMap,
   schema: Schema.Schema,
 ): AbstractType {
-  const ty = schema.types[id]
+  const ty = schema.types[id]!
   const name = toPascal(ty.name)
   const ident = tsf.createIdentifier(name)
   const baseIdent = tsf.createIdentifier('AbstractBase')

app/gui2/parser-codegen/index.ts

@@ -0,0 +1,17 @@
+import * as fs from 'node:fs'
+import * as process from 'node:process'
+import * as codegen from './codegen.js'
+import * as Schema from './schema.js'
+
+const schemaPath = process.argv[2]
+const outputPath = process.argv[3]
+
+if (!schemaPath || !outputPath) {
+  console.error('Usage: parser-codegen <schemaPath> <outputPath>')
+  process.exit(1)
+}
+
+console.log(`Generating ${outputPath} from ${schemaPath}.`)
+const schema: Schema.Schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8'))
+const code = codegen.implement(schema)
+fs.writeFileSync(outputPath, code)

app/gui2/parser-codegen/src/util.ts → app/gui2/parser-codegen/util.ts

@@ -1,6 +1,6 @@
 import * as changeCase from 'change-case'
 import ts from 'typescript'
-const { factory: tsf } = ts
+const tsf = ts.factory
 
 // === Identifier utilities ===
 

app/gui2/rust-ffi/src/lib.rs

@@ -14,12 +14,6 @@ thread_local! {
     pub static PARSER: Parser = Parser::new();
 }
 
-#[wasm_bindgen]
-pub fn parse_to_json(code: &str) -> String {
-    let ast = PARSER.with(|parser| parser.run(code));
-    serde_json::to_string(&ast).expect("Failed to serialize AST to JSON")
-}
-
 #[wasm_bindgen]
 pub fn parse_doc_to_json(docs: &str) -> String {
     let docs = enso_doc_parser::parse(docs);

app/gui2/src/stores/graph.ts

@@ -1,6 +1,6 @@
 import { assert, assertNever } from '@/util/assert'
+import { Ast, parseEnso } from '@/util/ast'
 import { useObserveYjs } from '@/util/crdt'
-import { parseEnso, type Ast } from '@/util/ffi'
 import type { Opt } from '@/util/opt'
 import { Vec2 } from '@/util/vec2'
 import * as map from 'lib0/map'

app/gui2/src/stores/suggestionDatabase/__tests__/lsUpdate.test.ts

@@ -1,4 +1,4 @@
-import { parseDocs } from '@/util/ffi'
+import { parseDocs } from '@/util/docParser'
 import { tryIdentifier, tryQualifiedName } from '@/util/qualifiedName'
 import { unwrap } from '@/util/result'
 import * as lsTypes from 'shared/languageServerTypes/suggestions'

app/gui2/src/stores/suggestionDatabase/documentation.ts

@@ -1,6 +1,6 @@
 import type { Group } from '@/stores/suggestionDatabase'
 import { findIndexOpt } from '@/util/array'
-import { parseDocs, type Doc } from '@/util/ffi'
+import { parseDocs, type Doc } from '@/util/docParser'
 import { isSome, type Opt } from '@/util/opt'
 import { tryQualifiedName, type QualifiedName } from '@/util/qualifiedName'
 import { unwrap } from '@/util/result'

app/gui2/src/stores/suggestionDatabase/entry.ts

@@ -1,5 +1,5 @@
 import { assert } from '@/util/assert'
-import type { Doc } from '@/util/ffi'
+import type { Doc } from '@/util/docParser'
 import {
   isIdentifier,
   isQualifiedName,
@@ -13,7 +13,6 @@ import type {
   SuggestionEntryArgument,
   SuggestionEntryScope,
 } from 'shared/languageServerTypes/suggestions'
-export type { Doc } from '@/util/ffi'
 export type {
   SuggestionEntryArgument,
   SuggestionEntryScope,

app/gui2/src/stores/suggestionDatabase/lsUpdate.ts

@@ -1,13 +1,13 @@
 import { SuggestionDb, type Group } from '@/stores/suggestionDatabase'
 import {
   SuggestionKind,
-  type Doc,
   type SuggestionEntry,
   type SuggestionEntryArgument,
   type SuggestionEntryScope,
   type Typename,
 } from '@/stores/suggestionDatabase/entry'
 import { assert } from '@/util/assert'
+import type { Doc } from '@/util/docParser'
 import { type Opt } from '@/util/opt'
 import {
   qnJoin,