Add tree sitter to auto api (qwikifiers#984)

* feat(auto-api): adds tree sitter & query working tree-sitter instance and finally figured out how to query * feat(auto-api): saves tree-sitter query as obj * fix(auto-api): general minor improvements Adds better comments and rms console logs * feat(auto api): use AST instead of regex Adds tree-sitter to parse and query the file's AST instead of relying on regex. Unfortunately, because type declarations are object-like and type declarations can have have objects, regex cannot handle the recursive nature of TS types. An oversight that has been corrected. * fix(auto api): removes comment syntax from table text * fix(auto api): adds dependencies
JerryWu1234 · Oct 10, 2024 · f366873 · f366873
1 parent 35c923e
commit f366873
Show file tree

Hide file tree

Showing 3 changed files with 8,332 additions and 10,064 deletions.
diff --git a/apps/website/auto-api.ts b/apps/website/auto-api.ts
@@ -1,6 +1,41 @@
 import * as fs from 'fs';
 import { resolve } from 'path';
 import { ViteDevServer } from 'vite';
+import { default as Parser, Query } from 'tree-sitter';
+import { default as TS } from 'tree-sitter-typescript';
+const parser = new Parser();
+
+/**
+TO WHOM IT MAY CONCERN:
+
+if by some reason you need to refactor the query below and don't know where to starts, below are what I consider to be the must-know parts.
+
+1) Tree-Sitter query docs: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
+1b) Put particular attention to the following sections: capturing nodes, wildcard nodes, and anchors
+
+2) Have a way of being able to see the tree-sitter AST in realtime. The ideal setup comes included in Neovim. In ex mode, simply run
+the command below and you'll have the file's AST viewer open in realtime: InspectTree
+**/
+
+const query = new Query(
+  TS.tsx,
+  `declaration: (type_alias_declaration
+  name: (type_identifier) @subComponentName
+    (intersection_type
+    (object_type
+
+      (comment) @comment
+      .
+      (property_signature
+          name: (_) @prop
+          type: (type_annotation (_) @type)
+      )
+    )
+  )
+)
+`,
+);
+parser.setLanguage(TS.tsx);
 export default function autoAPI() {
   return {
     name: 'watch-monorepo-changes',
@@ -15,9 +50,9 @@ export default function autoAPI() {
   };
 }
 // the object should have this general structure, arranged from parent to child
-// componentName:[subComponent,subcomponent,...]
-// subComponentName:[publicType,publicType,...]
-// publicType:[{ comment,prop,type },{ comment,prop,type },...]
+// componentName:[subComponent,subcomponent,...] & componentName comes from the dir
+// subComponentName/type alias:[publicType,publicType,...] & subcomponent comes from the file under dir
+// publicType:[{ comment,prop,type },{ comment,prop,type },...] & publicType comes from type inside file
 // THEY UPPER-MOST KEY IS ALWAYS USED AS A HEADING
 export type ComponentParts = Record<string, SubComponents>;
 type SubComponents = SubComponent[];
@@ -28,51 +63,57 @@ type ParsedProps = {
   prop: string;
   type: string;
 };
-function parseSingleComponentFromDir(path: string, ref: SubComponents) {
+function parseSingleComponentFromDir(
+  path: string,
+  ref: SubComponents,
+): SubComponents | undefined {
   const component_name = /\/([\w-]*).tsx/.exec(path);
   if (component_name === null || component_name[1] === null) {
     // may need better behavior
     return;
   }
   const sourceCode = fs.readFileSync(path, 'utf-8');
-  const comments = extractPublicTypes(sourceCode);
+  const tree = parser.parse(sourceCode);
   const parsed: PublicType[] = [];
-  for (const comment of comments) {
-    const api = extractComments(comment.string);
-    const pair: PublicType = { [comment.label]: api };
-    parsed.push(pair);
+  function topKey(obj: { [x: string]: any } | undefined) {
+    return obj ? Object.keys(obj)[0] : '';
   }
+  const matches = query.matches(tree.rootNode);
+  matches.forEach((match) => {
+    const last: PublicType = parsed[parsed.length - 1];
+    let subComponentName = '';
+    const parsedProps: ParsedProps = { comment: '', prop: '', type: '' };
+    match.captures.forEach((lol) => {
+      //statetements are ordered as they appear in capture array
+      if (lol.name === 'subComponentName' && subComponentName != lol.node.text) {
+        subComponentName = lol.node.text;
+      }
+      if (lol.name === 'comment') {
+        //this removes the comment syntax
+        const justText = lol.node.text.replaceAll(/[/*]/g, '');
+        parsedProps.comment = justText;
+      }
+
+      if (lol.name === 'prop') {
+        parsedProps.prop = lol.node.text;
+      }
+
+      if (lol.name === 'type') {
+        parsedProps.type = lol.node.text;
+        if (subComponentName === topKey(last)) {
+          last[topKey(last)].push(parsedProps);
+        } else {
+          parsed.push({ [subComponentName]: [parsedProps] });
+        }
+      }
+    });
+  });
+
   const completeSubComponent: SubComponent = { [component_name[1]]: parsed };
   ref.push(completeSubComponent);
   return ref;
 }
 
-function extractPublicTypes(strg: string) {
-  const getPublicTypes = /type Public([A-Z][\w]*)*[\w\W]*?{([\w|\W]*?)}(;| &)/gm;
-  const cms = [];
-  let groups;
-  while ((groups = getPublicTypes.exec(strg)) !== null) {
-    const string = groups[2];
-    cms.push({ label: groups[1], string });
-  }
-  return cms;
-}
-function extractComments(strg: string): ParsedProps[] {
-  const magical_regex =
-    /^\s*?\/[*]{2}\n?([\w|\W|]*?)\s*[*]{1,2}[/]\n[ ]*([\w|\W]*?): ([\w|\W]*?);?$/gm;
-
-  const cms = [];
-  let groups;
-
-  while ((groups = magical_regex.exec(strg)) !== null) {
-    const trimStart = /^ *|(\* *)/g;
-    const comment = groups[1].replaceAll(trimStart, '');
-    const prop = groups[2];
-    const type = groups[3];
-    cms.push({ comment, prop, type });
-  }
-  return cms;
-}
 function writeToDocs(fullPath: string, componentName: string, api: ComponentParts) {
   if (fullPath.includes('kit-headless')) {
     const relDocPath = `../website/src/routes//docs/headless/${componentName}`;

diff --git a/package.json b/package.json
@@ -119,6 +119,8 @@
     "tailwind-merge": "^2.3.0",
     "tailwindcss": "^3.4.3",
     "tailwindcss-animate": "^1.0.7",
+    "tree-sitter": "0.21.1",
+    "tree-sitter-typescript": "0.23.0",
     "ts-jest": "^29.1.3",
     "tslib": "^2.6.2",
     "typescript": "5.4.5",