feat(expressions): better language support (#1895)

* feat(expressions): better language support

* fix(expressions): optimize codepath and fix string token handling

* fix(expressions): mysterious bug

* feat(*): add functions in schema and update docs

* fix(expressions): update sandbox

* docs(expressions): update docs

* fix(expressions): export types correctly

* fix(expressions): type imports

* fix(expressions): include schema name in language ID

packages/core/expressions/docs/expressions-editor.md

@@ -76,6 +76,40 @@ To control whether the editor should show the details of the autocompletion item
 
 Options to pass when creating the Monaco editor.
 
+#### `provideRhsValueCompletion`
+
+- type: `ProvideRhsValueCompletion`
+- required: `false`
+- default: `undefined`
+
+A function to provide completion items for the value of the right-hand side (RHS) of the expression. The function should return a `Promise` that resolves to a `CompletionList` or `undefined`.
+
+For example, in the following case:
+
+```
+http.path == "fo"
+                ^cursor
+
+↓↓ type "o" ↓↓
+
+http.path == "foo"
+                 ^cursor
+```
+
+… the `provideRhsValueCompletion` function will be called with the following arguments:
+
+| Argument        | Value                                                                      |
+| :-------------- | :------------------------------------------------------------------------- |
+| `lhsValue`      | `"http.path"`                                                              |
+| `rhsValueValue` | `"foo"`                                                                    |
+| `lhsRange`      | `{ startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 9 }`   |
+| `rhsValueRange` | `{ startLineNumber: 1, startColumn: 15, endLineNumber: 1, endColumn: 17 }` |
+
+This function will only be triggered in either of the following cases:
+
+- The completion is manually triggered by the user (e.g., with `Ctrl` `I`)
+- A character is typed while the cursor is inside the range of the string value
+
 ### Events
 
 #### update:modelValue

packages/core/expressions/sandbox/App.vue

@@ -25,6 +25,7 @@
 
       <ExpressionsEditor
         v-model="expression"
+        :provide-rhs-value-completion="provideRhsCompletion"
         :schema="schemaDefinition"
         @parse-result-update="onParseResultUpdate"
       />
@@ -57,9 +58,11 @@
 </template>
 
 <script setup lang="ts">
+import * as monaco from 'monaco-editor'
 import { ref, watch } from 'vue'
 import type { SchemaDefinition } from '../src'
-import { ExpressionsEditor, HTTP_SCHEMA_DEFINITION, STREAM_SCHEMA_DEFINITION, RouterPlaygroundModal } from '../src'
+import { ExpressionsEditor, HTTP_SCHEMA_DEFINITION, RouterPlaygroundModal, STREAM_SCHEMA_DEFINITION } from '../src'
+import type { ProvideRhsValueCompletion } from '../src/components/ExpressionsEditor.vue'
 
 type NamedSchemaDefinition = { name: string; definition: SchemaDefinition }
 
@@ -84,7 +87,7 @@ const expressionPresets = [
 
 const btoa = (s: string) => window.btoa(s)
 
-const expression = ref(expressionPresets[0])
+const expression = ref('lower(http.path) == "/kong" || lower(lower(http.path)) == "/kong"')
 const schemaDefinition = ref<NamedSchemaDefinition>(schemaPresets[0])
 const parseResult = ref('')
 const isVisible = ref(false)
@@ -98,6 +101,21 @@ const handleCommit = (exp: string) => {
   isVisible.value = false
 }
 
+const provideRhsCompletion: ProvideRhsValueCompletion = async (lhsValue, rhsValueValue, lhsRange, rhsValueRange) => {
+  return {
+    suggestions: new Array(10).fill(0).map(() => {
+      const text = `${rhsValueValue}+${Math.random().toString(36).slice(2, 6)}`
+      return {
+        label: text,
+        kind: monaco.languages.CompletionItemKind.Value,
+        detail: `lhs = ${lhsValue}`,
+        insertText:text,
+        range: rhsValueRange,
+      }
+    }),
+  }
+}
+
 watch(schemaDefinition, (newSchemaDefinition) => {
   schemaDefinition.value = newSchemaDefinition
 })

packages/core/expressions/src/components/ExpressionsEditor.vue

@@ -7,13 +7,14 @@
 
 <script setup lang="ts">
 import { useDebounce } from '@kong-ui-public/core'
-import type { ParseResult, ParseResultOk, Schema as AtcSchema } from '@kong/atc-router'
+import type { AstType, Schema as AtcSchema, ParseResult, ParseResultOk } from '@kong/atc-router'
 import { Parser } from '@kong/atc-router'
 import type * as Monaco from 'monaco-editor'
 import * as monaco from 'monaco-editor'
 import { computed, onBeforeUnmount, onMounted, ref, shallowRef, watch } from 'vue'
+import { buildLanguageId, getRangeFromTokens, locateLhsIdent, locateToken, registerLanguage, registerTheme, scanTokens, theme, TokenType, transformTokens } from '../monaco'
 import { createSchema, type Schema } from '../schema'
-import { registerLanguage, registerTheme, theme } from '../monaco'
+import type { ProvideCompletionItems, ProvideRhsValueCompletion } from '../types'
 
 let editor: Monaco.editor.IStandaloneCodeEditor | undefined
 let editorModel: Monaco.editor.ITextModel
@@ -22,15 +23,17 @@ const editorRef = shallowRef<Monaco.editor.IStandaloneCodeEditor>()
 const { debounce } = useDebounce()
 
 const props = withDefaults(defineProps<{
-  schema: Schema,
-  parseDebounce?: number,
-  inactiveUntilFocused?: boolean,
-  allowEmptyInput?: boolean,
-  defaultShowDetails?: boolean,
+  schema: Schema
+  parseDebounce?: number
+  inactiveUntilFocused?: boolean
+  allowEmptyInput?: boolean
+  defaultShowDetails?: boolean
   editorOptions?: Monaco.editor.IEditorOptions
+  provideRhsValueCompletion?: ProvideRhsValueCompletion
 }>(), {
   parseDebounce: 500,
   editorOptions: undefined,
+  provideRhsValueCompletion: undefined,
 })
 
 const parse = (expression: string, schema: AtcSchema) => {
@@ -56,9 +59,107 @@ const editorClass = computed(() => [
   { invalid: isParsingActive.value && parseResult.value?.status !== 'ok' },
 ])
 
-const registerSchema = (schema: Schema) => {
-  const { languageId } = registerLanguage(schema)
-  monaco.editor.setModelLanguage(editorModel, languageId)
+interface Item {
+  property: string
+  kind: AstType
+  documentation?: string
+}
+
+const flattenProperties = (schema: Schema): Array<Item> => {
+  const { definition, documentation } = schema
+  const properties: Array<Item> = []
+  Object.entries(definition).forEach(([kind, fields]) => {
+    fields.forEach((field) => {
+      properties.push({
+        property: field,
+        kind: kind as AstType,
+        documentation: documentation?.[field],
+      })
+    })
+  })
+  return properties
+}
+
+const schema = computed(() => createSchema(props.schema.definition))
+const flatSchemaProperties = computed(() => flattenProperties(props.schema))
+
+const provideCompletionItems: ProvideCompletionItems = async (model, position) => {
+  const [flatTokens, nestedTokens] = transformTokens(model, monaco.editor.tokenize(model.getValue(), model.getLanguageId()))
+  const token = locateToken(nestedTokens, position.lineNumber - 1, position.column - 2)
+
+  if (token) {
+    switch (token.shortType) {
+      case TokenType.QUOTE_OPEN:
+        return { suggestions: [] }
+      case TokenType.STR_LITERAL:
+      case TokenType.STR_ESCAPE:
+      case TokenType.STR_INVALID_ESCAPE: {
+        if (props.provideRhsValueCompletion) {
+          const [rhsValueRange, rhsValueFirstTokenIndex] = scanTokens(model, flatTokens, token.flatIndex, (t) =>
+            !(t.shortType === TokenType.STR_LITERAL || t.shortType === TokenType.STR_ESCAPE || t.shortType === TokenType.STR_INVALID_ESCAPE),
+          )
+          if (rhsValueRange) {
+            const rhsValueValue = model.getValueInRange(rhsValueRange)
+            const lhsIdentTokenIndex = locateLhsIdent(flatTokens, rhsValueFirstTokenIndex)
+            if (lhsIdentTokenIndex >= 0) {
+              const lhsIdentRange = getRangeFromTokens(model, flatTokens, lhsIdentTokenIndex, lhsIdentTokenIndex + 1)
+              const lhsIdentValue = model.getValueInRange(lhsIdentRange)
+              const completion = await props.provideRhsValueCompletion(lhsIdentValue, rhsValueValue, lhsIdentRange, rhsValueRange)
+              if (completion) {
+                return completion
+              }
+            }
+          }
+        }
+        break
+      }
+      case TokenType.IDENT: {
+        const identRange = getRangeFromTokens(model, flatTokens, token.flatIndex, token.flatIndex + 1)
+        return {
+          suggestions: [
+            ...flatSchemaProperties.value.map((item) => ({
+              label: item.property,
+              kind: monaco.languages.CompletionItemKind.Property,
+              detail: item.kind,
+              documentation: item.documentation,
+              insertText: item.property.replace(/\*/g, ''),
+              range: identRange,
+            })),
+            ...(props.schema.functions?.map((func) => ({
+              label: func,
+              kind: monaco.languages.CompletionItemKind.Function,
+              insertText: `${func}($${1})`,
+              insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
+              range: identRange,
+            })) ?? []),
+          ],
+        }
+      }
+      default:
+        break
+    }
+  }
+
+  const range = new monaco.Range(position.lineNumber, position.column, position.lineNumber, position.column)
+  return {
+    suggestions: [
+      ...flatSchemaProperties.value.map((item) => ({
+        label: item.property,
+        kind: monaco.languages.CompletionItemKind.Property,
+        detail: item.kind,
+        documentation: item.documentation,
+        insertText: item.property.replace(/\*/g, ''),
+        range,
+      })),
+      ...(props.schema.functions?.map((func) => ({
+        label: func,
+        kind: monaco.languages.CompletionItemKind.Function,
+        insertText: `${func}($${1})`,
+        insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
+        range,
+      })) ?? []),
+    ],
+  }
 }
 
 onMounted(() => {
@@ -79,6 +180,7 @@ onMounted(() => {
     scrollBeyondLastLine: false,
     theme,
     value: expression.value,
+    maxTokenizationLineLength: 1000,
     ...props.editorOptions,
   })
 
@@ -89,7 +191,27 @@ onMounted(() => {
       ?.widget?.value._setDetailsVisible(true)
   }
   editor.onDidChangeModelContent(() => {
-    const value = editor!.getValue()
+    const model = editor!.getModel()!
+    const value = model.getValue()!
+
+    if (props.provideRhsValueCompletion) {
+      const position = editor!.getPosition()
+      if (position) {
+        const [, nestedTokens] = transformTokens(model, monaco.editor.tokenize(value, model.getLanguageId()))
+        const token = locateToken(nestedTokens, position.lineNumber - 1, position.column - 2)
+        switch (token?.shortType) {
+          case TokenType.STR_LITERAL:
+          case TokenType.STR_ESCAPE:
+          case TokenType.STR_INVALID_ESCAPE:
+            editor!.getContribution<Record<string, any> & Monaco.editor.IEditorContribution>('editor.contrib.suggestController')
+              ?.triggerSuggest()
+            break
+          default:
+            break
+        }
+      }
+    }
+
     expression.value = value
   })
 
@@ -98,13 +220,15 @@ onMounted(() => {
   if (props.inactiveUntilFocused) {
     editor.onDidFocusEditorWidget(() => {
       if (!isParsingActive.value) {
-        registerSchema(props.schema)
+        const { languageId } = registerLanguage(buildLanguageId(props.schema), provideCompletionItems)
+        monaco.editor.setModelLanguage(editorModel, languageId)
         isParsingActive.value = true
         parseResult.value = parse(expression.value, createSchema(props.schema.definition))
       }
     })
   } else {
-    registerSchema(props.schema)
+    const { languageId } = registerLanguage(buildLanguageId(props.schema), provideCompletionItems)
+    monaco.editor.setModelLanguage(editorModel, languageId)
     isParsingActive.value = true
     parseResult.value = parse(expression.value, createSchema(props.schema.definition))
   }
@@ -114,13 +238,6 @@ onBeforeUnmount(() => {
   editor?.dispose()
 })
 
-const schema = computed(() => createSchema(props.schema.definition))
-
-watch(() => props.schema, (newSchema) => {
-  const { languageId } = registerLanguage(newSchema)
-  monaco.editor.setModelLanguage(editorModel, languageId)
-}, { deep: true })
-
 watch(expression, (newExpression) => {
   if (!isParsingActive.value) {
     isParsingActive.value = true

packages/core/expressions/src/index.ts

@@ -3,6 +3,7 @@ import RouterPlaygroundModal from './components/RouterPlaygroundModal.vue'
 
 export * as Atc from '@kong/atc-router'
 export * from './schema'
+export * from './types'
 export { ExpressionsEditor, RouterPlaygroundModal }
 
 declare const asyncInit: Promise<any>