fix(message-compiler): cannot resolve none-identifier characters at l…

…inked key (#1813) * fix(message-compiler): cannot resolve none-identifier characters at linked key * fix: remove comment
intlify · Apr 20, 2024 · 8934933 · 8934933
1 parent a1d857e
commit 8934933
Show file tree

Hide file tree

Showing 6 changed files with 129 additions and 71 deletions.
diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts
@@ -321,7 +321,7 @@ export function createTokenizer(
         return fn()
       } else {
         // other characters
-        return isIdentifierStart(ch)
+        return isTextStart(scnr, false)
       }
     }
 
@@ -680,7 +680,7 @@ export function createTokenizer(
   }
 
   function readLinkedRefer(scnr: Scanner): string {
-    const fn = (detect = false, buf: string): string => {
+    const fn = (buf: string): string => {
       const ch = scnr.currentChar()
       if (
         ch === TokenChars.BraceLeft ||
@@ -697,15 +697,15 @@ export function createTokenizer(
       } else if (ch === NEW_LINE || ch === DOT) {
         buf += ch
         scnr.next()
-        return fn(detect, buf)
+        return fn(buf)
       } else {
         buf += ch
         scnr.next()
-        return fn(true, buf)
+        return fn(buf)
       }
     }
 
-    return fn(false, '')
+    return fn('')
   }
 
   function readPlural(scnr: Scanner): string {

diff --git a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap
@@ -525,28 +525,6 @@ exports[`parse > linked key paren error with modifier: "@.lower:(foo)" 1`] = `
 }
 `;
 
-exports[`parse > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
-[
-  {
-    "code": 13,
-    "domain": "parser",
-    "location": {
-      "end": {
-        "column": 14,
-        "line": 1,
-        "offset": 13,
-      },
-      "start": {
-        "column": 8,
-        "line": 1,
-        "offset": 7,
-      },
-    },
-    "message": "Unexpected empty linked key",
-  },
-]
-`;
-
 exports[`parse > linked key paren error: "@:(foo)" 1`] = `
 {
   "body": {
@@ -640,28 +618,6 @@ exports[`parse > linked key paren error: "@:(foo)" 1`] = `
 }
 `;
 
-exports[`parse > linked key paren error: "@:(foo)" errors 1`] = `
-[
-  {
-    "code": 13,
-    "domain": "parser",
-    "location": {
-      "end": {
-        "column": 8,
-        "line": 1,
-        "offset": 7,
-      },
-      "start": {
-        "column": 2,
-        "line": 1,
-        "offset": 1,
-      },
-    },
-    "message": "Unexpected empty linked key",
-  },
-]
-`;
-
 exports[`parse > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
 {
   "body": {
@@ -2024,16 +1980,6 @@ exports[`parser options > location disable > linked key paren error with modifie
 }
 `;
 
-exports[`parser options > location disable > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
-[
-  {
-    "code": 13,
-    "domain": "parser",
-    "message": "Unexpected empty linked key",
-  },
-]
-`;
-
 exports[`parser options > location disable > linked key paren error: "@:(foo)" 1`] = `
 {
   "body": {
@@ -2056,16 +2002,6 @@ exports[`parser options > location disable > linked key paren error: "@:(foo)" 1
 }
 `;
 
-exports[`parser options > location disable > linked key paren error: "@:(foo)" errors 1`] = `
-[
-  {
-    "code": 13,
-    "domain": "parser",
-    "message": "Unexpected empty linked key",
-  },
-]
-`;
-
 exports[`parser options > location disable > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
 {
   "body": {

diff --git a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap
@@ -4370,6 +4370,90 @@ exports[`token analysis > "hi @:name !" tokens 1`] = `
 ]
 `;
 
+exports[`token analysis > "hi @:名前" tokens 1`] = `
+[
+  {
+    "loc": {
+      "end": {
+        "column": 4,
+        "line": 1,
+        "offset": 3,
+      },
+      "start": {
+        "column": 1,
+        "line": 1,
+        "offset": 0,
+      },
+    },
+    "type": 0,
+    "value": "hi ",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 5,
+        "line": 1,
+        "offset": 4,
+      },
+      "start": {
+        "column": 4,
+        "line": 1,
+        "offset": 3,
+      },
+    },
+    "type": 8,
+    "value": "@",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 6,
+        "line": 1,
+        "offset": 5,
+      },
+      "start": {
+        "column": 5,
+        "line": 1,
+        "offset": 4,
+      },
+    },
+    "type": 10,
+    "value": ":",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 8,
+        "line": 1,
+        "offset": 7,
+      },
+      "start": {
+        "column": 6,
+        "line": 1,
+        "offset": 5,
+      },
+    },
+    "type": 11,
+    "value": "名前",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 8,
+        "line": 1,
+        "offset": 7,
+      },
+      "start": {
+        "column": 8,
+        "line": 1,
+        "offset": 7,
+      },
+    },
+    "type": 14,
+  },
+]
+`;
+
 exports[`token analysis > "hi @\\n. upper\\n:  {'name'}\\n !" errors 1`] = `
 [
   {
@@ -10520,6 +10604,30 @@ exports[`tokenize options: location disable > "hi @:name !" tokens 1`] = `
 ]
 `;
 
+exports[`tokenize options: location disable > "hi @:名前" tokens 1`] = `
+[
+  {
+    "type": 0,
+    "value": "hi ",
+  },
+  {
+    "type": 8,
+    "value": "@",
+  },
+  {
+    "type": 10,
+    "value": ":",
+  },
+  {
+    "type": 11,
+    "value": "名前",
+  },
+  {
+    "type": 14,
+  },
+]
+`;
+
 exports[`tokenize options: location disable > "hi @\\n. upper\\n:  {'name'}\\n !" errors 1`] = `
 [
   {

diff --git a/packages/message-compiler/test/tokenizer.test.ts b/packages/message-compiler/test/tokenizer.test.ts
@@ -80,6 +80,7 @@ const CASES = [
   `hi @.upper\n{name} !`,
   `hi @.upper {name} !`,
   `hi @:\nname !`,
+  `hi @:名前`,
   `hi @: {'name'} !`,
   `hi @\n. upper\n:  {'name'}\n !`,
   ` | | |`,

diff --git a/packages/vue-i18n-core/test/issues.test.ts b/packages/vue-i18n-core/test/issues.test.ts
@@ -1437,3 +1437,16 @@ test('#1796', async () => {
     })
   ).toEqual('My message with hello world.')
 })
+
+test('#1809', async () => {
+  const i18n = createI18n({
+    locale: 'en',
+    messages: {
+      en: {
+        hi: 'hi @:名前',
+        名前: 'kazupon'
+      }
+    }
+  })
+  expect(i18n.global.t('hi')).toEqual('hi kazupon')
+})
diff --git a/spec/syntax.ebnf b/spec/syntax.ebnf
@@ -1,5 +1,5 @@
 (*
- * Inltify message syntax v0.3
+ * Inltify message syntax v0.4.0
  * (vue-i18n compatible)
  *)
 
@@ -18,7 +18,7 @@ Named ::= Modulo? "{" Space? (NamedIdentifier) Space? "}";
 List ::= "{" Space? (NumberLiteral) Space? "}";
 Linked ::= "@" (LinkedModifier)? LinkedDelimiter LinkedRefer;
 LinkedRefer ::= LinkedKey | Placeholder;
-LinkedKey ::= Identifier;
+LinkedKey ::= Text;
 LinkedModifier ::= LinkedDot Identifier;
 LinkedDelimiter ::= ":";
 LinkedDot ::= ".";