fix: alpha numerics on rest of places

xebia-functional · May 30, 2024 · 1110c85 · 1110c85
1 parent f1ec76a
commit 1110c85
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt b/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt
@@ -67,8 +67,8 @@ actual val o200k_base_regex: Regex
   get() =
     Regex(
       listOf(
-          """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]*[$lowerCaseAndOtherLetters]+('s|'t|'re|'ve|'m|'ll|'d)?""",
-          """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]+[$lowerCaseAndOtherLetters]*('s|'t|'re|'ve|'m|'ll|'d)?""",
+          """[^\r\n\$alphaNumerics]?[$upperCaseAndOtherLetters]*[$lowerCaseAndOtherLetters]+('s|'t|'re|'ve|'m|'ll|'d)?""",
+          """[^\r\n\$alphaNumerics]?[$upperCaseAndOtherLetters]+[$lowerCaseAndOtherLetters]*('s|'t|'re|'ve|'m|'ll|'d)?""",
           """[$decimalNumbers]{1,3}""",
           """ ?[^\s\$alphaNumerics]+[\r\n/]*""",
           """\s*[\r\n]+""",