diff --git a/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt b/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt index edee5bee3..7d536ba5a 100644 --- a/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt +++ b/tokenizer/src/nativeMain/kotlin/com/xebia/functional/tokenizer/regex.kt @@ -67,8 +67,8 @@ actual val o200k_base_regex: Regex get() = Regex( listOf( - """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]*[$lowerCaseAndOtherLetters]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", - """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]+[$lowerCaseAndOtherLetters]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", + """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]*[$lowerCaseAndOtherLetters]+('s|'t|'re|'ve|'m|'ll|'d)?""", + """[^\r\n\$alphaChars]?[$upperCaseAndOtherLetters]+[$lowerCaseAndOtherLetters]*('s|'t|'re|'ve|'m|'ll|'d)?""", """[$decimalNumbers]{1,3}""", """?[^\s\$alphaChars]+[\r\n/]*""", """\s*[\r\n]+""",