From 86be7ff32f79b7b34bc94ce6c2a18a7db74ccd9c Mon Sep 17 00:00:00 2001 From: mmatera Date: Mon, 25 Nov 2024 10:45:09 -0300 Subject: [PATCH] fix #94 --- mathics_scanner/tokeniser.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 2559599..f276d3e 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -470,12 +470,29 @@ def _skip_blank(self): if self.pos >= len(self.code): if comment: try: + # Since we are inside a comment, + # we can just do + # ``` + # self.prescanner.incomplete() + # self.code = self.prescanner.input_line + # ``` + # avoiding the call to the method + # `Prescanner.replace_escape_sequences()` + # which have issues with escape characters... self.incomplete() except ValueError: - # Funny symbols like | in comments can cause a ValueError. - # Until we have a better fix -- like noting we are inside a - # comment and should not try to substitute symbols -- ignore. - pass + # `incomplete` tries to parse substrings like `\|AAAAA` + # that can be interpreted as a character reference. + # To do that, it tries to get the + # new line using the method + # `Prescanner.replace_escape_sequences()` + # Inside a comment, the special meaning of escape sequences + # like `\|` should not be taken into account. + # + # In case of error, just let's pick the code + # from the `input_line` attribute of + # prescanner: + self.code = self.prescanner.input_line else: break if comment: