diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 2559599..442b434 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -472,10 +472,20 @@ def _skip_blank(self): try: self.incomplete() except ValueError: - # Funny symbols like | in comments can cause a ValueError. - # Until we have a better fix -- like noting we are inside a - # comment and should not try to substitute symbols -- ignore. - pass + # `incomplete` tries to parse substrings like `\|AAAAA` + # that can be interpreted as a character reference. + # To do that, it tries to get the + # new line using the method + # `Prescanner.replace_escape_sequences()` + # Inside a comment, the special meaning of escape sequences + # like `\|` should not be taken into account. + # + # In case of error, just let's pick the code + # from the `input_line` attribute of + # prescanner: + self.code = self.prescanner.input_line + # TODO: handle the corner case where the rest of the line + # include escaped sequences, out of the comment. else: break if comment: