Add option to preserve comments when parsing templates

CHANGES.rst

@@ -9,6 +9,7 @@ Unreleased
 -   Use modern packaging metadata with ``pyproject.toml`` instead of ``setup.cfg``.
     :pr:`1793`
 -   Use ``flit_core`` instead of ``setuptools`` as build backend.
+-   Preserve comments in ASTs when parsing templates with ``Environment.parse``. :pr:`2037`
 
 
 Version 3.1.5

src/jinja2/lexer.py

@@ -146,17 +146,7 @@
     f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})"
 )
 
-ignored_tokens = frozenset(
-    [
-        TOKEN_COMMENT_BEGIN,
-        TOKEN_COMMENT,
-        TOKEN_COMMENT_END,
-        TOKEN_WHITESPACE,
-        TOKEN_LINECOMMENT_BEGIN,
-        TOKEN_LINECOMMENT_END,
-        TOKEN_LINECOMMENT,
-    ]
-)
+ignored_tokens = frozenset([TOKEN_WHITESPACE])
 ignore_if_empty = frozenset(
     [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT]
 )
@@ -631,6 +621,10 @@ def wrap(
                 token = TOKEN_BLOCK_BEGIN
             elif token == TOKEN_LINESTATEMENT_END:
                 token = TOKEN_BLOCK_END
+            elif token == TOKEN_LINECOMMENT_BEGIN:
+                token = TOKEN_COMMENT_BEGIN
+            elif token == TOKEN_LINECOMMENT_END:
+                token = TOKEN_COMMENT_END
             # we are not interested in those tokens in the parser
             elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
                 continue

src/jinja2/nodes.py

@@ -715,6 +715,13 @@ def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any:
         return self.expr2.as_const(eval_ctx)
 
 
+class Comment(Stmt):
+    """A template comment."""
+
+    fields = ("data",)
+    data: str
+
+
 def args_as_const(
     node: t.Union["_FilterTestCommon", "Call"], eval_ctx: t.Optional[EvalContext]
 ) -> t.Tuple[t.List[t.Any], t.Dict[t.Any, t.Any]]:

src/jinja2/parser.py

@@ -315,10 +315,13 @@ def parse_block(self) -> nodes.Block:
         # with whitespace data
         if node.required:
             for body_node in node.body:
-                if not isinstance(body_node, nodes.Output) or any(
-                    not isinstance(output_node, nodes.TemplateData)
-                    or not output_node.data.isspace()
-                    for output_node in body_node.nodes
+                if not isinstance(body_node, (nodes.Output, nodes.Comment)) or (
+                    isinstance(body_node, nodes.Output)
+                    and any(
+                        not isinstance(output_node, nodes.TemplateData)
+                        or not output_node.data.isspace()
+                        for output_node in body_node.nodes
+                    )
                 ):
                     self.fail("Required blocks can only contain comments or whitespace")
 
@@ -1025,6 +1028,11 @@ def flush_data() -> None:
                     else:
                         body.append(rv)
                     self.stream.expect("block_end")
+                elif token.type == "comment_begin":
+                    flush_data()
+                    next(self.stream)
+                    body.append(nodes.Comment(next(self.stream).value))
+                    self.stream.expect("comment_end")
                 else:
                     raise AssertionError("internal parsing error")
 

tests/test_lexnparse.py

@@ -314,6 +314,19 @@ def assert_error(code, expected):
         )
         assert_error("{% unknown_tag %}", "Encountered unknown tag 'unknown_tag'.")
 
+    def test_comment_preservation(self, env):
+        ast = env.parse("{# foo #}{{ bar }}")
+        assert len(ast.body) == 2
+        assert isinstance(ast.body[0], nodes.Comment)
+        assert ast.body[0].data == " foo "
+
+    def test_line_comment_preservation(self, env):
+        env = Environment(line_comment_prefix="#")
+        ast = env.parse("# foo\n{{ bar }}")
+        assert len(ast.body) == 2
+        assert isinstance(ast.body[0], nodes.Comment)
+        assert ast.body[0].data == " foo"
+
 
 class TestSyntax:
     def test_call(self, env):