From 98c202a549aba583aeb7796af37d4cdf0b11eb86 Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Sun, 6 Oct 2024 10:49:35 +1000 Subject: [PATCH] Parse docstrings within structs as `K"doc"` (#511) Julia's ecosystem (including Base.Docs and flisp lowering) assumes that strings within `struct` definitions are per-field docstrings, but the flisp parser doesn't handle these - they are only recognized when the struct itself has a docstring and are processed by the `@doc` macro recursing into the struct's internals. For example, the following doesn't result in any docs attached to `A`. ```julia struct A "x_docs" x "y_docs" y end ``` This change adds `K"doc"` node parsing to the insides of a struct, making the semantics clearer in the parser tree and making it possible to address this problems in the future within JuliaLowering. Also ensure that the `Expr` form is unaffected by this change. --- src/expr.jl | 13 +++++++++++++ src/parser.jl | 8 ++++---- test/expr.jl | 3 +++ test/parser.jl | 1 + 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/expr.jl b/src/expr.jl index 265fd022..b436e744 100644 --- a/src/expr.jl +++ b/src/expr.jl @@ -466,6 +466,19 @@ function _internal_node_to_Expr(source, srcrange, head, childranges, childheads, headsym = :call pushfirst!(args, :*) elseif k == K"struct" + @assert args[2].head == :block + orig_fields = args[2].args + fields = Expr(:block) + for field in orig_fields + if @isexpr(field, :macrocall) && field.args[1] == GlobalRef(Core, Symbol("@doc")) + # @doc macro calls don't occur within structs, in Expr form. + push!(fields.args, field.args[3]) + push!(fields.args, field.args[4]) + else + push!(fields.args, field) + end + end + args[2] = fields pushfirst!(args, has_flags(head, MUTABLE_FLAG)) elseif k == K"importpath" headsym = :. diff --git a/src/parser.jl b/src/parser.jl index 7502eb7a..0cd65f7a 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -536,9 +536,9 @@ end # Parse docstrings attached by a space or single newline # # flisp: parse-docstring -function parse_docstring(ps::ParseState) +function parse_docstring(ps::ParseState, down=parse_eq) mark = position(ps) - parse_eq(ps) + down(ps) if peek_behind(ps).kind == K"string" is_doc = true k = peek(ps) @@ -563,7 +563,7 @@ function parse_docstring(ps::ParseState) # """\n doc\n """ foo ==> (doc (string-s "doc\n") foo) end if is_doc - parse_eq(ps) + down(ps) emit(ps, mark, K"doc") end end @@ -1947,7 +1947,7 @@ function parse_resword(ps::ParseState) @check peek(ps) == K"struct" bump(ps, TRIVIA_FLAG) parse_subtype_spec(ps) - parse_block(ps, parse_struct_field) + parse_block(ps, ps1->parse_docstring(ps1, parse_struct_field)) bump_closing_token(ps, K"end") emit(ps, mark, K"struct", is_mut ? MUTABLE_FLAG : EMPTY_FLAGS) elseif word == K"primitive" diff --git a/test/expr.jl b/test/expr.jl index cf9b881d..9361937f 100644 --- a/test/expr.jl +++ b/test/expr.jl @@ -727,6 +727,9 @@ Expr(:block, LineNumberNode(2), :a, LineNumberNode(3), :b)) @test parsestmt("struct A const a end", version=v"1.8") == Expr(:struct, false, :A, Expr(:block, LineNumberNode(1), Expr(:const, :a))) + + @test parsestmt("struct A \n \"doc\" \n a end") == + Expr(:struct, false, :A, Expr(:block, LineNumberNode(2), "doc", :a)) end @testset "export" begin diff --git a/test/parser.jl b/test/parser.jl index 37c82bc8..e6115ad4 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -502,6 +502,7 @@ tests = [ # struct "struct A <: B \n a::X \n end" => "(struct (<: A B) (block (::-i a X)))" "struct A \n a \n b \n end" => "(struct A (block a b))" + "struct A \n \"doca\" \n a \n \"docb\" \n b \n end" => "(struct A (block (doc (string \"doca\") a) (doc (string \"docb\") b)))" "mutable struct A end" => "(struct-mut A (block))" ((v=v"1.8",), "struct A const a end") => "(struct A (block (const a)))" ((v=v"1.7",), "struct A const a end") => "(struct A (block (error (const a))))"