From 46e45dd64b072621ad3b3bd79dcfcc0070dcaec8 Mon Sep 17 00:00:00 2001 From: ascandone Date: Sat, 24 Aug 2024 22:14:25 +0200 Subject: [PATCH 1/5] parsing comments --- Kestrel.g4 | 2 +- src/parser/__snapshots__/parser.test.ts.snap | 140 +++++++++++++++++++ src/parser/antlr/Kestrel.interp | 2 +- src/parser/antlr/Kestrel.tokens | 2 +- src/parser/antlr/KestrelLexer.interp | 4 +- src/parser/antlr/KestrelLexer.tokens | 2 +- src/parser/antlr/KestrelLexer.ts | 6 +- src/parser/antlr/KestrelParser.ts | 4 +- src/parser/ast.ts | 2 + src/parser/parser.test.ts | 2 + src/parser/parser.ts | 19 ++- 11 files changed, 172 insertions(+), 13 deletions(-) diff --git a/Kestrel.g4 b/Kestrel.g4 index 3103e7a8..d37775f5 100644 --- a/Kestrel.g4 +++ b/Kestrel.g4 @@ -4,7 +4,7 @@ SLASH_4: '////'; SLASH_3: '///'; SLASH_2: '//'; -LineComment: SLASH_2 ~[\r\n]* -> channel(HIDDEN); +LINE_COMMENT: SLASH_2 ~[\r\n]* -> channel(HIDDEN); EXPOSING_NESTED: '(' '..' ')'; INFIX_ID: '(' INFIX_CHAR+ ')'; diff --git a/src/parser/__snapshots__/parser.test.ts.snap b/src/parser/__snapshots__/parser.test.ts.snap index 79575798..1def758e 100644 --- a/src/parser/__snapshots__/parser.test.ts.snap +++ b/src/parser/__snapshots__/parser.test.ts.snap @@ -112,6 +112,7 @@ exports[`+ is left-associative 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -169,6 +170,7 @@ exports[`Comments > doc comments 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -177,6 +179,7 @@ exports[`Comments > doc comments on extern types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "docComment": " first line @@ -264,6 +267,7 @@ exports[`Comments > doc comments on externs 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -272,6 +276,7 @@ exports[`Comments > doc comments on types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "docComment": " first line @@ -394,6 +399,7 @@ exports[`Comments > doc comments with many declrs 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -448,6 +454,7 @@ exports[`Comments > moduledoc comments 1`] = ` }, ], "imports": [], + "lineComments": [], "moduleDoc": " Module level comment Second line ", @@ -505,6 +512,7 @@ exports[`Decorators > inline decorator 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -584,6 +592,7 @@ exports[`Fault tolerance > dot notation in struct 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -638,6 +647,7 @@ exports[`Fault tolerance > faulty decl 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -719,6 +729,7 @@ exports[`Fault tolerance > faulty infix 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -769,6 +780,7 @@ exports[`Fault tolerance > missing expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -819,6 +831,7 @@ exports[`Fault tolerance > missing string lit termination 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -883,6 +896,7 @@ exports[`extern bindings > let decls 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -947,6 +961,7 @@ exports[`extern bindings > let decls defining infix operators 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -955,6 +970,7 @@ exports[`extern bindings > types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -1026,6 +1042,21 @@ exports[`ignoring comments 1`] = ` }, ], "imports": [], + "lineComments": [ + { + "comment": "// ignoring comments", + "range": { + "end": { + "character": 24, + "line": 1, + }, + "start": { + "character": 4, + "line": 1, + }, + }, + }, + ], "typeDeclarations": [], } `; @@ -1078,6 +1109,7 @@ exports[`imports > identifiers can be qualified 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1101,6 +1133,7 @@ exports[`imports > import nested modules 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1124,6 +1157,7 @@ exports[`imports > import single module 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1178,6 +1212,7 @@ exports[`imports > parse pub modifier 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1186,6 +1221,7 @@ exports[`imports > parse pub modifier on extern types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -1267,6 +1303,7 @@ exports[`imports > parse pub modifier on extern values 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1275,6 +1312,7 @@ exports[`imports > parse pub modifier on types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -1301,6 +1339,7 @@ exports[`imports > parse pub(..) modifier on types 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -1371,6 +1410,7 @@ exports[`imports > type constructors can be qualified 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1436,6 +1476,7 @@ exports[`imports > type defs can be qualified 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1474,6 +1515,7 @@ exports[`imports > unqualified import of a value 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1512,6 +1554,7 @@ exports[`imports > unqualified import of an infix value 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1551,6 +1594,7 @@ exports[`imports > unqualified import of types (non-opaque) 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1605,6 +1649,7 @@ exports[`imports > unqualified import of types 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1671,6 +1716,7 @@ exports[`imports > unqualified import of values 1`] = ` }, }, ], + "lineComments": [], "typeDeclarations": [], } `; @@ -1840,6 +1886,7 @@ exports[`it should be possible to mix pipe with infix 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -1948,6 +1995,7 @@ exports[`monadic let syntax sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2056,6 +2104,7 @@ exports[`monadic let syntax sugar should handle qualified names 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2141,6 +2190,7 @@ exports[`parse * expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2257,6 +2307,7 @@ exports[`parse + and * prec 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2373,6 +2424,7 @@ exports[`parse + and * prec with parens 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2458,6 +2510,7 @@ exports[`parse + expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2543,6 +2596,7 @@ exports[`parse +. expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2653,6 +2707,7 @@ exports[`parse <= and && prec 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2738,6 +2793,7 @@ exports[`parse - expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2820,6 +2876,7 @@ exports[`parse appl with 1 arg 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -2932,6 +2989,7 @@ exports[`parse appl with 3 args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3047,6 +3105,7 @@ exports[`parse appl with nested expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3131,6 +3190,7 @@ exports[`parse appl with nested parens 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3197,6 +3257,7 @@ exports[`parse appl with no args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3294,6 +3355,7 @@ exports[`parse appl with trailing comma 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3348,6 +3410,7 @@ exports[`parse block with no let 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3402,6 +3465,7 @@ exports[`parse chars 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3544,6 +3608,7 @@ exports[`parse cons operator is right-associative 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3641,6 +3706,7 @@ exports[`parse conslist sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3692,6 +3758,7 @@ exports[`parse empty list sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3746,6 +3813,7 @@ exports[`parse empty strings 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3800,6 +3868,7 @@ exports[`parse float 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3883,6 +3952,7 @@ exports[`parse fn with 1 arg 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -3980,6 +4050,7 @@ exports[`parse fn with 2 args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4092,6 +4163,7 @@ exports[`parse fn with let 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4160,6 +4232,7 @@ exports[`parse fn with no args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4212,6 +4285,7 @@ exports[`parse ident 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4311,6 +4385,7 @@ exports[`parse if expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4452,6 +4527,7 @@ exports[`parse if expr with a let expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4550,6 +4626,7 @@ exports[`parse let block with one let 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4692,6 +4769,7 @@ exports[`parse let block with two let stmts 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4795,6 +4873,7 @@ exports[`parse list sugar with many values 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4849,6 +4928,7 @@ exports[`parse nonempty strings 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4918,6 +4998,7 @@ exports[`parse singleton list sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -4972,6 +5053,7 @@ exports[`parse strings with escaped quotes 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5026,6 +5108,7 @@ exports[`parse strings with newlines 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5127,6 +5210,7 @@ exports[`parse tuple sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5208,6 +5292,7 @@ exports[`parse unary ! expr 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5306,6 +5391,7 @@ exports[`pattern matching > binding with identifier 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5372,6 +5458,7 @@ exports[`pattern matching > empty match expression 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5473,6 +5560,7 @@ exports[`pattern matching > matching char literals 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5602,6 +5690,7 @@ exports[`pattern matching > matching cons literal (syntax sugar) 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5764,6 +5853,7 @@ exports[`pattern matching > matching cons literal is right assoc 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -5926,6 +6016,7 @@ exports[`pattern matching > matching cons nested in tuple 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6026,6 +6117,7 @@ exports[`pattern matching > matching constructor with no args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6141,6 +6233,7 @@ exports[`pattern matching > matching constructor with one arg 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6242,6 +6335,7 @@ exports[`pattern matching > matching float literals 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6343,6 +6437,7 @@ exports[`pattern matching > matching int literals 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6472,6 +6567,7 @@ exports[`pattern matching > matching many clauses, with trailing comma 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6601,6 +6697,7 @@ exports[`pattern matching > matching many clauses, without trailing comma 1`] = }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6715,6 +6812,7 @@ exports[`pattern matching > matching pattern in fn param 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -6871,6 +6969,7 @@ exports[`pattern matching > matching pattern in let 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7041,6 +7140,7 @@ exports[`pattern matching > matching pattern in let# 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7172,6 +7272,7 @@ exports[`pattern matching > matching qualified constructors 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7273,6 +7374,7 @@ exports[`pattern matching > matching str literals 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7402,6 +7504,7 @@ exports[`pattern matching > matching tuples literal (syntax sugar) 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7527,6 +7630,7 @@ exports[`pipe syntax sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7663,6 +7767,7 @@ exports[`pipe syntax sugar should be chainable 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7788,6 +7893,7 @@ exports[`pipe syntax sugar should handle qualified names 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -7938,6 +8044,7 @@ exports[`structs > construct fields with many fields 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8003,6 +8110,7 @@ exports[`structs > construct fields with no fields 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8011,6 +8119,7 @@ exports[`structs > doc comments 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "docComment": " example docs @@ -8039,6 +8148,7 @@ exports[`structs > empty struct 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [], @@ -8065,6 +8175,7 @@ exports[`structs > field 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [ @@ -8191,6 +8302,7 @@ exports[`structs > field access 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8199,6 +8311,7 @@ exports[`structs > many fields 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [ @@ -8280,6 +8393,7 @@ exports[`structs > pub modifier 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [], @@ -8306,6 +8420,7 @@ exports[`structs > pub(..) modifier 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [], @@ -8403,6 +8518,7 @@ exports[`structs > qualified field access 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8411,6 +8527,7 @@ exports[`structs > type params 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "fields": [], @@ -8620,6 +8737,7 @@ exports[`structs > update a struct 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8691,6 +8809,7 @@ exports[`traits > parses many traits in a polytype for the same tvar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8798,6 +8917,7 @@ exports[`traits > parses traits for many vars 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8868,6 +8988,7 @@ exports[`traits > parses traits in a polytype 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -8876,6 +8997,7 @@ exports[`type declarations > many type params 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -8942,6 +9064,7 @@ exports[`type declarations > single type param 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -8982,6 +9105,7 @@ exports[`type declarations > trailing comma after variants 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9037,6 +9161,7 @@ exports[`type declarations > type with a variant with complex args 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9140,6 +9265,7 @@ exports[`type declarations > type with a variant with many args 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9212,6 +9338,7 @@ exports[`type declarations > type with a variant with no args 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9253,6 +9380,7 @@ exports[`type declarations > type with a variant with one arg 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9310,6 +9438,7 @@ exports[`type declarations > type with many variants 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "T", @@ -9365,6 +9494,7 @@ exports[`type declarations > type with no variants 1`] = ` { "declarations": [], "imports": [], + "lineComments": [], "typeDeclarations": [ { "name": "Never", @@ -9510,6 +9640,7 @@ exports[`type hints > parses Fn type with args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -9606,6 +9737,7 @@ exports[`type hints > parses Fn type with no args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -9670,6 +9802,7 @@ exports[`type hints > parses a concrete type with no args as a type hint (extern }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -9752,6 +9885,7 @@ exports[`type hints > parses a concrete type with no args as a type hint (no whi }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -9834,6 +9968,7 @@ exports[`type hints > parses a concrete type with no args as a type hint 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -9932,6 +10067,7 @@ exports[`type hints > parses concrete type with 1 arg 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -10045,6 +10181,7 @@ exports[`type hints > parses concrete type with 2 args 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -10175,6 +10312,7 @@ exports[`type hints > parses tuple type syntax sugar 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -10256,6 +10394,7 @@ exports[`type hints > parses type vars hints 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; @@ -10336,6 +10475,7 @@ exports[`type hints > parses underscore type 1`] = ` }, ], "imports": [], + "lineComments": [], "typeDeclarations": [], } `; diff --git a/src/parser/antlr/Kestrel.interp b/src/parser/antlr/Kestrel.interp index 2637e638..16160885 100644 --- a/src/parser/antlr/Kestrel.interp +++ b/src/parser/antlr/Kestrel.interp @@ -123,7 +123,7 @@ null SLASH_4 SLASH_3 SLASH_2 -LineComment +LINE_COMMENT EXPOSING_NESTED INFIX_ID ID diff --git a/src/parser/antlr/Kestrel.tokens b/src/parser/antlr/Kestrel.tokens index 94d96506..873d97e0 100644 --- a/src/parser/antlr/Kestrel.tokens +++ b/src/parser/antlr/Kestrel.tokens @@ -51,7 +51,7 @@ T__49=50 SLASH_4=51 SLASH_3=52 SLASH_2=53 -LineComment=54 +LINE_COMMENT=54 EXPOSING_NESTED=55 INFIX_ID=56 ID=57 diff --git a/src/parser/antlr/KestrelLexer.interp b/src/parser/antlr/KestrelLexer.interp index 8231c04b..250c33a9 100644 --- a/src/parser/antlr/KestrelLexer.interp +++ b/src/parser/antlr/KestrelLexer.interp @@ -123,7 +123,7 @@ null SLASH_4 SLASH_3 SLASH_2 -LineComment +LINE_COMMENT EXPOSING_NESTED INFIX_ID ID @@ -192,7 +192,7 @@ T__49 SLASH_4 SLASH_3 SLASH_2 -LineComment +LINE_COMMENT EXPOSING_NESTED INFIX_ID ID diff --git a/src/parser/antlr/KestrelLexer.tokens b/src/parser/antlr/KestrelLexer.tokens index 94d96506..873d97e0 100644 --- a/src/parser/antlr/KestrelLexer.tokens +++ b/src/parser/antlr/KestrelLexer.tokens @@ -51,7 +51,7 @@ T__49=50 SLASH_4=51 SLASH_3=52 SLASH_2=53 -LineComment=54 +LINE_COMMENT=54 EXPOSING_NESTED=55 INFIX_ID=56 ID=57 diff --git a/src/parser/antlr/KestrelLexer.ts b/src/parser/antlr/KestrelLexer.ts index 9c51b236..86f5c9c1 100644 --- a/src/parser/antlr/KestrelLexer.ts +++ b/src/parser/antlr/KestrelLexer.ts @@ -66,7 +66,7 @@ export default class KestrelLexer extends Lexer { public static readonly SLASH_4 = 51; public static readonly SLASH_3 = 52; public static readonly SLASH_2 = 53; - public static readonly LineComment = 54; + public static readonly LINE_COMMENT = 54; public static readonly EXPOSING_NESTED = 55; public static readonly INFIX_ID = 56; public static readonly ID = 57; @@ -140,7 +140,7 @@ export default class KestrelLexer extends Lexer { null, "SLASH_4", "SLASH_3", "SLASH_2", - "LineComment", + "LINE_COMMENT", "EXPOSING_NESTED", "INFIX_ID", "ID", "TYPE_ID", @@ -159,7 +159,7 @@ export default class KestrelLexer extends Lexer { "T__25", "T__26", "T__27", "T__28", "T__29", "T__30", "T__31", "T__32", "T__33", "T__34", "T__35", "T__36", "T__37", "T__38", "T__39", "T__40", "T__41", "T__42", "T__43", "T__44", "T__45", "T__46", "T__47", "T__48", - "T__49", "SLASH_4", "SLASH_3", "SLASH_2", "LineComment", "EXPOSING_NESTED", + "T__49", "SLASH_4", "SLASH_3", "SLASH_2", "LINE_COMMENT", "EXPOSING_NESTED", "INFIX_ID", "ID", "TYPE_ID", "INT", "CHAR", "STRING", "DoubleStringCharacter", "FLOAT", "NEWLINE", "WS", "MODULEDOC_COMMENT_LINE", "DOC_COMMENT_LINE", "INFIX_CHAR", diff --git a/src/parser/antlr/KestrelParser.ts b/src/parser/antlr/KestrelParser.ts index 3c6e01e4..fb70be04 100644 --- a/src/parser/antlr/KestrelParser.ts +++ b/src/parser/antlr/KestrelParser.ts @@ -73,7 +73,7 @@ export default class KestrelParser extends Parser { public static readonly SLASH_4 = 51; public static readonly SLASH_3 = 52; public static readonly SLASH_2 = 53; - public static readonly LineComment = 54; + public static readonly LINE_COMMENT = 54; public static readonly EXPOSING_NESTED = 55; public static readonly INFIX_ID = 56; public static readonly ID = 57; @@ -173,7 +173,7 @@ export default class KestrelParser extends Parser { null, "SLASH_4", "SLASH_3", "SLASH_2", - "LineComment", + "LINE_COMMENT", "EXPOSING_NESTED", "INFIX_ID", "ID", "TYPE_ID", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 92735e40..7bf502c4 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -345,11 +345,13 @@ export type Import = RangeMeta & { exposing: Exposing[]; }; +export type LineComment = { comment: string } & RangeMeta; export type UntypedModule = { moduleDoc?: string; imports: UntypedImport[]; typeDeclarations: UntypedTypeDeclaration[]; declarations: UntypedDeclaration[]; + lineComments: LineComment[]; }; export type Position = { diff --git a/src/parser/parser.test.ts b/src/parser/parser.test.ts index b4f123cf..51ac9cc7 100644 --- a/src/parser/parser.test.ts +++ b/src/parser/parser.test.ts @@ -6,6 +6,7 @@ import { rangeOf } from "../typecheck/typedAst/__test__/utils"; test("parsing a declaration", () => { const src = "let x = 0"; expect(unsafeParse(src)).toEqual({ + lineComments: [], imports: [], typeDeclarations: [], declarations: [ @@ -31,6 +32,7 @@ test("parsing a declaration", () => { test("parsing two declarations", () => { const src = `let x = 0\nlet y = 1`; expect(unsafeParse(src)).toEqual({ + lineComments: [], imports: [], typeDeclarations: [], declarations: [ diff --git a/src/parser/parser.ts b/src/parser/parser.ts index a814f29d..19cd161c 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -51,6 +51,7 @@ import Parser, { } from "./antlr/KestrelParser"; import Visitor from "./antlr/KestrelVisitor"; import { + LineComment, Position, Range, TypeAst, @@ -63,6 +64,8 @@ import { UntypedTypeDeclaration, } from "./ast"; +const COMMENTS_CHANNEL = 1; + interface InfixExprContext extends ExprContext { _op: { text: string }; expr(nth: number): ExprContext; @@ -749,8 +752,19 @@ export function parse(input: string): ParseResult { lexer.removeErrorListeners(); lexer.addErrorListener(lexerErrorListener); - const tokens = new antlr4.CommonTokenStream(lexer); - const parser = new Parser(tokens); + const tkStream = new antlr4.CommonTokenStream(lexer); + const parser = new Parser(tkStream); + + tkStream.fill(); + + const lineComments = tkStream.tokens + .filter((tk) => tk.channel === COMMENTS_CHANNEL) + .map( + (tk): LineComment => ({ + comment: tk.text, + range: rangeOfTk(tk), + }), + ); const parsingErrorListener = new ParsingErrorListener(); parser.removeErrorListeners(); @@ -768,6 +782,7 @@ export function parse(input: string): ParseResult { .map((d) => new DeclarationVisitor().visit(d)); const parsed: UntypedModule = { + lineComments, ...(docs === "" ? {} : { moduleDoc: docs }), imports: declCtx.import__list().map((i): UntypedImport => { return { From f0623a760a66a3888aa0388ad651a19e445c8b37 Mon Sep 17 00:00:00 2001 From: ascandone Date: Sun, 25 Aug 2024 17:09:43 +0200 Subject: [PATCH 2/5] printing some comments --- src/formatter.test.ts | 60 ++++++++++++++++++++++++++++++- src/formatter.ts | 82 ++++++++++++++++++++++++++++++++++--------- src/parser/ast.ts | 2 +- 3 files changed, 125 insertions(+), 19 deletions(-) diff --git a/src/formatter.test.ts b/src/formatter.test.ts index 99435183..b07084b9 100644 --- a/src/formatter.test.ts +++ b/src/formatter.test.ts @@ -396,7 +396,65 @@ test("order between type declrs and declrs", () => { }); describe("comments", () => { - test.todo("regular comments"); + test("doc comments on declrs", () => { + expect(`let f = + // c1 + 0 + + // c2 + 1 +`).toBeFormatted(`let f = + // c1 + // c2 + 0 + 1 +`); + }); + + test("doc comments on declrs", () => { + expect(`let f = fn { + // c + 42 + } +`).toBeFormatted(`let f = fn { + // c + 42 +} +`); + }); + + test("doc comments in if expr", () => { + expect(`let f = if b { + // c + 42 +} else { + // d + 100 +} +`).toBeFormatted(`let f = + if b { + // c + 42 + } else { + // d + 100 + } + +`); + }); + + test.todo("doc comments in lists", () => { + expect(`let f = [ + 0, + // comment + 1, +] +`).toBeFormatted(`let f = [ + 0, + // comment + 1, +] + +`); + }); test("doc comments on declrs", () => { expect(`/// First line diff --git a/src/formatter.ts b/src/formatter.ts index a40d2d44..33460758 100644 --- a/src/formatter.ts +++ b/src/formatter.ts @@ -1,7 +1,9 @@ import { ConstLiteral, + LineComment, MatchPattern, PolyTypeAst, + RangeMeta, TypeAst, UntypedDeclaration, UntypedExpr, @@ -28,6 +30,23 @@ import { } from "./pretty"; import { gtEqPos } from "./typecheck/typedAst/common"; +let currentLineComments: LineComment[] = []; +function popComments(ast: RangeMeta): Doc[] { + const poppedComments: string[] = []; + // eslint-disable-next-line no-constant-condition + while (true) { + const comment = currentLineComments.at(-1); + + if (comment === undefined || comment.range.end.line >= ast.range.end.line) { + break; + } + + poppedComments.push(comment.comment); + currentLineComments.pop(); + } + return poppedComments.map((comment) => concat(text(comment), lines())); +} + const ORDERED_PREFIX_SYMBOLS = [["!"]]; const ORDERED_INFIX_SYMBOLS = [ @@ -114,7 +133,7 @@ function constToDoc(lit: ConstLiteral): Doc { } function indentWithSpaceBreak(docs: Doc[], unbroken?: string): Doc { - return concat( + return group( nest( // break_(""), @@ -148,6 +167,10 @@ function asBlock(isBlock: boolean, docs: Doc[]): Doc { return block_(...docs); } +function exprToDocWithComments(ast: UntypedExpr, block: boolean): Doc { + return concat(...popComments(ast), exprToDoc(ast, block)); +} + function exprToDoc(ast: UntypedExpr, block: boolean): Doc { switch (ast.type) { /* v8 ignore next 2 */ @@ -163,7 +186,10 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { sepBy( concat(text(","), break_()), ast.values.map((expr) => - exprToDoc(expr, expr.type !== "let" && expr.type !== "let#"), + exprToDocWithComments( + expr, + expr.type !== "let" && expr.type !== "let#", + ), ), ), ], @@ -308,7 +334,7 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { text("fn"), sepByString(",", params), text(" "), - block_(exprToDoc(ast.body, true)), + block_(exprToDocWithComments(ast.body, true)), ); } @@ -317,10 +343,10 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { text("if "), exprToDoc(ast.condition, false), text(" "), - block_(exprToDoc(ast.then, true)), + block_(exprToDocWithComments(ast.then, true)), text(" else "), - block_(exprToDoc(ast.else, true)), + block_(exprToDocWithComments(ast.else, true)), ); case "let#": { @@ -330,8 +356,8 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { const inner = concat( text(`let#${ns}${ast.mapper.name} `), patternToDoc(ast.pattern), - text(` = `), - exprToDoc(ast.value, false), + text(` =`), + declarationValueToDoc(ast.value), text(";"), break_(), exprToDoc(ast.body, true), @@ -348,8 +374,8 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { const inner = concat( text("let "), patternToDoc(ast.pattern), - text(" = "), - exprToDoc(ast.value, false), + text(" ="), + declarationValueToDoc(ast.value), text(";"), break_(), exprToDoc(ast.body, true), @@ -488,6 +514,32 @@ function handleDocComment(content: string, init = "///") { ); } +function declarationValueToDoc(expr: UntypedExpr): Doc { + const exprDoc = exprToDoc(expr, false); + + switch (expr.type) { + case "if": + return indentWithSpaceBreak([exprDoc]); + + default: { + const poppedComments = popComments(expr); + + if (poppedComments.length === 0) { + return concat(text(" "), exprDoc); + } + + return broken( + nest( + // + break_(), + ...poppedComments, + exprDoc, + ), + ); + } + } +} + function declToDoc(ast: UntypedDeclaration): Doc { const name = isInfix(ast.binding.name) || isPrefix(ast.binding.name) @@ -503,14 +555,7 @@ function declToDoc(ast: UntypedDeclaration): Doc { ast.typeHint === undefined ? nil : concat(text(": "), typeHintToDoc(ast.typeHint)), - ast.extern - ? nil - : concat( - text(" ="), - ["if"].includes(ast.value.type) - ? indentWithSpaceBreak([exprToDoc(ast.value, false)]) - : concat(text(" "), exprToDoc(ast.value, false)), - ), + ast.extern ? nil : concat(text(" ="), declarationValueToDoc(ast.value)), ); } @@ -631,6 +676,9 @@ export function formatExpr(expr: UntypedExpr): string { } export function format(ast: UntypedModule): string { + currentLineComments = [...(ast.lineComments ?? [])]; + currentLineComments.reverse(); + const importsDocs = ast.imports .sort((i1, i2) => (i1.ns > i2.ns ? 1 : -1)) .map(importToDoc) diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 7bf502c4..231f7974 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -351,7 +351,7 @@ export type UntypedModule = { imports: UntypedImport[]; typeDeclarations: UntypedTypeDeclaration[]; declarations: UntypedDeclaration[]; - lineComments: LineComment[]; + lineComments?: LineComment[]; }; export type Position = { From 4c38c1a54044affc277a2dfc34644e2b9d166fd5 Mon Sep 17 00:00:00 2001 From: ascandone Date: Sun, 25 Aug 2024 18:46:21 +0200 Subject: [PATCH 3/5] allowing newlines between let declrs --- src/formatter.test.ts | 62 +++++++++++++++++++++++++++++++++++++++++++ src/formatter.ts | 13 ++++++++- src/parser/parser.ts | 6 +++-- src/pretty.test.ts | 20 ++++++++++++++ 4 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/formatter.test.ts b/src/formatter.test.ts index b07084b9..f9adaed5 100644 --- a/src/formatter.test.ts +++ b/src/formatter.test.ts @@ -311,6 +311,68 @@ test("toplevel nested let expr", () => { `).toBeFormatted(); }); +test("allows spaces in toplevel nested let expr", () => { + expect(`let f = { + let x = value; + + let y = value2; + + body +} +`).toBeFormatted(`let f = { + let x = value; + + let y = value2; + + body +} +`); +}); + +test("allows at most 1 space in toplevel nested let expr", () => { + expect(`let f = { + let x = value; + + + + let y = value2; + + + + body +} +`).toBeFormatted(`let f = { + let x = value; + + let y = value2; + + body +} +`); +}); + +test("allow zero lines after struct", () => { + expect(`let f = { + let p = Person { + name: "hello", + age: 42, + }; + body +} +`).toBeFormatted(); +}); + +test("nested let", () => { + expect(`let a = { + let let_hash = { + let nested = value; + e + }; + body +} +`).toBeFormatted(); +}); + test("toplevel nested let# expr", () => { expect(`let f = { let#and_then x = value; diff --git a/src/formatter.ts b/src/formatter.ts index 33460758..e3e77d51 100644 --- a/src/formatter.ts +++ b/src/formatter.ts @@ -371,13 +371,24 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { } case "let": { + const linesDiff = Math.min( + ast.body.range.start.line - ast.value.range.end.line - 1, + 1, + ); + + console.log( + { linesDiff }, + ast.body.range.start.line, + ast.value.range.end.line, + ); + const inner = concat( text("let "), patternToDoc(ast.pattern), text(" ="), declarationValueToDoc(ast.value), text(";"), - break_(), + lines(linesDiff), exprToDoc(ast.body, true), ); diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 19cd161c..5caf4478 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -318,8 +318,10 @@ class ExpressionVisitor extends Visitor { range: rangeOfCtx(ctx), }); - visitBlockExpr = (ctx: BlockExprContext): UntypedExpr => - this.visit(ctx.block()); + visitBlockExpr = (ctx: BlockExprContext): UntypedExpr => { + const e = this.visit(ctx.block()); + return { ...e, range: rangeOfCtx(ctx) }; + }; visitBlockContentExpr = (ctx: BlockContentExprContext): UntypedExpr => this.visit(ctx.expr()); diff --git a/src/pretty.test.ts b/src/pretty.test.ts index 3437228d..824cc703 100644 --- a/src/pretty.test.ts +++ b/src/pretty.test.ts @@ -48,6 +48,26 @@ test("renders break", () => { ).toBe(`ab\ncd`); }); +test("does not render indentation", () => { + expect( + pprint( + group( + broken( + nest( + // + + text("if"), + break_(), + text("then"), + lines(2), + text("else"), + ), + ), + ), + ), + ).toBe(`if\n then\n\n\n else`); +}); + test("renders break of many lines", () => { expect( pprint( From 3c79617a920c4a15f0b75fc441cae424b2e6ea36 Mon Sep 17 00:00:00 2001 From: ascandone Date: Sun, 25 Aug 2024 18:50:01 +0200 Subject: [PATCH 4/5] removed console.log --- src/formatter.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/formatter.ts b/src/formatter.ts index e3e77d51..5549b7d9 100644 --- a/src/formatter.ts +++ b/src/formatter.ts @@ -376,12 +376,6 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { 1, ); - console.log( - { linesDiff }, - ast.body.range.start.line, - ast.value.range.end.line, - ); - const inner = concat( text("let "), patternToDoc(ast.pattern), From dcc14fc922a52788d8f11e4684aa8477c525274c Mon Sep 17 00:00:00 2001 From: ascandone Date: Sun, 25 Aug 2024 18:56:28 +0200 Subject: [PATCH 5/5] update snapshots --- src/formatter.test.ts | 13 +++++- src/formatter.ts | 2 +- src/parser/__snapshots__/parser.test.ts.snap | 44 ++++++++++---------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/formatter.test.ts b/src/formatter.test.ts index f9adaed5..1ae9b3ce 100644 --- a/src/formatter.test.ts +++ b/src/formatter.test.ts @@ -329,6 +329,15 @@ test("allows spaces in toplevel nested let expr", () => { `); }); +test("force at 1 space in toplevel nested let expr", () => { + expect(`let f = { let x = value; body } +`).toBeFormatted(`let f = { + let x = value; + body +} +`); +}); + test("allows at most 1 space in toplevel nested let expr", () => { expect(`let f = { let x = value; @@ -364,8 +373,8 @@ test("allow zero lines after struct", () => { test("nested let", () => { expect(`let a = { - let let_hash = { - let nested = value; + let l1 = { + let l2 = value; e }; body diff --git a/src/formatter.ts b/src/formatter.ts index 5549b7d9..bafb367e 100644 --- a/src/formatter.ts +++ b/src/formatter.ts @@ -372,7 +372,7 @@ function exprToDoc(ast: UntypedExpr, block: boolean): Doc { case "let": { const linesDiff = Math.min( - ast.body.range.start.line - ast.value.range.end.line - 1, + Math.max(ast.body.range.start.line - ast.value.range.end.line - 1, 0), 1, ); diff --git a/src/parser/__snapshots__/parser.test.ts.snap b/src/parser/__snapshots__/parser.test.ts.snap index 1def758e..3c93e866 100644 --- a/src/parser/__snapshots__/parser.test.ts.snap +++ b/src/parser/__snapshots__/parser.test.ts.snap @@ -1967,12 +1967,12 @@ exports[`monadic let syntax sugar 1`] = ` }, "range": { "end": { - "character": 10, - "line": 3, + "character": 5, + "line": 4, }, "start": { - "character": 6, - "line": 2, + "character": 12, + "line": 1, }, }, "type": "let#", @@ -2076,12 +2076,12 @@ exports[`monadic let syntax sugar should handle qualified names 1`] = ` }, "range": { "end": { - "character": 10, - "line": 3, + "character": 5, + "line": 4, }, "start": { - "character": 6, - "line": 2, + "character": 12, + "line": 1, }, }, "type": "let#", @@ -3393,11 +3393,11 @@ exports[`parse block with no let 1`] = ` "value": { "range": { "end": { - "character": 11, + "character": 13, "line": 1, }, "start": { - "character": 10, + "character": 8, "line": 1, }, }, @@ -4596,12 +4596,12 @@ exports[`parse let block with one let 1`] = ` }, "range": { "end": { - "character": 3, - "line": 3, + "character": 1, + "line": 4, }, "start": { - "character": 2, - "line": 2, + "character": 8, + "line": 1, }, }, "type": "let", @@ -4739,12 +4739,12 @@ exports[`parse let block with two let stmts 1`] = ` }, "range": { "end": { - "character": 3, - "line": 4, + "character": 1, + "line": 5, }, "start": { - "character": 2, - "line": 2, + "character": 8, + "line": 1, }, }, "type": "let", @@ -6941,11 +6941,11 @@ exports[`pattern matching > matching pattern in let 1`] = ` }, "range": { "end": { - "character": 35, + "character": 37, "line": 0, }, "start": { - "character": 10, + "character": 8, "line": 0, }, }, @@ -7112,11 +7112,11 @@ exports[`pattern matching > matching pattern in let# 1`] = ` }, "range": { "end": { - "character": 41, + "character": 43, "line": 0, }, "start": { - "character": 10, + "character": 8, "line": 0, }, },