diff --git a/runtime/ast/attachment.go b/runtime/ast/attachment.go index 32cd04747..ba21b854e 100644 --- a/runtime/ast/attachment.go +++ b/runtime/ast/attachment.go @@ -34,8 +34,8 @@ type AttachmentDeclaration struct { BaseType *NominalType Conformances []*NominalType Members *Members - DocString string Range + Comments } var _ Element = &AttachmentDeclaration{} @@ -50,8 +50,8 @@ func NewAttachmentDeclaration( baseType *NominalType, conformances []*NominalType, members *Members, - docString string, declarationRange Range, + comments Comments, ) *AttachmentDeclaration { common.UseMemory(memoryGauge, common.AttachmentDeclarationMemoryUsage) @@ -61,8 +61,8 @@ func NewAttachmentDeclaration( BaseType: baseType, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, + Comments: comments, } } @@ -99,7 +99,7 @@ func (d *AttachmentDeclaration) DeclarationMembers() *Members { } func (d *AttachmentDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (*AttachmentDeclaration) Kind() common.CompositeKind { @@ -199,9 +199,11 @@ func (d *AttachmentDeclaration) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Type string *Alias + DocString string }{ - Type: "AttachmentDeclaration", - Alias: (*Alias)(d), + Type: "AttachmentDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/attachment_test.go b/runtime/ast/attachment_test.go index d57c61e36..741350407 100644 --- a/runtime/ast/attachment_test.go +++ b/runtime/ast/attachment_test.go @@ -56,8 +56,12 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ), }, }, - Members: NewMembers(nil, []Declaration{}), - DocString: "test", + Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 1, Line: 2, Column: 3}, EndPos: Position{Offset: 4, Line: 5, Column: 6}, @@ -141,8 +145,12 @@ func TestAttachmentDeclaration_Doc(t *testing.T) { ), }, }, - Members: NewMembers(nil, []Declaration{}), - DocString: "test", + Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 1, Line: 2, Column: 3}, EndPos: Position{Offset: 4, Line: 5, Column: 6}, diff --git a/runtime/ast/block.go b/runtime/ast/block.go index 1f1b70a59..9528fbfcc 100644 --- a/runtime/ast/block.go +++ b/runtime/ast/block.go @@ -29,11 +29,12 @@ import ( type Block struct { Statements []Statement Range + Comments } var _ Element = &Block{} -func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range) *Block { +func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) return &Block{ diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go new file mode 100644 index 000000000..2daa66cde --- /dev/null +++ b/runtime/ast/comments.go @@ -0,0 +1,93 @@ +package ast + +import ( + "bytes" + "github.com/onflow/cadence/runtime/common" + "strings" +) + +type Comments struct { + Leading []*Comment `json:"-"` + Trailing []*Comment `json:"-"` +} + +func (c Comments) PackToList() []*Comment { + var comments []*Comment + comments = append(comments, c.Leading...) + comments = append(comments, c.Trailing...) + return comments +} + +// LeadingDocString prints the leading doc comments to string +func (c Comments) LeadingDocString() string { + var s strings.Builder + for _, comment := range c.Leading { + if comment.Doc() { + if s.Len() > 0 { + s.WriteRune('\n') + } + s.Write(comment.Text()) + } + } + return s.String() +} + +type Comment struct { + source []byte +} + +func NewComment(memoryGauge common.MemoryGauge, source []byte) *Comment { + // TODO(preserve-comments): Track memory usage + return &Comment{ + source: source, + } +} + +var blockCommentDocStringPrefix = []byte("/**") +var blockCommentStringPrefix = []byte("/*") +var lineCommentDocStringPrefix = []byte("///") +var lineCommentStringPrefix = []byte("//") +var blockCommentStringSuffix = []byte("*/") + +func (c Comment) Multiline() bool { + return bytes.HasPrefix(c.source, blockCommentStringPrefix) +} + +func (c Comment) Doc() bool { + if c.Multiline() { + return bytes.HasPrefix(c.source, blockCommentDocStringPrefix) + } else { + return bytes.HasPrefix(c.source, lineCommentDocStringPrefix) + } +} + +// Text without opening/closing comment characters /*, /**, */, // +func (c Comment) Text() []byte { + withoutPrefixes := cutOptionalPrefixes(c.source, [][]byte{ + blockCommentDocStringPrefix, // must be before blockCommentStringPrefix + blockCommentStringPrefix, + lineCommentDocStringPrefix, // must be before lineCommentStringPrefix + lineCommentStringPrefix, + }) + return cutOptionalSuffixes(withoutPrefixes, [][]byte{ + blockCommentStringSuffix, + }) +} + +func cutOptionalPrefixes(input []byte, prefixes [][]byte) (output []byte) { + output = input + for _, prefix := range prefixes { + cut, _ := bytes.CutPrefix(output, prefix) + output = cut + } + return +} + +func cutOptionalSuffixes(input []byte, suffixes [][]byte) (output []byte) { + output = input + for _, suffix := range suffixes { + cut, _ := bytes.CutSuffix(output, suffix) + output = cut + } + return +} diff --git a/runtime/ast/composite.go b/runtime/ast/composite.go index dae942b6a..e092c0fe8 100644 --- a/runtime/ast/composite.go +++ b/runtime/ast/composite.go @@ -51,12 +51,12 @@ func IsResourceDestructionDefaultEvent(identifier string) bool { type CompositeDeclaration struct { Members *Members - DocString string Conformances []*NominalType Identifier Identifier Range Access Access CompositeKind common.CompositeKind + Comments } var _ Element = &CompositeDeclaration{} @@ -71,8 +71,8 @@ func NewCompositeDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declarationRange Range, + comments Comments, ) *CompositeDeclaration { common.UseMemory(memoryGauge, common.CompositeDeclarationMemoryUsage) @@ -82,8 +82,8 @@ func NewCompositeDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, + Comments: comments, } } @@ -120,17 +120,19 @@ func (d *CompositeDeclaration) DeclarationMembers() *Members { } func (d *CompositeDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *CompositeDeclaration) MarshalJSON() ([]byte, error) { type Alias CompositeDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "CompositeDeclaration", - Alias: (*Alias)(d), + Type: "CompositeDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -304,12 +306,14 @@ const ( type FieldDeclaration struct { TypeAnnotation *TypeAnnotation - DocString string - Identifier Identifier + // TODO(preserve-comments): Remove + DocString string + Identifier Identifier Range Access Access VariableKind VariableKind Flags FieldDeclarationFlags + Comments } var _ Element = &FieldDeclaration{} @@ -323,8 +327,8 @@ func NewFieldDeclaration( variableKind VariableKind, identifier Identifier, typeAnnotation *TypeAnnotation, - docString string, declRange Range, + comments Comments, ) *FieldDeclaration { common.UseMemory(memoryGauge, common.FieldDeclarationMemoryUsage) @@ -342,8 +346,8 @@ func NewFieldDeclaration( VariableKind: variableKind, Identifier: identifier, TypeAnnotation: typeAnnotation, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -382,16 +386,18 @@ func (d *FieldDeclaration) MarshalJSON() ([]byte, error) { type Alias FieldDeclaration return json.Marshal(&struct { *Alias - Type string - Flags FieldDeclarationFlags `json:",omitempty"` - IsStatic bool - IsNative bool + Type string + Flags FieldDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + DocString string }{ - Type: "FieldDeclaration", - Alias: (*Alias)(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Flags: 0, + Type: "FieldDeclaration", + Alias: (*Alias)(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/composite_test.go b/runtime/ast/composite_test.go index 104e0cfa4..f9e725a14 100644 --- a/runtime/ast/composite_test.go +++ b/runtime/ast/composite_test.go @@ -406,8 +406,12 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/entitlement_declaration.go b/runtime/ast/entitlement_declaration.go index 6a53e4e44..de363dbe6 100644 --- a/runtime/ast/entitlement_declaration.go +++ b/runtime/ast/entitlement_declaration.go @@ -30,9 +30,9 @@ import ( type EntitlementDeclaration struct { Access Access - DocString string Identifier Identifier Range + Comments } var _ Element = &EntitlementDeclaration{} @@ -43,16 +43,16 @@ func NewEntitlementDeclaration( gauge common.MemoryGauge, access Access, identifier Identifier, - docString string, declRange Range, + comments Comments, ) *EntitlementDeclaration { common.UseMemory(gauge, common.EntitlementDeclarationMemoryUsage) return &EntitlementDeclaration{ Access: access, Identifier: identifier, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -85,17 +85,19 @@ func (d *EntitlementDeclaration) DeclarationMembers() *Members { } func (d *EntitlementDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -168,10 +170,10 @@ func (d *EntitlementMapRelation) Doc() prettier.Doc { // EntitlementMappingDeclaration type EntitlementMappingDeclaration struct { Access Access - DocString string Identifier Identifier Elements []EntitlementMapElement Range + Comments } var _ Element = &EntitlementMappingDeclaration{} @@ -183,8 +185,8 @@ func NewEntitlementMappingDeclaration( access Access, identifier Identifier, elements []EntitlementMapElement, - docString string, declRange Range, + comments Comments, ) *EntitlementMappingDeclaration { common.UseMemory(gauge, common.EntitlementMappingDeclarationMemoryUsage) @@ -192,8 +194,8 @@ func NewEntitlementMappingDeclaration( Access: access, Identifier: identifier, Elements: elements, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -244,17 +246,19 @@ func (d *EntitlementMappingDeclaration) DeclarationMembers() *Members { } func (d *EntitlementMappingDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementMappingDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementMappingDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementMappingDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementMappingDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/entitlement_declaration_test.go b/runtime/ast/entitlement_declaration_test.go index b1801abfe..d3ac5dbda 100644 --- a/runtime/ast/entitlement_declaration_test.go +++ b/runtime/ast/entitlement_declaration_test.go @@ -37,7 +37,11 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -131,7 +135,11 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -227,7 +235,11 @@ func TestEntitlementMappingDeclaration_Doc(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -300,7 +312,11 @@ func TestEntitlementMappingDeclaration_String(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 0142b92b2..992adf37e 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -227,6 +227,7 @@ type IntegerExpression struct { PositiveLiteral []byte Range Base int + Comments } var _ Element = &IntegerExpression{} @@ -238,6 +239,7 @@ func NewIntegerExpression( value *big.Int, base int, tokenRange Range, + comments Comments, ) *IntegerExpression { common.UseMemory(gauge, common.IntegerExpressionMemoryUsage) @@ -246,6 +248,7 @@ func NewIntegerExpression( Value: value, Base: base, Range: tokenRange, + Comments: comments, } } diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index f86f6be67..d5598077a 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -20,7 +20,6 @@ package ast import ( "encoding/json" - "github.com/turbolent/prettier" "github.com/onflow/cadence/runtime/common" @@ -69,17 +68,50 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access Flags FunctionDeclarationFlags + Comments } var _ Element = &FunctionDeclaration{} var _ Declaration = &FunctionDeclaration{} var _ Statement = &FunctionDeclaration{} +// TODO(preserve-comments): Temporary, add `comments` param to NewFunctionDeclaration in the future +func NewFunctionDeclarationWithComments( + gauge common.MemoryGauge, + access Access, + purity FunctionPurity, + isStatic bool, + isNative bool, + identifier Identifier, + typeParameterList *TypeParameterList, + parameterList *ParameterList, + returnTypeAnnotation *TypeAnnotation, + functionBlock *FunctionBlock, + startPos Position, + comments Comments, +) *FunctionDeclaration { + decl := NewFunctionDeclaration( + gauge, + access, + purity, + isStatic, + isNative, + identifier, + typeParameterList, + parameterList, + returnTypeAnnotation, + functionBlock, + startPos, + "", + ) + decl.Comments = comments + return decl +} + func NewFunctionDeclaration( gauge common.MemoryGauge, access Access, @@ -114,7 +146,6 @@ func NewFunctionDeclaration( ReturnTypeAnnotation: returnTypeAnnotation, FunctionBlock: functionBlock, StartPos: startPos, - DocString: docString, } } @@ -176,7 +207,7 @@ func (d *FunctionDeclaration) DeclarationMembers() *Members { } func (d *FunctionDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *FunctionDeclaration) Doc() prettier.Doc { @@ -200,16 +231,18 @@ func (d *FunctionDeclaration) MarshalJSON() ([]byte, error) { *Alias Type string Range - IsStatic bool - IsNative bool - Flags FunctionDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + Flags FunctionDeclarationFlags `json:",omitempty"` + DocString string }{ - Type: "FunctionDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Alias: (*Alias)(d), - Flags: 0, + Type: "FunctionDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Alias: (*Alias)(d), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/function_declaration_test.go b/runtime/ast/function_declaration_test.go index c164a67da..6531a8aab 100644 --- a/runtime/ast/function_declaration_test.go +++ b/runtime/ast/function_declaration_test.go @@ -113,8 +113,10 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []*Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, } actual, err := json.Marshal(decl) @@ -584,14 +586,17 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []*Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, }, } actual, err := json.Marshal(decl) require.NoError(t, err) + // TODO(preserve-comments): Do we need to include comments in the JSON AST? assert.JSONEq(t, // language=json ` diff --git a/runtime/ast/identifier.go b/runtime/ast/identifier.go index 515ea7699..dff0b1073 100644 --- a/runtime/ast/identifier.go +++ b/runtime/ast/identifier.go @@ -20,7 +20,6 @@ package ast import ( "encoding/json" - "github.com/onflow/cadence/runtime/common" ) diff --git a/runtime/ast/import.go b/runtime/ast/import.go index 312cc9903..17f4f8da4 100644 --- a/runtime/ast/import.go +++ b/runtime/ast/import.go @@ -33,6 +33,7 @@ type ImportDeclaration struct { Identifiers []Identifier Range LocationPos Position + Comments } var _ Element = &ImportDeclaration{} @@ -44,6 +45,7 @@ func NewImportDeclaration( location common.Location, declRange Range, locationPos Position, + comments Comments, ) *ImportDeclaration { common.UseMemory(gauge, common.ImportDeclarationMemoryUsage) @@ -52,6 +54,7 @@ func NewImportDeclaration( Location: location, Range: declRange, LocationPos: locationPos, + Comments: comments, } } diff --git a/runtime/ast/interface.go b/runtime/ast/interface.go index 8fa4c622f..b4565ed45 100644 --- a/runtime/ast/interface.go +++ b/runtime/ast/interface.go @@ -30,12 +30,12 @@ import ( type InterfaceDeclaration struct { Members *Members - DocString string Identifier Identifier Conformances []*NominalType Range Access Access CompositeKind common.CompositeKind + Comments } var _ Element = &InterfaceDeclaration{} @@ -49,8 +49,8 @@ func NewInterfaceDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declRange Range, + comments Comments, ) *InterfaceDeclaration { common.UseMemory(gauge, common.InterfaceDeclarationMemoryUsage) @@ -60,8 +60,8 @@ func NewInterfaceDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -96,17 +96,19 @@ func (d *InterfaceDeclaration) DeclarationMembers() *Members { } func (d *InterfaceDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *InterfaceDeclaration) MarshalJSON() ([]byte, error) { type Alias InterfaceDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "InterfaceDeclaration", - Alias: (*Alias)(d), + Type: "InterfaceDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/interface_test.go b/runtime/ast/interface_test.go index 24819d9c3..98888255c 100644 --- a/runtime/ast/interface_test.go +++ b/runtime/ast/interface_test.go @@ -42,8 +42,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -95,8 +99,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/parameter.go b/runtime/ast/parameter.go index 937ae2a3a..34c1e6f68 100644 --- a/runtime/ast/parameter.go +++ b/runtime/ast/parameter.go @@ -32,6 +32,7 @@ type Parameter struct { Label string Identifier Identifier StartPos Position `json:"-"` + Comments } func NewParameter( @@ -41,6 +42,7 @@ func NewParameter( typeAnnotation *TypeAnnotation, defaultArgument Expression, startPos Position, + comments Comments, ) *Parameter { common.UseMemory(gauge, common.ParameterMemoryUsage) return &Parameter{ @@ -49,6 +51,7 @@ func NewParameter( TypeAnnotation: typeAnnotation, DefaultArgument: defaultArgument, StartPos: startPos, + Comments: comments, } } diff --git a/runtime/ast/parameterlist.go b/runtime/ast/parameterlist.go index 97eef6a0f..638f0aee1 100644 --- a/runtime/ast/parameterlist.go +++ b/runtime/ast/parameterlist.go @@ -30,6 +30,7 @@ type ParameterList struct { _parametersByIdentifier map[string]*Parameter Parameters []*Parameter Range + Comments once sync.Once } @@ -37,11 +38,13 @@ func NewParameterList( gauge common.MemoryGauge, parameters []*Parameter, astRange Range, + comments Comments, ) *ParameterList { common.UseMemory(gauge, common.ParameterListMemoryUsage) return &ParameterList{ Parameters: parameters, Range: astRange, + Comments: comments, } } diff --git a/runtime/ast/position.go b/runtime/ast/position.go index 3a7dc42f8..1183985a9 100644 --- a/runtime/ast/position.go +++ b/runtime/ast/position.go @@ -143,6 +143,12 @@ func (e Range) EndPosition(common.MemoryGauge) Position { return e.EndPos } +func (e Range) Source(input []byte) []byte { + startOffset := e.StartPos.Offset + endOffset := e.EndPos.Offset + 1 + return input[startOffset:endOffset] +} + // NewRangeFromPositioned func NewRangeFromPositioned(memoryGauge common.MemoryGauge, hasPosition HasPosition) Range { diff --git a/runtime/ast/statement.go b/runtime/ast/statement.go index e1e3b6c51..79520d307 100644 --- a/runtime/ast/statement.go +++ b/runtime/ast/statement.go @@ -39,16 +39,23 @@ type Statement interface { type ReturnStatement struct { Expression Expression Range + Comments } var _ Element = &ReturnStatement{} var _ Statement = &ReturnStatement{} -func NewReturnStatement(gauge common.MemoryGauge, expression Expression, stmtRange Range) *ReturnStatement { +func NewReturnStatement( + gauge common.MemoryGauge, + expression Expression, + stmtRange Range, + comments Comments, +) *ReturnStatement { common.UseMemory(gauge, common.ReturnStatementMemoryUsage) return &ReturnStatement{ Expression: expression, Range: stmtRange, + Comments: comments, } } diff --git a/runtime/ast/transaction_declaration.go b/runtime/ast/transaction_declaration.go index a9b6d91b2..32cb9ccad 100644 --- a/runtime/ast/transaction_declaration.go +++ b/runtime/ast/transaction_declaration.go @@ -35,6 +35,7 @@ type TransactionDeclaration struct { DocString string Fields []*FieldDeclaration Range + Comments } var _ Element = &TransactionDeclaration{} @@ -49,8 +50,8 @@ func NewTransactionDeclaration( preConditions *Conditions, postConditions *Conditions, execute *SpecialFunctionDeclaration, - docString string, declRange Range, + comments Comments, ) *TransactionDeclaration { common.UseMemory(gauge, common.TransactionDeclarationMemoryUsage) @@ -61,8 +62,8 @@ func NewTransactionDeclaration( PreConditions: preConditions, PostConditions: postConditions, Execute: execute, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -104,17 +105,19 @@ func (d *TransactionDeclaration) DeclarationMembers() *Members { } func (d *TransactionDeclaration) DeclarationDocString() string { - return "" + return d.Comments.LeadingDocString() } func (d *TransactionDeclaration) MarshalJSON() ([]byte, error) { type Alias TransactionDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "TransactionDeclaration", - Alias: (*Alias)(d), + Type: "TransactionDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/transaction_declaration_test.go b/runtime/ast/transaction_declaration_test.go index 44f4afd39..f73ccf106 100644 --- a/runtime/ast/transaction_declaration_test.go +++ b/runtime/ast/transaction_declaration_test.go @@ -45,8 +45,12 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { Prepare: nil, PreConditions: &Conditions{}, PostConditions: &Conditions{}, - DocString: "test", - Execute: nil, + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, + Execute: nil, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/type.go b/runtime/ast/type.go index 095653afd..d893b50f7 100644 --- a/runtime/ast/type.go +++ b/runtime/ast/type.go @@ -109,6 +109,7 @@ func IsEmptyType(t Type) bool { type NominalType struct { NestedIdentifiers []Identifier `json:",omitempty"` Identifier Identifier + Comments } var _ Type = &NominalType{} @@ -125,6 +126,21 @@ func NewNominalType( } } +// TODO(preserve-comments): Should we remove this and use only NewNominalType (requires updating all dependants) +func NewNominalTypeWithComments( + memoryGauge common.MemoryGauge, + identifier Identifier, + nestedIdentifiers []Identifier, + comments Comments, +) *NominalType { + common.UseMemory(memoryGauge, common.NominalTypeMemoryUsage) + return &NominalType{ + Identifier: identifier, + NestedIdentifiers: nestedIdentifiers, + Comments: comments, + } +} + func (*NominalType) isType() {} func (t *NominalType) String() string { diff --git a/runtime/ast/variable_declaration.go b/runtime/ast/variable_declaration.go index bc82b55a5..a88dbc887 100644 --- a/runtime/ast/variable_declaration.go +++ b/runtime/ast/variable_declaration.go @@ -33,11 +33,11 @@ type VariableDeclaration struct { Transfer *Transfer SecondTransfer *Transfer ParentIfStatement *IfStatement `json:"-"` - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access IsConstant bool + Comments } var _ Element = &VariableDeclaration{} @@ -55,7 +55,7 @@ func NewVariableDeclaration( startPos Position, secondTransfer *Transfer, secondValue Expression, - docString string, + comments Comments, ) *VariableDeclaration { common.UseMemory(gauge, common.VariableDeclarationMemoryUsage) @@ -69,7 +69,7 @@ func NewVariableDeclaration( StartPos: startPos, SecondTransfer: secondTransfer, SecondValue: secondValue, - DocString: docString, + Comments: comments, } } @@ -127,7 +127,7 @@ func (d *VariableDeclaration) DeclarationMembers() *Members { } func (d *VariableDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } var varKeywordDoc prettier.Doc = prettier.Text("var") @@ -228,12 +228,14 @@ func (d *VariableDeclaration) MarshalJSON() ([]byte, error) { type Alias VariableDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string Range }{ - Type: "VariableDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - Alias: (*Alias)(d), + Type: "VariableDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/variable_declaration_test.go b/runtime/ast/variable_declaration_test.go index c34430f63..ead6891cd 100644 --- a/runtime/ast/variable_declaration_test.go +++ b/runtime/ast/variable_declaration_test.go @@ -71,7 +71,11 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { EndPos: Position{Offset: 28, Line: 29, Column: 30}, }, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, } actual, err := json.Marshal(decl) diff --git a/runtime/interpreter/interpreter_statement.go b/runtime/interpreter/interpreter_statement.go index 9170eaa0c..c5cc58843 100644 --- a/runtime/interpreter/interpreter_statement.go +++ b/runtime/interpreter/interpreter_statement.go @@ -192,6 +192,7 @@ func (interpreter *Interpreter) VisitSwitchStatement(switchStatement *ast.Switch interpreter, switchCase.Statements, ast.EmptyRange, + ast.Comments{}, ) result := interpreter.visitBlock(block) diff --git a/runtime/old_parser/comment.go b/runtime/old_parser/comment.go deleted file mode 100644 index 766abdd13..000000000 --- a/runtime/old_parser/comment.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package old_parser - -import ( - "github.com/onflow/cadence/runtime/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index 05f92e5d7..517f5c0c4 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -100,7 +100,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { accessPos, staticPos, nativePos, - docString, + "", ) case keywordImport: @@ -415,7 +415,7 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - docString, + ast.Comments{}, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) @@ -698,6 +698,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { endPos, ), locationPos, + ast.Comments{}, ), nil } @@ -819,12 +820,12 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, startPos, parameterList.EndPos, ), + ast.Comments{}, ), nil } @@ -910,12 +911,12 @@ func parseFieldWithVariableKind( variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } @@ -1059,8 +1060,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + ast.Comments{}, ), nil } } @@ -1465,12 +1466,12 @@ func parseFieldDeclarationWithoutVariableKind( ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } diff --git a/runtime/old_parser/declaration_test.go b/runtime/old_parser/declaration_test.go index 1ecbc1e9b..b8ef72a44 100644 --- a/runtime/old_parser/declaration_test.go +++ b/runtime/old_parser/declaration_test.go @@ -816,8 +816,10 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -854,8 +856,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -892,8 +899,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -6870,8 +6881,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -6885,8 +6900,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -6910,8 +6929,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -6938,6 +6961,10 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 245, Line: 11, Column: 43}, EndPos: ast.Position{Offset: 246, Line: 11, Column: 44}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, + }, }, }, StartPos: ast.Position{Offset: 216, Line: 11, Column: 14}, @@ -6988,8 +7015,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7006,8 +7033,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7024,8 +7051,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " destroyWithBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/runtime/old_parser/expression.go b/runtime/old_parser/expression.go index b4fd4c2a1..7a2f8638f 100644 --- a/runtime/old_parser/expression.go +++ b/runtime/old_parser/expression.go @@ -1771,7 +1771,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange, ast.Comments{}) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/runtime/old_parser/expression_test.go b/runtime/old_parser/expression_test.go index f1fb98ea7..25af7d15c 100644 --- a/runtime/old_parser/expression_test.go +++ b/runtime/old_parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" - "github.com/onflow/cadence/runtime/parser/lexer" "github.com/onflow/cadence/runtime/tests/utils" ) @@ -2134,71 +2133,6 @@ func TestParseBlockComment(t *testing.T) { ) }) - t.Run("invalid content", func(t *testing.T) { - - t.Parallel() - - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 0, - Column: 0, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 1, - Column: 1, - }, - }, - }, - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 4, - Column: 4, - }, - }, - }, - {Type: lexer.TokenEOF}, - }, - input: []byte(`/*foo`), - } - - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, - ) - utils.AssertEqualWithDiff(t, - []error{ - &SyntaxError{ - Message: "unexpected token identifier in block comment", - Pos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - }, - }, - errs, - ) - }) } func BenchmarkParseInfix(b *testing.B) { diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index 7d0054faa..216ff0bb2 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -108,6 +108,7 @@ func parseParameterList(p *parser) (*ast.ParameterList, error) { startPos, endPos, ), + ast.Comments{}, ), nil } @@ -167,6 +168,7 @@ func parseParameter(p *parser) (*ast.Parameter, error) { typeAnnotation, nil, startPos, + ast.Comments{}, ), nil } @@ -295,8 +297,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -329,7 +331,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, ast.FunctionPurityUnspecified, @@ -341,7 +343,9 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - docString, + ast.Comments{ + Leading: startToken.Leading, + }, ), nil } diff --git a/runtime/old_parser/parser.go b/runtime/old_parser/parser.go index c4114f7f5..5364257f3 100644 --- a/runtime/old_parser/parser.go +++ b/runtime/old_parser/parser.go @@ -19,7 +19,6 @@ package old_parser import ( - "bytes" "os" "strings" @@ -425,9 +424,6 @@ func (p *parser) skipSpaceAndComments() (containsNewline bool) { return } -var blockCommentDocStringPrefix = []byte("/**") -var lineCommentDocStringPrefix = []byte("///") - func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { var docStringBuilder strings.Builder defer func() { @@ -436,7 +432,7 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt } }() - var atEnd, insideLineDocString bool + var atEnd bool for !atEnd { switch p.current.Type { @@ -457,43 +453,6 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() - case lexer.TokenBlockCommentStart: - commentStartOffset := p.current.StartPos.Offset - endToken, ok := p.parseBlockComment() - - if ok && options.parseDocStrings { - commentEndOffset := endToken.EndPos.Offset - - contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] - - insideLineDocString = false - docStringBuilder.Reset() - if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { - // Strip prefix (`/**`) - docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) - } - } - - case lexer.TokenLineComment: - if options.parseDocStrings { - comment := p.currentTokenSource() - if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { - if insideLineDocString { - docStringBuilder.WriteByte('\n') - } else { - insideLineDocString = true - docStringBuilder.Reset() - } - // Strip prefix - docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) - } else { - insideLineDocString = false - docStringBuilder.Reset() - } - } - - p.next() - default: atEnd = true } diff --git a/runtime/old_parser/statement.go b/runtime/old_parser/statement.go index dfa73bb61..723f699ec 100644 --- a/runtime/old_parser/statement.go +++ b/runtime/old_parser/statement.go @@ -250,6 +250,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + ast.Comments{}, ), nil } @@ -359,6 +360,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.Comments{}, ) result = outer } @@ -476,6 +478,7 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{}, ), nil } @@ -535,6 +538,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{}, ), preConditions, postConditions, diff --git a/runtime/old_parser/transaction.go b/runtime/old_parser/transaction.go index a6c72419a..188de3a1d 100644 --- a/runtime/old_parser/transaction.go +++ b/runtime/old_parser/transaction.go @@ -202,12 +202,12 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, startPos, endPos, ), + ast.Comments{}, ), nil } diff --git a/runtime/parser/comment.go b/runtime/parser/comment.go deleted file mode 100644 index 1ca171889..000000000 --- a/runtime/parser/comment.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package parser - -import ( - "github.com/onflow/cadence/runtime/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 2c9775ae1..a57e9c146 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -32,9 +32,8 @@ import ( func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations []ast.Declaration, err error) { for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -48,7 +47,7 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] default: var declaration ast.Declaration - declaration, err = parseDeclaration(p, docString) + declaration, err = parseDeclaration(p) if err != nil { return } @@ -62,7 +61,8 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] } } -func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { +func parseDeclaration(p *parser) (ast.Declaration, error) { + var startComments []*ast.Comment var access ast.Access = ast.AccessNotSpecified var accessPos *ast.Position @@ -77,7 +77,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { nativeModifierEnabled := p.config.NativeModifierEnabled for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenPragma: @@ -101,7 +101,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") } - return parseVariableDeclaration(p, access, accessPos, docString) + return parseVariableDeclaration(p, access, accessPos) case KeywordFun: return parseFunctionDeclaration( @@ -113,7 +113,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { purityPos, staticPos, nativePos, - docString, + startComments, ) case KeywordImport: @@ -134,7 +134,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for event") } - return parseEventDeclaration(p, access, accessPos, docString) + return parseEventDeclaration(p, access, accessPos, startComments) case KeywordStruct: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindStructure) @@ -144,7 +144,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for struct") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordResource: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindResource) @@ -154,7 +154,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for resource") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEntitlement: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) @@ -164,14 +164,14 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) case KeywordAttachment: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindAttachment) if err != nil { return nil, err } - return parseAttachmentDeclaration(p, access, accessPos, docString) + return parseAttachmentDeclaration(p, access, accessPos, startComments) case KeywordContract: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindContract) @@ -181,7 +181,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for contract") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEnum: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEnum) @@ -191,7 +191,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for enum") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordTransaction: err := rejectAllModifiers(p, access, accessPos, staticPos, nativePos, common.DeclarationKindTransaction) @@ -202,7 +202,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { return nil, NewSyntaxError(*purityPos, "invalid view modifier for transaction") } - return parseTransactionDeclaration(p, docString) + return parseTransactionDeclaration(p, startComments) case KeywordView: if purity != ast.FunctionPurityUnspecified { @@ -238,7 +238,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { pos := p.current.StartPos accessPos = &pos var err error - access, err = parseAccess(p) + access, startComments, err = parseAccess(p) if err != nil { return nil, err } @@ -373,7 +373,7 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() entitlements := []*ast.NominalType{firstTy} var separator lexer.TokenType @@ -420,22 +420,22 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { // : 'access(self)' // | 'access(all)' ( '(' 'set' ')' )? // | 'access' '(' ( 'self' | 'contract' | 'account' | 'all' | entitlementList ) ')' -func parseAccess(p *parser) (ast.Access, error) { - +func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { switch string(p.currentTokenSource()) { case KeywordAccess: + accessTokenComments := p.current.Comments.PackToList() // Skip the `access` keyword p.nextSemanticToken() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { - return ast.AccessNotSpecified, p.syntaxError( + return ast.AccessNotSpecified, accessTokenComments, p.syntaxError( "expected keyword %s, got %s", enumeratedAccessModifierKeywords, p.current.Type, @@ -474,29 +474,29 @@ func parseAccess(p *parser) (ast.Access, error) { entitlementMapName, err := parseNominalType(p, lowestBindingPower) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } access = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } access = ast.NewEntitlementAccess(entitlements) } _, err = p.mustOne(lexer.TokenParenClose) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } - return access, nil + return access, accessTokenComments, nil default: - return ast.AccessNotSpecified, errors.NewUnreachableError() + return ast.AccessNotSpecified, nil, errors.NewUnreachableError() } } @@ -512,10 +512,10 @@ func parseVariableDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, ) (*ast.VariableDeclaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -525,6 +525,8 @@ func parseVariableDeclaration( // Skip the `let` or `var` keyword p.nextSemanticToken() + identifierToken := p.current + identifier, err := p.nonReservedIdentifier("after start of variable declaration") if err != nil { return nil, err @@ -545,7 +547,9 @@ func parseVariableDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() + + transferToken := p.current transfer := parseTransfer(p) if transfer == nil { return nil, p.syntaxError("expected transfer") @@ -556,7 +560,7 @@ func parseVariableDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() secondTransfer := parseTransfer(p) var secondValue ast.Expression @@ -578,7 +582,12 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - docString, + ast.Comments{ + Leading: append( + append(startToken.PackToList(), identifierToken.PackToList()...), + transferToken.PackToList()..., + ), + }, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) @@ -649,17 +658,19 @@ func parsePragmaDeclaration(p *parser) (*ast.PragmaDeclaration, error) { // ( string | hexadecimalLiteral | identifier ) func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { - startPosition := p.current.StartPos + startToken := p.current var identifiers []ast.Identifier var location common.Location var locationPos ast.Position var endPos ast.Position + var trailingComments []*ast.Comment parseStringOrAddressLocation := func() { locationPos = p.current.StartPos endPos = p.current.EndPos + trailingComments = p.current.Comments.Trailing switch p.current.Type { case lexer.TokenString: @@ -678,10 +689,11 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { p.next() } - setIdentifierLocation := func(identifier ast.Identifier) { + setIdentifierLocation := func(identifier ast.Identifier, identifierToken lexer.Token) { location = common.IdentifierLocation(identifier.Identifier) locationPos = identifier.Pos endPos = identifier.EndPosition(p.memoryGauge) + trailingComments = identifierToken.Comments.Trailing } parseLocation := func() error { @@ -691,7 +703,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { case lexer.TokenIdentifier: identifier := p.tokenToIdentifier(p.current) - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, p.current) p.next() default: @@ -778,7 +790,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return nil } - maybeParseFromIdentifier := func(identifier ast.Identifier) error { + maybeParseFromIdentifier := func(identifier ast.Identifier, identifierToken lexer.Token) error { // The current identifier is maybe the `from` keyword, // in which case the given (previous) identifier was // an imported identifier and not the import location. @@ -796,7 +808,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return err } } else { - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) } return nil @@ -810,6 +822,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { parseStringOrAddressLocation() case lexer.TokenIdentifier: + identifierToken := p.current identifier := p.tokenToIdentifier(p.current) // Skip the identifier p.nextSemanticToken() @@ -824,13 +837,13 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return nil, err } case lexer.TokenIdentifier: - err := maybeParseFromIdentifier(identifier) + err := maybeParseFromIdentifier(identifier, identifierToken) if err != nil { return nil, err } case lexer.TokenEOF: // The previous identifier is the identifier location - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) default: return nil, p.syntaxError( @@ -857,10 +870,14 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { location, ast.NewRange( p.memoryGauge, - startPosition, + startToken.StartPos, endPos, ), locationPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: trailingComments, + }, ), nil } @@ -923,10 +940,10 @@ func parseEventDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (*ast.CompositeDeclaration, error) { - - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -974,6 +991,10 @@ func parseEventDeclaration( }, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewCompositeDeclaration( p.memoryGauge, access, @@ -981,12 +1002,14 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, startPos, parameterList.EndPos, ), + ast.Comments{ + Leading: leadingComments, + }, ), nil } @@ -1025,9 +1048,8 @@ func parseFieldWithVariableKind( accessPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - docString string, + startComments []*ast.Comment, ) (*ast.FieldDeclaration, error) { - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) var variableKind ast.VariableKind @@ -1057,7 +1079,7 @@ func parseFieldWithVariableKind( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() typeAnnotation, err := parseTypeAnnotation(p) if err != nil { @@ -1072,12 +1094,14 @@ func parseFieldWithVariableKind( variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{ + Leading: startComments, + }, ), nil } @@ -1097,14 +1121,14 @@ func parseEntitlementMapping(p *parser, docString string) (*ast.EntitlementMapRe ) } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenRightArrow) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() outputType, err := parseType(p, lowestBindingPower) if err != nil { @@ -1119,7 +1143,7 @@ func parseEntitlementMapping(p *parser, docString string) (*ast.EntitlementMapRe ) } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewEntitlementMapRelation(p.memoryGauge, inputNominalType, outputNominalType), nil } @@ -1129,9 +1153,10 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy var elements []ast.EntitlementMapElement for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + // TODO(preserve-comments): Compute doc string + var docString string + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -1156,7 +1181,7 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy ) } - p.skipSpaceAndComments() + p.skipSpace() elements = append(elements, outputNominalType) } else { mapping, err := parseEntitlementMapping(p, docString) @@ -1180,9 +1205,10 @@ func parseEntitlementOrMappingDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1217,6 +1243,10 @@ func parseEntitlementOrMappingDeclaration( } p.nextSemanticToken() + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + if isMapping { _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { @@ -1227,7 +1257,7 @@ func parseEntitlementOrMappingDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1244,8 +1274,10 @@ func parseEntitlementOrMappingDeclaration( access, identifier, elements, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { declarationRange := ast.NewRange( @@ -1258,8 +1290,10 @@ func parseEntitlementOrMappingDeclaration( p.memoryGauge, access, identifier, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } } @@ -1285,7 +1319,7 @@ func parseConformances(p *parser) ([]*ast.NominalType, error) { } } - p.skipSpaceAndComments() + p.skipSpace() return conformances, nil } @@ -1302,10 +1336,11 @@ func parseCompositeOrInterfaceDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1319,7 +1354,7 @@ func parseCompositeOrInterfaceDeclaration( var identifier ast.Identifier for { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( "expected %s, got %s", @@ -1355,7 +1390,7 @@ func parseCompositeOrInterfaceDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1372,7 +1407,7 @@ func parseCompositeOrInterfaceDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1385,6 +1420,10 @@ func parseCompositeOrInterfaceDeclaration( endToken.EndPos, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + if isInterface { return ast.NewInterfaceDeclaration( p.memoryGauge, @@ -1393,8 +1432,10 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { return ast.NewCompositeDeclaration( @@ -1404,8 +1445,10 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } } @@ -1414,9 +1457,11 @@ func parseAttachmentDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1429,7 +1474,7 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.isToken(p.current, lexer.TokenIdentifier, KeywordFor) { return nil, p.syntaxError( @@ -1461,7 +1506,7 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1473,14 +1518,14 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() members, err := parseMembersAndNestedDeclarations(p, lexer.TokenBraceClose) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1493,6 +1538,10 @@ func parseAttachmentDeclaration( endToken.EndPos, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewAttachmentDeclaration( p.memoryGauge, access, @@ -1500,8 +1549,10 @@ func parseAttachmentDeclaration( baseNominalType, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } @@ -1514,9 +1565,8 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) var declarations []ast.Declaration for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -1529,7 +1579,7 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) return ast.NewMembers(p.memoryGauge, declarations), nil default: - memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p, docString) + memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p) if err != nil { return nil, err } @@ -1554,7 +1604,8 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | eventDeclaration // | enumCase // | pragmaDeclaration -func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaration, error) { +func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { + var startComments []*ast.Comment const functionBlockIsOptional = true @@ -1573,7 +1624,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio nativeModifierEnabled := p.config.NativeModifierEnabled for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -1599,7 +1650,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio accessPos, staticPos, nativePos, - docString, + startComments, ) case KeywordCase: @@ -1610,7 +1661,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseEnumCase(p, access, accessPos, docString) + return parseEnumCase(p, access, accessPos) case KeywordFun: return parseFunctionDeclaration( @@ -1622,7 +1673,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio purityPos, staticPos, nativePos, - docString, + startComments, ) case KeywordEvent: @@ -1633,7 +1684,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseEventDeclaration(p, access, accessPos, docString) + return parseEventDeclaration(p, access, accessPos, startComments) case KeywordStruct: if purity != ast.FunctionPurityUnspecified { @@ -1643,7 +1694,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordResource: if purity != ast.FunctionPurityUnspecified { @@ -1653,7 +1704,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordContract: if purity != ast.FunctionPurityUnspecified { @@ -1663,7 +1714,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEntitlement: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) @@ -1673,7 +1724,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) case KeywordEnum: if purity != ast.FunctionPurityUnspecified { @@ -1683,10 +1734,10 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordAttachment: - return parseAttachmentDeclaration(p, access, accessPos, docString) + return parseAttachmentDeclaration(p, access, accessPos, startComments) case KeywordView: if purity != ast.FunctionPurityUnspecified { @@ -1721,7 +1772,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio pos := p.current.StartPos accessPos = &pos var err error - access, err = parseAccess(p) + access, startComments, err = parseAccess(p) if err != nil { return nil, err } @@ -1798,7 +1849,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio staticPos, nativePos, identifier, - docString, ) case lexer.TokenParenOpen: @@ -1806,7 +1856,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio return nil, p.syntaxError("unexpected %s", p.current.Type) } - identifier := p.tokenToIdentifier(*previousIdentifierToken) return parseSpecialFunctionDeclaration( p, functionBlockIsOptional, @@ -1816,8 +1865,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio purityPos, staticPos, nativePos, - identifier, - docString, + *previousIdentifierToken, ) } @@ -1861,9 +1909,7 @@ func parseFieldDeclarationWithoutVariableKind( staticPos *ast.Position, nativePos *ast.Position, identifier ast.Identifier, - docString string, ) (*ast.FieldDeclaration, error) { - startPos := ast.EarliestPosition(identifier.Pos, accessPos, staticPos, nativePos) _, err := p.mustOne(lexer.TokenColon) @@ -1871,7 +1917,7 @@ func parseFieldDeclarationWithoutVariableKind( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() typeAnnotation, err := parseTypeAnnotation(p) if err != nil { @@ -1886,12 +1932,12 @@ func parseFieldDeclarationWithoutVariableKind( ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } @@ -1904,10 +1950,10 @@ func parseSpecialFunctionDeclaration( purityPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - identifier ast.Identifier, - docString string, + identifierToken lexer.Token, ) (*ast.SpecialFunctionDeclaration, error) { + identifier := p.tokenToIdentifier(identifierToken) startPos := ast.EarliestPosition(identifier.Pos, accessPos, purityPos, staticPos, nativePos) parameterList, returnTypeAnnotation, functionBlock, err := @@ -1946,7 +1992,7 @@ func parseSpecialFunctionDeclaration( return ast.NewSpecialFunctionDeclaration( p.memoryGauge, declarationKind, - ast.NewFunctionDeclaration( + ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, purity, @@ -1958,7 +2004,9 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - docString, + ast.Comments{ + Leading: identifierToken.Leading, + }, ), ), nil } @@ -1970,8 +2018,9 @@ func parseEnumCase( p *parser, access ast.Access, accessPos *ast.Position, - docString string, ) (*ast.EnumCaseDeclaration, error) { + // TODO(preserve-comments): Implement + var docString string startPos := p.current.StartPos if accessPos != nil { diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index b2c47abda..e3e2acd85 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -72,6 +72,59 @@ func TestParseVariableDeclaration(t *testing.T) { ) }) + t.Run("var, no type annotation, copy, one value, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before x +var x = /* Before 1 */ 1 // After 1 +// Ignored +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.VariableDeclaration{ + Access: ast.AccessNotSpecified, + IsConstant: false, + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 4, Offset: 17}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before x")), + ast.NewComment(nil, []byte("/* Before 1 */")), + }, + Trailing: []*ast.Comment{}, + }, + Value: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + EndPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After 1")), + }, + }, + }, + Transfer: &ast.Transfer{ + Operation: ast.TransferOperationCopy, + Pos: ast.Position{Line: 3, Column: 6, Offset: 19}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + }, + }, + result, + ) + }) + t.Run("var, no type annotation, copy, one value, access(all)", func(t *testing.T) { t.Parallel() @@ -389,7 +442,8 @@ func TestParseParameterList(t *testing.T) { nil, []byte(input), func(p *parser) (*ast.ParameterList, error) { - return parseParameterList(p, false) + parameters, err := parseParameterList(p, false) + return parameters, err }, Config{}, ) @@ -469,6 +523,62 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("one, resource type, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := parse(`( a : /* After colon */ +// Before type +@Int /* After type */ )`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: true, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 1, Offset: 41}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + // This comment should be attached to Type instead of TypeAnnotation + // (even tho it's technically attached to the @ resource symbol) for simplicity. + ast.NewComment(nil, []byte("// Before type")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After type */")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 40}, + }, + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After colon */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 3, Column: 22, Offset: 62}, + }, + }, + result, + ) + }) + t.Run("one, with argument label", func(t *testing.T) { t.Parallel() @@ -563,6 +673,91 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("two, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := parse(`( /* Before param b */ a b : /* Before b type annotation */ Int /* After param b */, +// Before param c +c /* After c identifier */ : Int // After param c +)`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "a", + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Line: 1, Column: 25, Offset: 25}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After param b */")), + }, + }, + }, + StartPos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + StartPos: ast.Position{Line: 1, Column: 23, Offset: 23}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* Before param b */")), + ast.NewComment(nil, []byte("/* Before b type annotation */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "c", + Pos: ast.Position{Line: 3, Column: 0, Offset: 104}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After param c ")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 104}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before param c")), + ast.NewComment(nil, []byte("/* After c identifier */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 4, Column: 0, Offset: 155}, + }, + }, + result, + ) + }) + t.Run("two, with and without argument label, missing comma", func(t *testing.T) { t.Parallel() @@ -881,8 +1076,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// Test")), + }, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -919,8 +1118,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -957,8 +1161,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -1368,7 +1576,10 @@ func TestParseFunctionDeclaration(t *testing.T) { PreConditions: (*ast.Conditions)(nil), PostConditions: (*ast.Conditions)(nil), }, - DocString: "", + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ @@ -1662,7 +1873,10 @@ func TestParseAccess(t *testing.T) { return Parse( nil, []byte(input), - parseAccess, + func(p *parser) (ast.Access, error) { + access, _, err := parseAccess(p) + return access, err + }, Config{}, ) } @@ -2107,6 +2321,39 @@ func TestParseImportDeclaration(t *testing.T) { ) }) + t.Run("no identifiers, string location, with leading/trailing comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before foo +import "foo" /* After foo */`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.ImportDeclaration{ + Identifiers: nil, + Location: common.StringLocation("foo"), + LocationPos: ast.Position{Line: 3, Column: 7, Offset: 22}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 15}, + EndPos: ast.Position{Line: 3, Column: 11, Offset: 26}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After foo */")), + }, + }, + }, + }, + result, + ) + }) + t.Run("no identifiers, address location", func(t *testing.T) { t.Parallel() @@ -2244,11 +2491,11 @@ func TestParseImportDeclaration(t *testing.T) { ) }) - t.Run("three identifiers, address location", func(t *testing.T) { + t.Run("three identifiers, address location, with trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo , bar , baz from 0x42`) + result, errs := testParseDeclarations(` import foo , bar , baz from 0x42 // After address`) require.Empty(t, errs) utils.AssertEqualWithDiff(t, @@ -2276,6 +2523,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 32, Offset: 32}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After address")), + }, + }, }, }, result, @@ -2323,11 +2575,11 @@ func TestParseImportDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, expected, result) }) - t.Run("no identifiers, identifier location", func(t *testing.T) { + t.Run("no identifiers, identifier location, with trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo`) + result, errs := testParseDeclarations(` import foo // After foo`) require.Empty(t, errs) utils.AssertEqualWithDiff(t, @@ -2340,6 +2592,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 10, Offset: 10}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After foo")), + }, + }, }, }, result, @@ -2359,6 +2616,7 @@ func TestParseImportDeclaration(t *testing.T) { }, errs) }) + t.Run("from keyword as second identifier", func(t *testing.T) { t.Parallel() @@ -2468,6 +2726,61 @@ func TestParseEvent(t *testing.T) { ) }) + t.Run("no parameters, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before E +event E() // After E +// Ignored +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Offset: 19, Line: 3, Column: 6}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After E")), + }, + }, + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + Kind: common.DeclarationKindInitializer, + FunctionDeclaration: &ast.FunctionDeclaration{ + Access: ast.AccessNotSpecified, + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + }, + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + }, + }, + }, + ), + Range: ast.Range{ + StartPos: ast.Position{Offset: 13, Line: 3, Column: 0}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + }, + }, + result, + ) + }) + t.Run("two parameters, private", func(t *testing.T) { t.Parallel() @@ -2601,11 +2914,14 @@ func TestParseEvent(t *testing.T) { ) }) - t.Run("default event", func(t *testing.T) { + t.Run("one parameter with comments", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` access(all) event ResourceDestroyed ( a : String = "foo")`) + result, errs := testParseDeclarations(`event E2 ( + // Before a + a: Int // After a +)`) require.Empty(t, errs) utils.AssertEqualWithDiff(t, @@ -2621,67 +2937,62 @@ func TestParseEvent(t *testing.T) { TypeAnnotation: &ast.TypeAnnotation{ Type: &ast.NominalType{ Identifier: ast.Identifier{ - Identifier: "String", + Identifier: "Int", Pos: ast.Position{ - Offset: 43, - Line: 1, - Column: 43, + Offset: 28, + Line: 3, + Column: 4, + }, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After a")), }, }, }, StartPos: ast.Position{ - Offset: 43, - Line: 1, - Column: 43, - }, - }, - DefaultArgument: &ast.StringExpression{ - Value: "foo", - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 52, - Line: 1, - Column: 52, - }, - EndPos: ast.Position{ - Offset: 56, - Line: 1, - Column: 56, - }, + Offset: 28, + Line: 3, + Column: 4, }, }, Identifier: ast.Identifier{ Identifier: "a", Pos: ast.Position{ - Offset: 39, - Line: 1, - Column: 39, + Offset: 25, + Line: 3, + Column: 1, }, }, StartPos: ast.Position{ - Offset: 39, - Line: 1, - Column: 39, + Offset: 25, + Line: 3, + Column: 1, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before a")), + }, }, }, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 37, + Offset: 9, Line: 1, - Column: 37, + Column: 9, }, EndPos: ast.Position{ - Offset: 57, - Line: 1, - Column: 57, + Offset: 43, + Line: 4, + Column: 0, }, }, }, StartPos: ast.Position{ - Offset: 37, + Offset: 9, Line: 1, - Column: 37, + Column: 9, }, Access: ast.AccessNotSpecified, }, @@ -2690,24 +3001,140 @@ func TestParseEvent(t *testing.T) { }, ), Identifier: ast.Identifier{ - Identifier: "ResourceDestroyed", + Identifier: "E2", Pos: ast.Position{ - Offset: 19, + Offset: 6, Line: 1, - Column: 19, + Column: 6, }, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 1, + Offset: 0, Line: 1, - Column: 1, + Column: 0, }, EndPos: ast.Position{ - Offset: 57, - Line: 1, - Column: 57, - }, + Offset: 43, + Line: 4, + Column: 0, + }, + }, + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + }, + }, + result, + ) + }) + + t.Run("default event", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` access(all) event ResourceDestroyed ( a : String = "foo")`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "String", + Pos: ast.Position{ + Offset: 43, + Line: 1, + Column: 43, + }, + }, + }, + StartPos: ast.Position{ + Offset: 43, + Line: 1, + Column: 43, + }, + }, + DefaultArgument: &ast.StringExpression{ + Value: "foo", + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 52, + Line: 1, + Column: 52, + }, + EndPos: ast.Position{ + Offset: 56, + Line: 1, + Column: 56, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{ + Offset: 39, + Line: 1, + Column: 39, + }, + }, + StartPos: ast.Position{ + Offset: 39, + Line: 1, + Column: 39, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 37, + Line: 1, + Column: 37, + }, + EndPos: ast.Position{ + Offset: 57, + Line: 1, + Column: 57, + }, + }, + }, + StartPos: ast.Position{ + Offset: 37, + Line: 1, + Column: 37, + }, + Access: ast.AccessNotSpecified, + }, + Kind: common.DeclarationKindInitializer, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "ResourceDestroyed", + Pos: ast.Position{ + Offset: 19, + Line: 1, + Column: 19, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 1, + Line: 1, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 57, + Line: 1, + Column: 57, + }, }, Access: ast.AccessAll, CompositeKind: common.CompositeKindEvent, @@ -2772,7 +3199,7 @@ func TestParseFieldWithVariableKind(t *testing.T) { nil, nil, nil, - "", + nil, ) }, Config{}, @@ -2857,10 +3284,7 @@ func TestParseField(t *testing.T) { nil, []byte(input), func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) + return parseMemberOrNestedDeclaration(p) }, config, ) @@ -3313,28 +3737,379 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ - Offset: 84, - Line: 5, - Column: 19, + Offset: 84, + Line: 5, + Column: 19, + }, + }, + StartPos: ast.Position{ + Offset: 84, + Line: 5, + Column: 19, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 83, + Line: 5, + Column: 18, + }, + EndPos: ast.Position{ + Offset: 92, + Line: 5, + Column: 27, + }, + }, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.AssignmentStatement{ + Target: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{ + Offset: 114, + Line: 6, + Column: 18, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 119, + Line: 6, + Column: 23, + }, + }, + AccessPos: ast.Position{ + Offset: 118, + Line: 6, + Column: 22, + }, + Optional: false, + }, + Transfer: &ast.Transfer{ + Operation: 0x1, + Pos: ast.Position{ + Offset: 123, + Line: 6, + Column: 27, + }, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 125, + Line: 6, + Column: 29, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 94, + Line: 5, + Column: 29, + }, + EndPos: ast.Position{ + Offset: 143, + Line: 7, + Column: 14, + }, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "init", + Pos: ast.Position{ + Offset: 79, + Line: 5, + Column: 14, + }, + }, + StartPos: ast.Position{ + Offset: 79, + Line: 5, + Column: 14, + }, + Access: ast.AccessNotSpecified, + Flags: 0x00, + }, + Kind: 0xd, + }, + &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 182, + Line: 9, + Column: 36, + }, + EndPos: ast.Position{ + Offset: 183, + Line: 9, + Column: 37, + }, + }, + }, + ReturnTypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 186, + Line: 9, + Column: 40, + }, + }, + }, + StartPos: ast.Position{ + Offset: 186, + Line: 9, + Column: 40, + }, + IsResource: false, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{ + Offset: 217, + Line: 10, + Column: 25, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 222, + Line: 10, + Column: 30, + }, + }, + AccessPos: ast.Position{ + Offset: 221, + Line: 10, + Column: 29, + }, + Optional: false, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 210, + Line: 10, + Column: 18, + }, + EndPos: ast.Position{ + Offset: 224, + Line: 10, + Column: 32, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 190, + Line: 9, + Column: 44, + }, + EndPos: ast.Position{ + Offset: 240, + Line: 11, + Column: 14, + }, + }, + }, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, + }, + Identifier: ast.Identifier{ + Identifier: "getFoo", + Pos: ast.Position{ + Offset: 176, + Line: 9, + Column: 30, + }, + }, + StartPos: ast.Position{ + Offset: 160, + Line: 9, + Column: 14, + }, + Access: ast.AccessAll, + Flags: 0x00, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "Test", + Pos: ast.Position{ + Offset: 18, + Line: 2, + Column: 17, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 11, + Line: 2, + Column: 10, + }, + EndPos: ast.Position{ + Offset: 252, + Line: 12, + Column: 10, + }, + }, + Access: ast.AccessNotSpecified, + CompositeKind: 0x1, + }, + }, + result, + ) + }) + + t.Run("struct, with fields, functions, special functions, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before Test + struct Test { + // Before foo + access(all) var foo: Int + + // Before init + init(foo: Int) { + self.foo = foo + } + + // Before getFoo + access(all) fun getFoo(): Int { + return self.foo + } + } + `) + + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.FieldDeclaration{ + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, + }, + }, + StartPos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 92, + Line: 5, + Column: 26, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 76, + Line: 5, + Column: 10, + }, + EndPos: ast.Position{ + Offset: 99, + Line: 5, + Column: 33, + }, + }, + Access: ast.AccessAll, + VariableKind: 0x1, + Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + }, + }, + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, + }, + }, + StartPos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 142, + Line: 8, + Column: 15, }, }, StartPos: ast.Position{ - Offset: 84, - Line: 5, - Column: 19, + Offset: 142, + Line: 8, + Column: 15, }, }, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 83, - Line: 5, - Column: 18, + Offset: 141, + Line: 8, + Column: 14, }, EndPos: ast.Position{ - Offset: 92, - Line: 5, - Column: 27, + Offset: 150, + Line: 8, + Column: 23, }, }, }, @@ -3347,42 +4122,42 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "self", Pos: ast.Position{ - Offset: 114, - Line: 6, - Column: 18, + Offset: 168, + Line: 9, + Column: 14, }, }, }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ - Offset: 119, - Line: 6, - Column: 23, + Offset: 173, + Line: 9, + Column: 19, }, }, AccessPos: ast.Position{ - Offset: 118, - Line: 6, - Column: 22, + Offset: 172, + Line: 9, + Column: 18, }, Optional: false, }, Transfer: &ast.Transfer{ Operation: 0x1, Pos: ast.Position{ - Offset: 123, - Line: 6, - Column: 27, + Offset: 177, + Line: 9, + Column: 23, }, }, Value: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ - Offset: 125, - Line: 6, - Column: 29, + Offset: 179, + Line: 9, + Column: 25, }, }, }, @@ -3390,14 +4165,14 @@ func TestParseCompositeDeclaration(t *testing.T) { }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 94, - Line: 5, - Column: 29, + Offset: 152, + Line: 8, + Column: 25, }, EndPos: ast.Position{ - Offset: 143, - Line: 7, - Column: 14, + Offset: 193, + Line: 10, + Column: 10, }, }, }, @@ -3405,18 +4180,23 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{ - Offset: 79, - Line: 5, - Column: 14, + Offset: 137, + Line: 8, + Column: 10, }, }, StartPos: ast.Position{ - Offset: 79, - Line: 5, - Column: 14, + Offset: 137, + Line: 8, + Column: 10, }, Access: ast.AccessNotSpecified, Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before init")), + }, + }, }, Kind: 0xd, }, @@ -3424,14 +4204,14 @@ func TestParseCompositeDeclaration(t *testing.T) { ParameterList: &ast.ParameterList{ Range: ast.Range{ StartPos: ast.Position{ - Offset: 182, - Line: 9, - Column: 36, + Offset: 255, + Line: 13, + Column: 32, }, EndPos: ast.Position{ - Offset: 183, - Line: 9, - Column: 37, + Offset: 256, + Line: 13, + Column: 33, }, }, }, @@ -3440,16 +4220,16 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "Int", Pos: ast.Position{ - Offset: 186, - Line: 9, - Column: 40, + Offset: 259, + Line: 13, + Column: 36, }, }, }, StartPos: ast.Position{ - Offset: 186, - Line: 9, - Column: 40, + Offset: 259, + Line: 13, + Column: 36, }, IsResource: false, }, @@ -3462,68 +4242,72 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "self", Pos: ast.Position{ - Offset: 217, - Line: 10, - Column: 25, + Offset: 286, + Line: 14, + Column: 21, }, }, }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ - Offset: 222, - Line: 10, - Column: 30, + Offset: 291, + Line: 14, + Column: 26, }, }, AccessPos: ast.Position{ - Offset: 221, - Line: 10, - Column: 29, + Offset: 290, + Line: 14, + Column: 25, }, Optional: false, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 210, - Line: 10, - Column: 18, + Offset: 279, + Line: 14, + Column: 14, }, EndPos: ast.Position{ - Offset: 224, - Line: 10, - Column: 32, + Offset: 293, + Line: 14, + Column: 28, }, }, }, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 190, - Line: 9, - Column: 44, + Offset: 263, + Line: 13, + Column: 40, }, EndPos: ast.Position{ - Offset: 240, - Line: 11, - Column: 14, + Offset: 305, + Line: 15, + Column: 10, }, }, }, }, - DocString: "", + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before getFoo")), + }, + }, Identifier: ast.Identifier{ Identifier: "getFoo", Pos: ast.Position{ - Offset: 176, - Line: 9, - Column: 30, + Offset: 249, + Line: 13, + Column: 26, }, }, StartPos: ast.Position{ - Offset: 160, - Line: 9, - Column: 14, + Offset: 233, + Line: 13, + Column: 10, }, Access: ast.AccessAll, Flags: 0x00, @@ -3533,25 +4317,30 @@ func TestParseCompositeDeclaration(t *testing.T) { Identifier: ast.Identifier{ Identifier: "Test", Pos: ast.Position{ - Offset: 18, - Line: 2, - Column: 17, + Offset: 35, + Line: 3, + Column: 13, }, }, Range: ast.Range{ StartPos: ast.Position{ - Offset: 11, - Line: 2, - Column: 10, + Offset: 28, + Line: 3, + Column: 6, }, EndPos: ast.Position{ - Offset: 252, - Line: 12, - Column: 10, + Offset: 313, + Line: 16, + Column: 6, }, }, Access: ast.AccessNotSpecified, - CompositeKind: 0x1, + CompositeKind: common.CompositeKindStructure, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before Test")), + }, + }, }, }, result, @@ -3784,6 +4573,45 @@ func TestParseAttachmentDeclaration(t *testing.T) { ) }) + t.Run("no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before E + access(all) attachment E for S {}`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.AttachmentDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Line: 3, Column: 27, Offset: 44}, + }, + BaseType: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 33, Offset: 50}, + }, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 4, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 36, Offset: 53}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested in contract", func(t *testing.T) { t.Parallel() @@ -4331,6 +5159,40 @@ func TestParseInterfaceDeclaration(t *testing.T) { ) }) + t.Run("struct, no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before S +access(all) struct interface S { }`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.InterfaceDeclaration{ + Access: ast.AccessAll, + CompositeKind: common.CompositeKindStructure, + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 29, Offset: 42}, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + EndPos: ast.Position{Line: 3, Column: 33, Offset: 46}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before S")), + }, + }, + }, + }, + result, + ) + }) + t.Run("struct, interface keyword as name", func(t *testing.T) { t.Parallel() @@ -4911,6 +5773,38 @@ func TestParseTransactionDeclaration(t *testing.T) { ) }) + t.Run("EmptyTransaction, comments", func(t *testing.T) { + + const code = ` + // Before tx + transaction {} + ` + result, errs := testParseProgram(code) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.TransactionDeclaration{ + Fields: nil, + Prepare: nil, + PreConditions: nil, + PostConditions: nil, + Execute: nil, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before tx")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 22, Line: 3, Column: 4}, + EndPos: ast.Position{Offset: 35, Line: 3, Column: 17}, + }, + }, + }, + result.Declarations(), + ) + }) + t.Run("SimpleTransaction", func(t *testing.T) { const code = ` transaction { @@ -8511,10 +9405,7 @@ func TestParseNestedPragma(t *testing.T) { nil, []byte(input), func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) + return parseMemberOrNestedDeclaration(p) }, config, ) @@ -8749,6 +9640,38 @@ func TestParseEntitlementDeclaration(t *testing.T) { ) }) + t.Run("basic, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before ABC + access(all) entitlement ABC`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "ABC", + Pos: ast.Position{Line: 3, Column: 27, Offset: 45}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 29, Offset: 47}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before ABC")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested entitlement", func(t *testing.T) { t.Parallel() @@ -8889,8 +9812,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -8904,8 +9831,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -8929,8 +9860,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -9004,8 +9939,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -9022,8 +9961,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -9080,6 +10023,38 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { ) }) + t.Run("empty, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before M + access(all) entitlement mapping M { }`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementMappingDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "M", + Pos: ast.Position{Line: 3, Column: 35, Offset: 51}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 19}, + EndPos: ast.Position{Line: 3, Column: 39, Offset: 55}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before M")), + }, + }, + }, + }, + result, + ) + }) + t.Run("mappings", func(t *testing.T) { t.Parallel() @@ -9093,8 +10068,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{ @@ -9182,8 +10156,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{ diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 41f44df7a..7fce53cab 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -360,7 +360,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindBinary, - token.Range, + token, ), nil }, }) @@ -374,7 +374,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindOctal, - token.Range, + token, ), nil }, }) @@ -388,7 +388,7 @@ func init() { literal, literal, common.IntegerLiteralKindDecimal, - token.Range, + token, ), nil }, }) @@ -402,7 +402,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindHexadecimal, - token.Range, + token, ), nil }, }) @@ -416,7 +416,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindUnknown, - token.Range, + token, ), nil }, }) @@ -597,7 +597,7 @@ func defineLessThanOrTypeArgumentsExpression() { return err } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken, err := p.mustOne(lexer.TokenParenOpen) if err != nil { return err @@ -832,7 +832,7 @@ func defineIdentifierExpression() { current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // skip the `fun` keyword @@ -965,7 +965,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.isToken(p.current, lexer.TokenIdentifier, KeywordTo) { return nil, p.syntaxError( @@ -982,7 +982,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewAttachExpression(p.memoryGauge, base, attachment, token.StartPos), nil } @@ -1017,7 +1017,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as atEnd := false expectArgument := true for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -1057,7 +1057,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as return nil, ast.EmptyPosition, err } - p.skipSpaceAndComments() + p.skipSpace() argument.TrailingSeparatorPos = p.current.StartPos @@ -1081,7 +1081,7 @@ func parseArgument(p *parser) (*ast.Argument, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() // If a colon follows the expression, the expression was our label. if p.current.Is(lexer.TokenColon) { @@ -1122,7 +1122,7 @@ func defineNestedExpression() { setExprNullDenotation( lexer.TokenParenOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() // special case: parse a Void literal `()` if p.current.Type == lexer.TokenParenClose { @@ -1148,11 +1148,11 @@ func defineArrayExpression() { setExprNullDenotation( lexer.TokenBracketOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var values []ast.Expression for !p.current.Is(lexer.TokenBracketClose) { - p.skipSpaceAndComments() + p.skipSpace() if len(values) > 0 { if !p.current.Is(lexer.TokenComma) { break @@ -1189,11 +1189,11 @@ func defineDictionaryExpression() { setExprNullDenotation( lexer.TokenBraceOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var entries []ast.DictionaryEntry for !p.current.Is(lexer.TokenBraceClose) { - p.skipSpaceAndComments() + p.skipSpace() if len(entries) > 0 { if !p.current.Is(lexer.TokenComma) { break @@ -1370,7 +1370,7 @@ func parseMemberAccess(p *parser, token lexer.Token, left ast.Expression, option if p.current.Is(lexer.TokenSpace) { errorPos := p.current.StartPos - p.skipSpaceAndComments() + p.skipSpace() p.report(NewSyntaxError( errorPos, "invalid whitespace after %s", @@ -1438,7 +1438,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { p.expressionDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -1453,7 +1453,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { // Some left denotations do not support newlines before them, // to avoid ambiguities and potential underhanded code - p.parseTrivia(triviaOptions{ + p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) @@ -1730,7 +1730,7 @@ func parseHex(r rune) rune { return -1 } -func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, tokenRange ast.Range) *ast.IntegerExpression { +func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, token lexer.Token) *ast.IntegerExpression { report := func(invalidKind InvalidNumberLiteralKind) { p.report( @@ -1739,7 +1739,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit InvalidIntegerLiteralKind: invalidKind, // NOTE: not using text, because it has the base-prefix stripped Literal: string(literal), - Range: tokenRange, + Range: token.Range, }, ) } @@ -1789,7 +1789,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, token.Range, token.Comments) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index e162d3518..606724e09 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" - "github.com/onflow/cadence/runtime/parser/lexer" "github.com/onflow/cadence/runtime/tests/utils" ) @@ -2226,71 +2225,6 @@ func TestParseBlockComment(t *testing.T) { ) }) - t.Run("invalid content", func(t *testing.T) { - - t.Parallel() - - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 0, - Column: 0, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 1, - Column: 1, - }, - }, - }, - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 4, - Column: 4, - }, - }, - }, - {Type: lexer.TokenEOF}, - }, - input: []byte(`/*foo`), - } - - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, - ) - utils.AssertEqualWithDiff(t, - []error{ - &SyntaxError{ - Message: "unexpected token identifier in block comment", - Pos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - }, - }, - errs, - ) - }) } func TestParseMulInfixExpression(t *testing.T) { diff --git a/runtime/parser/function.go b/runtime/parser/function.go index a6028be80..4ca302cc3 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -34,8 +34,10 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { var parameters []*ast.Parameter + var endToken lexer.Token + var comments ast.Comments - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenParenOpen) { return nil, p.syntaxError( @@ -45,17 +47,15 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL ) } - startPos := p.current.StartPos + startToken := p.current // Skip the opening paren p.next() - var endPos ast.Position - expectParameter := true atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: if !expectParameter { @@ -83,7 +83,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL expectParameter = true case lexer.TokenParenClose: - endPos = p.current.EndPos + endToken = p.current // Skip the closing paren p.next() atEnd = true @@ -109,21 +109,43 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } } + if len(parameters) == 0 { + comments.Leading = append( + comments.Leading, + startToken.Comments.PackToList()..., + ) + } else { + comments.Leading = append( + comments.Leading, + startToken.Comments.Leading..., + ) + + var patched []*ast.Comment + patched = append(patched, startToken.Comments.Trailing...) + patched = append(patched, parameters[0].Comments.Leading...) + parameters[0].Comments.Leading = patched + } + comments.Trailing = append( + comments.Trailing, + endToken.Comments.PackToList()..., + ) + return ast.NewParameterList( p.memoryGauge, parameters, ast.NewRange( p.memoryGauge, - startPos, - endPos, + startToken.StartPos, + endToken.EndPos, ), + comments, ), nil } func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, error) { - p.skipSpaceAndComments() + p.skipSpace() - startPos := p.current.StartPos + startToken := p.current argumentLabel := "" identifier, err := p.nonReservedIdentifier("for argument label or parameter name") @@ -150,11 +172,12 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro p.nextSemanticToken() } - if !p.current.Is(lexer.TokenColon) { + colonToken := p.current + if !colonToken.Is(lexer.TokenColon) { return nil, p.syntaxError( "expected %s after parameter name, got %s", lexer.TokenColon, - p.current.Type, + colonToken.Type, ) } @@ -167,7 +190,7 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var defaultArgument ast.Expression @@ -197,14 +220,20 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro identifier, typeAnnotation, defaultArgument, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: append( + startToken.Comments.PackToList(), + colonToken.Comments.PackToList()..., + ), + }, ), nil } func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { var typeParameters []*ast.TypeParameter - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenLess) { return nil, nil @@ -220,7 +249,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: if !expectTypeParameter { @@ -286,7 +315,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { } func parseTypeParameter(p *parser) (*ast.TypeParameter, error) { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( @@ -326,10 +355,10 @@ func parseFunctionDeclaration( purityPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - docString string, + startComments []*ast.Comment, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, purityPos, staticPos, nativePos) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -360,7 +389,11 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, purity, @@ -372,7 +405,9 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - docString, + ast.Comments{ + Leading: leadingComments, + }, ), nil } @@ -396,7 +431,7 @@ func parseFunctionParameterListAndRest( current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenColon) { // Skip the colon p.nextSemanticToken() @@ -415,7 +450,7 @@ func parseFunctionParameterListAndRest( if functionBlockIsOptional { current = p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenBraceOpen) { p.tokens.Revert(cursor) p.current = current diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 8c13ee6ea..fbd563d17 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -74,6 +74,9 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool + // currentComments stores the current leading and/or trailing comments + // nil is used as sentinel value to track newlines + currentComments []*ast.Comment } var _ TokenStream = &lexer{} @@ -99,6 +102,9 @@ func (l *lexer) Next() Token { pos, pos, ), + Comments: ast.Comments{ + Leading: l.currentComments, + }, } } @@ -184,6 +190,8 @@ func (l *lexer) run(state stateFn) (err error) { state = state(l) } + l.updatePreviousTrailingComments() + return } @@ -251,6 +259,18 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos := l.endPos() + // Only track trivia for non-space tokens + var leadingComments []*ast.Comment + shouldConsumeTrivia := ty != TokenSpace + if shouldConsumeTrivia { + l.updatePreviousTrailingComments() + leadingComments = l.currentComments + defer (func() { + // Mark as fully consumed + l.currentComments = []*ast.Comment{} + })() + } + token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -264,24 +284,100 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), + Comments: ast.Comments{ + Leading: leadingComments, + // Trailing comments can't be determined, as it wasn't consumed yet at this point. + Trailing: []*ast.Comment{}, + }, } l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) if consume { - l.startOffset = l.endOffset + l.consume(endPos) + } +} - l.startPos = endPos - r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) +func (l *lexer) updatePreviousTrailingComments() { + if l.tokenCount == 0 { + return + } - if r == '\n' { - l.startPos.line++ - l.startPos.column = 0 + lastNonSpaceTokenIndex := -1 + for i := l.tokenCount - 1; i >= 0; i-- { + if l.tokens[i].Type != TokenSpace { + lastNonSpaceTokenIndex = i + break + } + } + + // Split the current comment into trailing comment of the previous token + // and leading comment of the next token. + var trailing []*ast.Comment + var leading []*ast.Comment + + trailingTriviaEnded := lastNonSpaceTokenIndex == -1 + for _, comment := range l.currentComments { + if comment == nil { + trailingTriviaEnded = true + continue + } + if trailingTriviaEnded { + leading = append(leading, comment) } else { - l.startPos.column++ + trailing = append(trailing, comment) } } + + l.currentComments = leading + + if lastNonSpaceTokenIndex != -1 { + lastNonSpaceToken := &l.tokens[lastNonSpaceTokenIndex] + lastNonSpaceToken.Trailing = append(lastNonSpaceToken.Trailing, trailing...) + } +} + +func (l *lexer) emitNewlineSentinelComment() { + l.currentComments = append(l.currentComments, nil) +} + +func (l *lexer) emitComment() { + endPos := l.endPos() + + currentRange := ast.NewRange( + l.memoryGauge, + l.startPosition(), + ast.NewPosition( + l.memoryGauge, + l.endOffset-1, + endPos.line, + endPos.column, + ), + ) + + if l.currentComments == nil { + l.currentComments = []*ast.Comment{} + } + + l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, currentRange.Source(l.input))) + + l.consume(endPos) +} + +// endPos pre-computed end-position by calling l.endPos() +func (l *lexer) consume(endPos position) { + l.startOffset = l.endOffset + + l.startPos = endPos + r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) + + if r == '\n' { + l.startPos.line++ + l.startPos.column = 0 + } else { + l.startPos.column++ + } } func (l *lexer) startPosition() ast.Position { diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 80f5cf3dd..41bf6ec65 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -876,6 +876,961 @@ func TestLexBasic(t *testing.T) { }, ) }) + + t.Run("Trivia", func(t *testing.T) { + testLex(t, + "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", + []token{ + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, + EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, + }, + }, + Source: "test", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + }, + }, + }, + }, + ) + }) +} + +func TestLexComments(t *testing.T) { + + t.Parallel() + + t.Run("simple", func(t *testing.T) { + testLex(t, + `/* before brace open */ { /* after brace open */ // after brace open 2 +// before brace close 1 +// before brace close 2 +}`, + []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* before brace open */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* after brace open */")), + ast.NewComment(nil, []byte("// after brace open 2")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + EndPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before brace close 1")), + ast.NewComment(nil, []byte("// before brace close 2")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + }, + }, + }, + }, + ) + }) + + t.Run("complex", func(t *testing.T) { + testLex(t, ` +// Before transaction identifier +transaction /* After transaction identifier */ ( + // Before first arg + a: Int, // After first arg + /* + Before second arg + */ + b: Int /* After second arg + + */ // After second arg 2 + // Before paren close +) /* After paren close */ { /* After brace open */ } // After brace close + +// Before EOF +`, []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + EndPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before transaction identifier")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After transaction identifier */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 34, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 44, + Line: 3, + Column: 10, + }, + }, + }, + Source: "transaction", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + EndPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + EndPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenParenOpen, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + EndPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + }, + }, + Source: "(", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 82, + Line: 3, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 83, + Line: 4, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 103, + Line: 4, + Column: 20, + }, + EndPos: ast.Position{ + Offset: 104, + Line: 5, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before first arg")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + }, + }, + Source: "a", + }, + { + Token: Token{ + Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + }, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 108, + Line: 5, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 110, + Line: 5, + Column: 6, + }, + }, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenComma, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After first arg ")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, + }, + }, + }, + Source: ",", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + EndPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 132, + Line: 5, + Column: 28, + }, + EndPos: ast.Position{ + Offset: 133, + Line: 6, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 159, + Line: 8, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 160, + Line: 9, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/*\n\tBefore second arg\n\t*/")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + }, + }, + Source: "b", + }, + { + Token: Token{ + Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + }, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After second arg\n\n\t*/")), + ast.NewComment(nil, []byte("// After second arg 2")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 164, + Line: 9, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 166, + Line: 9, + Column: 6, + }, + }, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 214, + Line: 11, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 215, + Line: 12, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + EndPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenParenClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before paren close")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After paren close */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, + }, + }, + }, + Source: ")", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After brace open */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, + }, + EndPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + EndPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + EndPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After brace close")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, + }, + EndPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + EndPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 311, + Line: 13, + Column: 73, + }, + EndPos: ast.Position{ + Offset: 312, + Line: 14, + Column: 0, + }, + }, + }, + Source: "\n\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 326, + Line: 15, + Column: 13, + }, + EndPos: ast.Position{ + Offset: 326, + Line: 15, + Column: 13, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenEOF, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before EOF")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 327, + Line: 16, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 327, + Line: 16, + Column: 0, + }, + }, + }, + }, + }) + }) } func TestLexString(t *testing.T) { @@ -1202,53 +2157,14 @@ func TestLexBlockComment(t *testing.T) { []token{ { Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Source: ` // *X `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 10, Offset: 10}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 17, Offset: 17}, - }, - }, - Source: ` \\* `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, + Type: TokenError, + SpaceOrError: errors.New(`missing comment end '*/'`), Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - EndPos: ast.Position{Line: 1, Column: 19, Offset: 19}, + StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, + EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, }, }, - Source: "*/", + Source: "\000", }, { Token: Token{ @@ -1267,89 +2183,6 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/* test foo /* bar */ asd */ `, []token{ - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: ` test foo `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 14, Offset: 14}, - EndPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - }, - }, - Source: ` bar `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 19, Offset: 19}, - EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 21, Offset: 21}, - EndPos: ast.Position{Line: 1, Column: 25, Offset: 25}, - }, - }, - Source: ` asd `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 26, Offset: 26}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, - EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenEOF, @@ -2272,16 +3105,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenEOF, @@ -2337,16 +3160,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenSpace, diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 4b252d6f0..435bb69ec 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -124,8 +124,7 @@ func rootState(l *lexer) stateFn { case '/': return lineCommentState case '*': - l.emitType(TokenBlockCommentStart) - return blockCommentState(0) + return blockCommentState(l, 0) default: l.backupOne() l.emitType(TokenSlash) @@ -265,16 +264,22 @@ func spaceState(startIsNewline bool) stateFn { containsNewline := l.scanSpace() containsNewline = containsNewline || startIsNewline + l.scanSpace() + + // TODO(preserve-comments): Do we need to track memory for other token types as well? common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) + if containsNewline { + l.emitNewlineSentinelComment() + } + l.emit( TokenSpace, - Space{ - ContainsNewline: containsNewline, - }, + Space{ContainsNewline: containsNewline}, l.startPosition(), true, ) + return rootState } } @@ -307,12 +312,13 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitType(TokenLineComment) + l.emitComment() return rootState } -func blockCommentState(nesting int) stateFn { +func blockCommentState(l *lexer, nesting int) stateFn { if nesting < 0 { + l.emitComment() return rootState } @@ -320,16 +326,15 @@ func blockCommentState(nesting int) stateFn { r := l.next() switch r { case EOF: + l.emitError(fmt.Errorf("missing comment end '*/'")) return nil case '/': beforeSlashOffset := l.prevEndOffset if l.acceptOne('*') { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitType(TokenBlockCommentContent) l.endOffset = starOffset - l.emitType(TokenBlockCommentStart) - return blockCommentState(nesting + 1) + return blockCommentState(l, nesting+1) } case '*': @@ -337,13 +342,13 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('/') { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitType(TokenBlockCommentContent) l.endOffset = slashOffset - l.emitType(TokenBlockCommentEnd) - return blockCommentState(nesting - 1) + return blockCommentState(l, nesting-1) } + + return blockCommentState(l, nesting) } - return blockCommentState(nesting) + return blockCommentState(l, nesting) } } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index cfda861bd..fca59a99f 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -24,8 +24,14 @@ import ( type Token struct { SpaceOrError any + Type TokenType ast.Range - Type TokenType + // TODO(preserve-comments): This is currently not true (first comment), + // as leading comments are just all comments after the trailing comments of the previous token. + // Leading comments span up to and including the first contiguous sequence of newlines characters. + // Trailing comments span up to, but not including, the next newline character. + // Not tracked for space token, since those are usually ignored in the parser. + ast.Comments } func (t Token) Is(ty TokenType) bool { @@ -33,7 +39,5 @@ func (t Token) Is(ty TokenType) bool { } func (t Token) Source(input []byte) []byte { - startOffset := t.StartPos.Offset - endOffset := t.EndPos.Offset + 1 - return input[startOffset:endOffset] + return t.Range.Source(input) } diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index 0a15c19b6..0bdd92371 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -69,10 +69,6 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual - TokenBlockCommentStart - TokenBlockCommentEnd - TokenBlockCommentContent - TokenLineComment TokenAmpersand TokenAmpersandAmpersand TokenCaret @@ -179,14 +175,6 @@ func (t TokenType) String() string { return `'!'` case TokenNotEqual: return `'!='` - case TokenBlockCommentStart: - return `'/*'` - case TokenBlockCommentContent: - return "block comment" - case TokenLineComment: - return "line comment" - case TokenBlockCommentEnd: - return `'*/'` case TokenAmpersand: return `'&'` case TokenAmpersandAmpersand: diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 8af423723..b2aa43f42 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -19,14 +19,11 @@ package parser import ( - "bytes" - "os" - "strings" - "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/parser/lexer" + "os" ) // expressionDepthLimit is the limit of how deeply nested an expression can get @@ -192,7 +189,7 @@ func ParseTokenStream[T any]( return result, p.errors } - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenEOF) { p.reportSyntaxError("unexpected token: %s", p.current.Type) @@ -279,7 +276,7 @@ func (p *parser) next() { // It skips whitespace, including newlines, and comments func (p *parser) nextSemanticToken() { p.next() - p.skipSpaceAndComments() + p.skipSpace() } func (p *parser) mustOne(tokenType lexer.TokenType) (lexer.Token, error) { @@ -429,31 +426,20 @@ func (p *parser) replayBuffered() error { return nil } -type triviaOptions struct { - skipNewlines bool - parseDocStrings bool +type skipSpaceOptions struct { + skipNewlines bool } -// skipSpaceAndComments skips whitespace, including newlines, and comments -func (p *parser) skipSpaceAndComments() (containsNewline bool) { - containsNewline, _ = p.parseTrivia(triviaOptions{ +// skipSpace skips whitespace, including newlines, and comments +func (p *parser) skipSpace() (containsNewline bool) { + containsNewline = p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) return } -var blockCommentDocStringPrefix = []byte("/**") -var lineCommentDocStringPrefix = []byte("///") - -func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { - var docStringBuilder strings.Builder - defer func() { - if options.parseDocStrings { - docString = docStringBuilder.String() - } - }() - - var atEnd, insideLineDocString bool +func (p *parser) skipSpaceWithOptions(options skipSpaceOptions) (containsNewline bool) { + var atEnd bool for !atEnd { switch p.current.Type { @@ -474,43 +460,6 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() - case lexer.TokenBlockCommentStart: - commentStartOffset := p.current.StartPos.Offset - endToken, ok := p.parseBlockComment() - - if ok && options.parseDocStrings { - commentEndOffset := endToken.EndPos.Offset - - contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] - - insideLineDocString = false - docStringBuilder.Reset() - if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { - // Strip prefix (`/**`) - docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) - } - } - - case lexer.TokenLineComment: - if options.parseDocStrings { - comment := p.currentTokenSource() - if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { - if insideLineDocString { - docStringBuilder.WriteByte('\n') - } else { - insideLineDocString = true - docStringBuilder.Reset() - } - // Strip prefix - docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) - } else { - insideLineDocString = false - docStringBuilder.Reset() - } - } - - p.next() - default: atEnd = true } @@ -672,7 +621,7 @@ func ParseArgumentList( memoryGauge, input, func(p *parser) (ast.Arguments, error) { - p.skipSpaceAndComments() + p.skipSpace() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index 5cd811211..dfecb8e6b 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -540,8 +540,8 @@ func TestParseBuffering(t *testing.T) { []error{ &RestrictedTypeError{ Range: ast.Range{ - StartPos: ast.Position{Offset: 138, Line: 4, Column: 55}, - EndPos: ast.Position{Offset: 139, Line: 4, Column: 56}, + StartPos: ast.Position{Offset: 145, Line: 4, Column: 62}, + EndPos: ast.Position{Offset: 145, Line: 4, Column: 62}, }, }, }, @@ -605,7 +605,7 @@ func TestParseEOF(t *testing.T) { if err != nil { return struct{}{}, err } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustToken(lexer.TokenIdentifier, "b") if err != nil { return struct{}{}, err @@ -1007,3 +1007,95 @@ func TestParseWhitespaceAtEnd(t *testing.T) { assert.Empty(t, errs) } + +func TestParseComments(t *testing.T) { + + t.Parallel() + + t.Run("special function declaration", func(t *testing.T) { + res, errs := ParseDeclarations( + nil, + []byte(` +/// Before MyEvent +event MyEvent() // After MyEvent +/// Before prepare +prepare() {} // After prepare +/// Before pre +pre {} // After pre +/// Before execute +execute {} // After execute +/// Before post +post {} // After post +`), + Config{}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + event, ok := res[0].(*ast.SpecialFunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, + Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, + }, event.FunctionDeclaration.Comments) + assert.Equal(t, " Before MyEvent", event.FunctionDeclaration.DeclarationDocString()) + + }) + + t.Run("function declaration", func(t *testing.T) { + res, errs := ParseProgram( + nil, + []byte(` +/// Inline doc 1 of first +/// Inline doc 2 of first +fun first() {} // Trailing inline comment of first + +/** +Multi-line doc 1 of second +*/ +/** +Multi-line doc 2 of second +*/ +fun second() {} /** +Trailing multi-line comment of second +*/ +`), + Config{}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// Inline doc 1 of first")), + ast.NewComment(nil, []byte("/// Inline doc 2 of first")), + }, + }, first.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// Trailing inline comment of first")), + }, + }, first.FunctionBlock.Block.Comments) + + second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), + ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), + }, + }, second.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), + }, + }, second.FunctionBlock.Block.Comments) + }) + +} diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index cd5c9b93e..f803f79b2 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -27,7 +27,7 @@ import ( func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statements []ast.Statement, err error) { sawSemicolon := false for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: sawSemicolon = true @@ -71,7 +71,7 @@ func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statem } func parseStatement(p *parser) (ast.Statement, error) { - p.skipSpaceAndComments() + p.skipSpace() // Flag for cases where we can tell early-on that the current token isn't being used as a keyword // e.g. soft keywords like `view` @@ -127,7 +127,7 @@ func parseStatement(p *parser) (ast.Statement, error) { if !tokenIsIdentifier { // If it is not a keyword for a statement, // it might start with a keyword for a declaration - declaration, err := parseDeclaration(p, "") + declaration, err := parseDeclaration(p) if err != nil { return nil, err } @@ -148,7 +148,7 @@ func parseStatement(p *parser) (ast.Statement, error) { // If the expression is followed by a transfer, // it is actually the target of an assignment or swap statement - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenEqual, lexer.TokenLeftArrow, lexer.TokenLeftArrowExclamation: transfer := parseTransfer(p) @@ -247,11 +247,13 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( } func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { - tokenRange := p.current.Range + var endToken lexer.Token + startToken := p.current + tokenRange := startToken.Range endPosition := tokenRange.EndPos p.next() - sawNewLine, _ := p.parseTrivia(triviaOptions{ + sawNewLine := p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) @@ -259,6 +261,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { var err error switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: + endToken = p.current break default: if !sawNewLine { @@ -279,6 +282,10 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: endToken.Comments.PackToList(), + }, ), nil } @@ -311,7 +318,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil, "") + parseVariableDeclaration(p, ast.AccessNotSpecified, nil) if err != nil { return nil, err } @@ -336,7 +343,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { parseNested := false - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordElse) { p.nextSemanticToken() if p.isToken(p.current, lexer.TokenIdentifier, KeywordIf) { @@ -388,6 +395,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.Comments{}, ) result = outer } @@ -431,7 +439,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var index *ast.Identifier var identifier ast.Identifier @@ -444,7 +452,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() } else { identifier = firstValue } @@ -505,18 +513,22 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: endToken.Comments.PackToList(), + }, ), nil } func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { - p.skipSpaceAndComments() + p.skipSpace() startToken, err := p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var preConditions *ast.Conditions if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { @@ -529,7 +541,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { preConditions = &conditions } - p.skipSpaceAndComments() + p.skipSpace() var postConditions *ast.Conditions if p.isToken(p.current, lexer.TokenIdentifier, KeywordPost) { @@ -564,6 +576,10 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Leading, + Trailing: endToken.Trailing, + }, ), preConditions, postConditions, @@ -573,7 +589,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { // parseConditions parses conditions (pre/post) func parseConditions(p *parser) (conditions ast.Conditions, err error) { - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err @@ -581,7 +597,7 @@ func parseConditions(p *parser) (conditions ast.Conditions, err error) { var done bool for !done { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: p.next() @@ -601,7 +617,7 @@ func parseConditions(p *parser) (conditions ast.Conditions, err error) { } } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceClose) if err != nil { return nil, err @@ -632,7 +648,7 @@ func parseCondition(p *parser) (ast.Condition, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var message ast.Expression if p.current.Is(lexer.TokenColon) { @@ -717,7 +733,7 @@ func parseSwitchCases(p *parser) (cases []*ast.SwitchCase, err error) { } for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -771,7 +787,7 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { return nil, err } } else { - p.skipSpaceAndComments() + p.skipSpace() } colonPos := p.current.StartPos @@ -831,7 +847,7 @@ func parseRemoveStatement( startPos := p.current.StartPos p.next() - p.skipSpaceAndComments() + p.skipSpace() attachment, err := parseType(p, lowestBindingPower) if err != nil { @@ -846,7 +862,7 @@ func parseRemoveStatement( ) } - p.skipSpaceAndComments() + p.skipSpace() // check and skip `from` keyword if !p.isToken(p.current, lexer.TokenIdentifier, KeywordFrom) { diff --git a/runtime/parser/statement_test.go b/runtime/parser/statement_test.go index 551760b67..37645da0d 100644 --- a/runtime/parser/statement_test.go +++ b/runtime/parser/statement_test.go @@ -78,6 +78,66 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("no expression, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + + t.Run("no expression, semicolon, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return ; // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on same line", func(t *testing.T) { t.Parallel() @@ -107,6 +167,49 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("expression on same line, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return 1 // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + // TODO(preserve-comments): Should we attach this to expression or statement node? + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on next line, no semicolon", func(t *testing.T) { t.Parallel() diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index 22b2b4618..2632d51c0 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -39,9 +39,10 @@ import ( // | /* no execute or postConditions */ // ) // '}' -func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionDeclaration, error) { +func parseTransactionDeclaration(p *parser, startComments []*ast.Comment) (*ast.TransactionDeclaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos // Skip the `transaction` keyword p.nextSemanticToken() @@ -58,7 +59,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD } } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err @@ -76,14 +77,14 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var prepare *ast.SpecialFunctionDeclaration var execute *ast.SpecialFunctionDeclaration - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenIdentifier) { keyword := p.currentTokenSource() switch string(keyword) { case KeywordPrepare: - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current // Skip the `prepare` keyword p.next() prepare, err = parseSpecialFunctionDeclaration( @@ -95,8 +96,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD nil, nil, nil, - identifier, - "", + identifierToken, ) if err != nil { return nil, err @@ -123,7 +123,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var preConditions *ast.Conditions if execute == nil { - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { // Skip the `pre` keyword p.next() @@ -145,7 +145,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD sawPost := false atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -196,6 +196,10 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD } } + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewTransactionDeclaration( p.memoryGauge, parameterList, @@ -204,20 +208,21 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, startPos, endPos, ), + ast.Comments{ + Leading: leadingComments, + }, ), nil } func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err error) { for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -238,7 +243,7 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro nil, nil, nil, - docString, + nil, ) if err != nil { return nil, err @@ -258,7 +263,8 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro } func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) { - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current + identifier := p.tokenToIdentifier(identifierToken) // Skip the `execute` keyword p.nextSemanticToken() @@ -271,7 +277,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) return ast.NewSpecialFunctionDeclaration( p.memoryGauge, common.DeclarationKindExecute, - ast.NewFunctionDeclaration( + ast.NewFunctionDeclarationWithComments( p.memoryGauge, ast.AccessNotSpecified, ast.FunctionPurityUnspecified, @@ -288,7 +294,9 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - "", + ast.Comments{ + Leading: identifierToken.Leading, + }, ), ), nil } diff --git a/runtime/parser/type.go b/runtime/parser/type.go index a9df066c6..ce8ee852f 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -155,12 +155,12 @@ func init() { func defineParenthesizedTypes() { setTypeNullDenotation(lexer.TokenParenOpen, func(p *parser, token lexer.Token) (ast.Type, error) { - p.skipSpaceAndComments() + p.skipSpace() innerType, err := parseType(p, lowestBindingPower) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenParenClose) return innerType, err }) @@ -168,12 +168,14 @@ func defineParenthesizedTypes() { func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, error) { var nestedIdentifiers []ast.Identifier + // Stores the last token at the end of the loop + var nestedToken *lexer.Token for p.current.Is(lexer.TokenDot) { // Skip the dot p.next() - nestedToken := p.current + nestedToken = &p.current if !nestedToken.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( @@ -183,7 +185,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, ) } - nestedIdentifier := p.tokenToIdentifier(nestedToken) + nestedIdentifier := p.tokenToIdentifier(*nestedToken) // Skip the identifier p.next() @@ -195,10 +197,21 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, } - return ast.NewNominalType( + var trailingComments []*ast.Comment + if nestedToken == nil { + trailingComments = token.Comments.Trailing + } else { + trailingComments = nestedToken.Comments.PackToList() + } + + return ast.NewNominalTypeWithComments( p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, + ast.Comments{ + Leading: token.Comments.Leading, + Trailing: trailingComments, + }, ), nil } @@ -212,7 +225,7 @@ func defineArrayType() { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var size *ast.IntegerExpression @@ -241,7 +254,7 @@ func defineArrayType() { } } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBracketClose) if err != nil { @@ -341,7 +354,7 @@ func defineIntersectionOrDictionaryType() { expectType := true for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -537,7 +550,7 @@ func parseNominalTypes( expectType := true atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case separator: @@ -588,7 +601,7 @@ func parseNominalTypes( func parseParameterTypeAnnotations(p *parser) (typeAnnotations []*ast.TypeAnnotation, err error) { - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenParenOpen) if err != nil { return @@ -598,7 +611,7 @@ func parseParameterTypeAnnotations(p *parser) (typeAnnotations []*ast.TypeAnnota atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: if expectTypeAnnotation { @@ -656,7 +669,7 @@ func parseType(p *parser, rightBindingPower int) (ast.Type, error) { p.typeDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -729,15 +742,22 @@ func defaultTypeMetaLeftDenotation( } func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { - p.skipSpaceAndComments() + p.skipSpace() - startPos := p.current.StartPos + startToken := p.current isResource := false if p.current.Is(lexer.TokenAt) { // Skip the `@` p.next() isResource = true + + // Append @ token comments to type token, + // so that we don't need to store them in TypeAnnotation node. + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startToken.Comments.Leading...) + leadingComments = append(leadingComments, p.current.Comments.Leading...) + p.current.Comments.Leading = leadingComments } ty, err := parseType(p, lowestBindingPower) @@ -749,7 +769,7 @@ func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { p.memoryGauge, isResource, ty, - startPos, + startToken.StartPos, ), nil } @@ -771,7 +791,7 @@ func applyTypeLeftDenotation(p *parser, token lexer.Token, left ast.Type) (ast.T } func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, error) { - p.skipSpaceAndComments() + p.skipSpace() identifier, err := p.mustOne(lexer.TokenIdentifier) if err != nil { return nil, err @@ -782,7 +802,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, return nil, err } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken, err := p.mustOne(lexer.TokenParenOpen) if err != nil { return nil, err @@ -830,7 +850,7 @@ func parseCommaSeparatedTypeAnnotations( expectTypeAnnotation := true atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -912,14 +932,14 @@ func defineIdentifierTypes() { func(p *parser, token lexer.Token) (ast.Type, error) { switch string(p.tokenSource(token)) { case KeywordAuth: - p.skipSpaceAndComments() + p.skipSpace() var authorization ast.Authorization if p.current.Is(lexer.TokenParenOpen) { p.next() - p.skipSpaceAndComments() + p.skipSpace() var err error authorization, err = parseAuthorization(p) @@ -930,7 +950,7 @@ func defineIdentifierTypes() { p.reportSyntaxError("expected authorization (entitlement list)") } - p.skipSpaceAndComments() + p.skipSpace() _, err := p.mustOne(lexer.TokenAmpersand) if err != nil { @@ -950,7 +970,7 @@ func defineIdentifierTypes() { ), nil case KeywordFun: - p.skipSpaceAndComments() + p.skipSpace() return parseFunctionType(p, token.StartPos, ast.FunctionPurityUnspecified) case KeywordView: @@ -959,7 +979,7 @@ func defineIdentifierTypes() { cursor := p.tokens.Cursor() // look ahead for the `fun` keyword, if it exists - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // skip the `fun` keyword @@ -990,7 +1010,7 @@ func parseAuthorization(p *parser) (auth ast.Authorization, err error) { return nil, err } auth = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) diff --git a/runtime/sema/check_composite_declaration.go b/runtime/sema/check_composite_declaration.go index 5bc46ecd5..1585a16c1 100644 --- a/runtime/sema/check_composite_declaration.go +++ b/runtime/sema/check_composite_declaration.go @@ -1013,7 +1013,7 @@ func (checker *Checker) declareContractValue( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: compositeType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: contracts are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindContract, @@ -1085,7 +1085,7 @@ func (checker *Checker) declareEnumConstructor( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: constructorType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: enums are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindEnum, @@ -1845,7 +1845,7 @@ func (checker *Checker) defaultMembersAndOrigins( TypeAnnotation: fieldTypeAnnotation, VariableKind: ast.VariableKindConstant, ArgumentLabels: argumentLabels, - DocString: function.DocString, + DocString: function.DeclarationDocString(), HasImplementation: hasImplementation, HasConditions: hasConditions, }) diff --git a/runtime/sema/check_function.go b/runtime/sema/check_function.go index 4a716a204..3ef555f3d 100644 --- a/runtime/sema/check_function.go +++ b/runtime/sema/check_function.go @@ -147,7 +147,7 @@ func (checker *Checker) declareFunctionDeclaration( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: functionType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: common.DeclarationKindFunction, pos: declaration.Identifier.Pos, diff --git a/runtime/sema/check_interface_declaration.go b/runtime/sema/check_interface_declaration.go index bfef6c742..458e0a302 100644 --- a/runtime/sema/check_interface_declaration.go +++ b/runtime/sema/check_interface_declaration.go @@ -286,7 +286,7 @@ func (checker *Checker) declareInterfaceType(declaration *ast.InterfaceDeclarati ty: interfaceType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) checker.report(err) @@ -485,7 +485,7 @@ func (checker *Checker) declareEntitlementType(declaration *ast.EntitlementDecla ty: entitlementType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) @@ -532,7 +532,7 @@ func (checker *Checker) declareEntitlementMappingType(declaration *ast.Entitleme ty: entitlementMapType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) diff --git a/runtime/sema/check_switch.go b/runtime/sema/check_switch.go index 84d6670f2..054c02264 100644 --- a/runtime/sema/check_switch.go +++ b/runtime/sema/check_switch.go @@ -185,6 +185,7 @@ func (checker *Checker) checkSwitchCaseStatements(switchCase *ast.SwitchCase) { switchCase.Statements[0].StartPosition(), switchCase.EndPos, ), + ast.Comments{}, ) checker.checkBlock(block) } diff --git a/runtime/sema/check_variable_declaration.go b/runtime/sema/check_variable_declaration.go index b45141626..fb26450aa 100644 --- a/runtime/sema/check_variable_declaration.go +++ b/runtime/sema/check_variable_declaration.go @@ -225,7 +225,7 @@ func (checker *Checker) declareVariableDeclaration(declaration *ast.VariableDecl variable, err := checker.valueActivations.declare(variableDeclaration{ identifier: identifier, ty: declarationType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: declaration.DeclarationKind(), pos: declaration.Identifier.Pos, diff --git a/runtime/sema/positioninfo.go b/runtime/sema/positioninfo.go index 8c4b19dc5..53f4fb0c0 100644 --- a/runtime/sema/positioninfo.go +++ b/runtime/sema/positioninfo.go @@ -111,7 +111,7 @@ func (i *PositionInfo) recordFunctionDeclarationOrigin( DeclarationKind: common.DeclarationKindFunction, StartPos: &startPosition, EndPos: &endPosition, - DocString: function.DocString, + DocString: function.DeclarationDocString(), } i.Occurrences.Put( @@ -246,7 +246,7 @@ func (i *PositionInfo) recordVariableDeclarationRange( Identifier: identifier, DeclarationKind: declaration.DeclarationKind(), Type: declarationType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), }, ) } diff --git a/runtime/stdlib/test.go b/runtime/stdlib/test.go index eb65c1398..86fa74590 100644 --- a/runtime/stdlib/test.go +++ b/runtime/stdlib/test.go @@ -476,7 +476,7 @@ func TestCheckerContractValueHandler( return StandardLibraryValue{ Name: declaration.Identifier.Identifier, Type: constructorType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), Kind: declaration.DeclarationKind(), Position: &declaration.Identifier.Pos, ArgumentLabels: constructorArgumentLabels, diff --git a/tools/maprange/go.sum b/tools/maprange/go.sum index 05680b312..7a9273fb4 100644 --- a/tools/maprange/go.sum +++ b/tools/maprange/go.sum @@ -1,6 +1,11 @@ +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= diff --git a/tools/storage-explorer/go.mod b/tools/storage-explorer/go.mod index e29530edb..676751b7c 100644 --- a/tools/storage-explorer/go.mod +++ b/tools/storage-explorer/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( github.com/gorilla/mux v1.8.1 - github.com/onflow/atree v0.8.0-rc.6 + github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.0.0-preview.52 github.com/onflow/flow-go v0.37.10 github.com/rs/zerolog v1.32.0 diff --git a/tools/storage-explorer/go.sum b/tools/storage-explorer/go.sum index 54aea1efc..40fa2d4f2 100644 --- a/tools/storage-explorer/go.sum +++ b/tools/storage-explorer/go.sum @@ -1910,6 +1910,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= +github.com/onflow/atree v0.8.0/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY=