diff --git a/ast/access_test.go b/ast/access_test.go index cd706b04f3..518169f50f 100644 --- a/ast/access_test.go +++ b/ast/access_test.go @@ -43,7 +43,7 @@ func TestMappedAccess_MarshalJSON(t *testing.T) { t.Parallel() - e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}) + e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}, EmptyComments) access := NewMappedAccess(e, Position{Offset: 0, Line: 0, Column: 0}) actual, err := json.Marshal(access) @@ -52,6 +52,7 @@ func TestMappedAccess_MarshalJSON(t *testing.T) { assert.JSONEq(t, `{ "EntitlementMap": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -68,8 +69,8 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { t.Parallel() - e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 0, Line: 0, Column: 0}), []Identifier{}) - f := NewNominalType(nil, NewIdentifier(nil, "F", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}) + e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 0, Line: 0, Column: 0}), []Identifier{}, EmptyComments) + f := NewNominalType(nil, NewIdentifier(nil, "F", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}, EmptyComments) t.Run("conjunction", func(t *testing.T) { t.Parallel() @@ -82,6 +83,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { "ConjunctiveElements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -92,6 +94,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "F", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -115,6 +118,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { "DisjunctiveElements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -125,6 +129,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "F", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/attachment.go b/ast/attachment.go index 6eed6d4ce5..6a8aac9cf7 100644 --- a/ast/attachment.go +++ b/ast/attachment.go @@ -34,7 +34,7 @@ type AttachmentDeclaration struct { BaseType *NominalType Conformances []*NominalType Members *Members - DocString string + Comments Comments Range } @@ -50,8 +50,8 @@ func NewAttachmentDeclaration( baseType *NominalType, conformances []*NominalType, members *Members, - docString string, declarationRange Range, + comments Comments, ) *AttachmentDeclaration { common.UseMemory(memoryGauge, common.AttachmentDeclarationMemoryUsage) @@ -61,8 +61,8 @@ func NewAttachmentDeclaration( BaseType: baseType, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, + Comments: comments, } } @@ -99,7 +99,7 @@ func (d *AttachmentDeclaration) DeclarationMembers() *Members { } func (d *AttachmentDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (*AttachmentDeclaration) Kind() common.CompositeKind { @@ -193,9 +193,11 @@ func (d *AttachmentDeclaration) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Type string *Alias + DocString string }{ - Type: "AttachmentDeclaration", - Alias: (*Alias)(d), + Type: "AttachmentDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -290,6 +292,7 @@ type RemoveStatement struct { Attachment *NominalType Value Expression StartPos Position `json:"-"` + Comments Comments } var _ Element = &RemoveStatement{} @@ -300,6 +303,7 @@ func NewRemoveStatement( attachment *NominalType, value Expression, startPos Position, + comments Comments, ) *RemoveStatement { common.UseMemory(gauge, common.RemoveStatementMemoryUsage) @@ -307,6 +311,7 @@ func NewRemoveStatement( Attachment: attachment, Value: value, StartPos: startPos, + Comments: comments, } } diff --git a/ast/attachment_test.go b/ast/attachment_test.go index 02c79cd235..d19701bddf 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -46,6 +46,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { Position{Offset: 1, Line: 2, Column: 3}, ), []Identifier{}, + EmptyComments, ), Conformances: []*NominalType{ { @@ -56,8 +57,12 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ), }, }, - Members: NewMembers(nil, []Declaration{}), - DocString: "test", + Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 1, Line: 2, Column: 3}, EndPos: Position{Offset: 4, Line: 5, Column: 6}, @@ -72,6 +77,9 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ` { "Type": "AttachmentDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -82,6 +90,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { }, "BaseType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Bar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -94,6 +103,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Baz", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -138,6 +148,11 @@ func TestAttachmentDeclaration_Doc(t *testing.T) { }, }, Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, } require.Equal( @@ -260,6 +275,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "foo", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), Attachment: NewInvocationExpression( nil, @@ -270,6 +286,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "bar", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), []*TypeAnnotation{}, Arguments{}, @@ -291,6 +308,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "EndPos": {"Offset": 3, "Line": 2, "Column": 5}, "Base": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foo", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -303,6 +321,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "bar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -437,6 +456,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { Position{Offset: 1, Line: 2, Column: 3}, ), []Identifier{}, + EmptyComments, ), Value: NewIdentifierExpression( nil, @@ -445,6 +465,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { "baz", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), StartPos: Position{Offset: 1, Line: 2, Column: 3}, } @@ -457,10 +478,12 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { ` { "Type": "RemoveStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 3, "Line": 2, "Column": 5}, "Value": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "baz", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -471,6 +494,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { }, "Attachment": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -503,6 +527,7 @@ func TestRemoveStatement_Doc(t *testing.T) { Identifier{ Identifier: "baz", }, + Comments{}, ), } @@ -563,6 +588,7 @@ func TestRemoveStatement_String(t *testing.T) { Identifier{ Identifier: "baz", }, + Comments{}, ), } diff --git a/ast/block.go b/ast/block.go index be02b717ed..c4fadd817f 100644 --- a/ast/block.go +++ b/ast/block.go @@ -28,17 +28,24 @@ import ( type Block struct { Statements []Statement + Comments Comments Range } var _ Element = &Block{} -func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range) *Block { +func NewBlock( + memoryGauge common.MemoryGauge, + statements []Statement, + astRange Range, + comments Comments, +) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) return &Block{ Statements: statements, Range: astRange, + Comments: comments, } } diff --git a/ast/block_test.go b/ast/block_test.go index 7caf615bca..1dcfed5e32 100644 --- a/ast/block_test.go +++ b/ast/block_test.go @@ -57,6 +57,7 @@ func TestBlock_MarshalJSON(t *testing.T) { ` { "Type": "Block", + "Comments": {}, "Statements": [ { "Type": "ExpressionStatement", @@ -225,6 +226,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [ { "Type": "ExpressionStatement", @@ -329,6 +331,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} @@ -356,10 +359,12 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { }, { "Type": "EmitCondition", + "Comments": {}, "InvocationExpression": { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 31, "Line": 32, "Column": 33}, diff --git a/ast/comments.go b/ast/comments.go new file mode 100644 index 0000000000..a3ff567acc --- /dev/null +++ b/ast/comments.go @@ -0,0 +1,129 @@ +package ast + +import ( + "bytes" + "encoding/json" + "strings" + + "github.com/onflow/cadence/common" +) + +type Comments struct { + Leading []*Comment + Trailing []*Comment +} + +var EmptyComments = Comments{} + +// All combines Leading and Trailing comments in a single array. +func (c Comments) All() []*Comment { + var comments []*Comment + comments = append(comments, c.Leading...) + comments = append(comments, c.Trailing...) + return comments +} + +// LeadingDocString prints the leading doc comments to string +func (c Comments) LeadingDocString() string { + var s strings.Builder + for _, comment := range c.Leading { + if comment.IsDoc() { + if s.Len() > 0 { + s.WriteRune('\n') + } + s.Write(comment.Text()) + } + } + return s.String() +} + +type Comment struct { + source []byte +} + +func NewComment(memoryGauge common.MemoryGauge, source []byte) *Comment { + common.UseMemory(memoryGauge, common.NewRawStringMemoryUsage(len(source))) + return &Comment{ + source: source, + } +} + +var blockCommentDocStringPrefix = []byte("/**") +var blockCommentStringPrefix = []byte("/*") +var lineCommentDocStringPrefix = []byte("///") +var lineCommentStringPrefix = []byte("//") +var blockCommentStringSuffix = []byte("*/") + +func (c Comment) Multiline() bool { + return bytes.HasPrefix(c.source, blockCommentStringPrefix) +} + +func (c Comment) IsDoc() bool { + if c.Multiline() { + return bytes.HasPrefix(c.source, blockCommentDocStringPrefix) + } else { + return bytes.HasPrefix(c.source, lineCommentDocStringPrefix) + } +} + +var commentPrefixes = [][]byte{ + blockCommentDocStringPrefix, // must be before blockCommentStringPrefix + blockCommentStringPrefix, + lineCommentDocStringPrefix, // must be before lineCommentStringPrefix + lineCommentStringPrefix, +} + +var commentSuffixes = [][]byte{ + blockCommentStringSuffix, +} + +func (c Comment) String() string { + return string(c.source) +} + +// Text without opening/closing comment characters /*, /**, */, // +func (c Comment) Text() []byte { + withoutPrefixes := cutOptionalPrefixes(c.source, commentPrefixes) + return cutOptionalSuffixes(withoutPrefixes, commentSuffixes) +} + +func cutOptionalPrefixes(input []byte, prefixes [][]byte) (output []byte) { + output = input + for _, prefix := range prefixes { + cut, _ := bytes.CutPrefix(output, prefix) + output = cut + } + return +} + +func cutOptionalSuffixes(input []byte, suffixes [][]byte) (output []byte) { + output = input + for _, suffix := range suffixes { + cut, _ := bytes.CutSuffix(output, suffix) + output = cut + } + return +} + +func (c Comments) MarshalJSON() ([]byte, error) { + cj := struct { + Leading []string `json:"Leading,omitempty"` + Trailing []string `json:"Trailing,omitempty"` + }{} + + if len(c.Leading) > 0 { + cj.Leading = make([]string, len(c.Leading)) + for i, comment := range c.Leading { + cj.Leading[i] = comment.String() + } + } + + if len(c.Trailing) > 0 { + cj.Trailing = make([]string, len(c.Trailing)) + for i, comment := range c.Trailing { + cj.Trailing[i] = comment.String() + } + } + + return json.Marshal(cj) +} diff --git a/ast/composite.go b/ast/composite.go index e6d984fa20..bc080ddbab 100644 --- a/ast/composite.go +++ b/ast/composite.go @@ -51,12 +51,12 @@ func IsResourceDestructionDefaultEvent(identifier string) bool { type CompositeDeclaration struct { Members *Members - DocString string Conformances []*NominalType Identifier Identifier Range Access Access CompositeKind common.CompositeKind + Comments Comments } var _ Element = &CompositeDeclaration{} @@ -71,8 +71,8 @@ func NewCompositeDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declarationRange Range, + comments Comments, ) *CompositeDeclaration { common.UseMemory(memoryGauge, common.CompositeDeclarationMemoryUsage) @@ -82,8 +82,8 @@ func NewCompositeDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, + Comments: comments, } } @@ -120,17 +120,19 @@ func (d *CompositeDeclaration) DeclarationMembers() *Members { } func (d *CompositeDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *CompositeDeclaration) MarshalJSON() ([]byte, error) { type Alias CompositeDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "CompositeDeclaration", - Alias: (*Alias)(d), + Type: "CompositeDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -306,12 +308,12 @@ const ( type FieldDeclaration struct { TypeAnnotation *TypeAnnotation - DocString string Identifier Identifier Range Access Access VariableKind VariableKind Flags FieldDeclarationFlags + Comments Comments } var _ Element = &FieldDeclaration{} @@ -325,8 +327,8 @@ func NewFieldDeclaration( variableKind VariableKind, identifier Identifier, typeAnnotation *TypeAnnotation, - docString string, declRange Range, + comments Comments, ) *FieldDeclaration { common.UseMemory(memoryGauge, common.FieldDeclarationMemoryUsage) @@ -344,8 +346,8 @@ func NewFieldDeclaration( VariableKind: variableKind, Identifier: identifier, TypeAnnotation: typeAnnotation, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -377,23 +379,25 @@ func (d *FieldDeclaration) DeclarationMembers() *Members { } func (d *FieldDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *FieldDeclaration) MarshalJSON() ([]byte, error) { type Alias FieldDeclaration return json.Marshal(&struct { *Alias - Type string - Flags FieldDeclarationFlags `json:",omitempty"` - IsStatic bool - IsNative bool + Type string + Flags FieldDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + DocString string }{ - Type: "FieldDeclaration", - Alias: (*Alias)(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Flags: 0, + Type: "FieldDeclaration", + Alias: (*Alias)(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Flags: 0, + DocString: d.DeclarationDocString(), }) } @@ -494,10 +498,10 @@ func (d *FieldDeclaration) IsNative() bool { // EnumCaseDeclaration type EnumCaseDeclaration struct { - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access + Comments Comments } var _ Element = &EnumCaseDeclaration{} @@ -507,7 +511,7 @@ func NewEnumCaseDeclaration( memoryGauge common.MemoryGauge, access Access, identifier Identifier, - docString string, + comments Comments, startPos Position, ) *EnumCaseDeclaration { common.UseMemory(memoryGauge, common.EnumCaseDeclarationMemoryUsage) @@ -515,7 +519,7 @@ func NewEnumCaseDeclaration( return &EnumCaseDeclaration{ Access: access, Identifier: identifier, - DocString: docString, + Comments: comments, StartPos: startPos, } } @@ -555,7 +559,7 @@ func (d *EnumCaseDeclaration) DeclarationMembers() *Members { } func (d *EnumCaseDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EnumCaseDeclaration) MarshalJSON() ([]byte, error) { diff --git a/ast/composite_test.go b/ast/composite_test.go index 7387b766a0..671cc901e2 100644 --- a/ast/composite_test.go +++ b/ast/composite_test.go @@ -51,7 +51,11 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { }, StartPos: Position{Offset: 7, Line: 8, Column: 9}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 10, Line: 11, Column: 12}, EndPos: Position{Offset: 13, Line: 14, Column: 15}, @@ -66,6 +70,9 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "FieldDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "IsStatic": true, "IsNative": true, @@ -79,6 +86,7 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -445,8 +453,12 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -461,6 +473,9 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "CompositeDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -471,6 +486,7 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, diff --git a/ast/entitlement_declaration.go b/ast/entitlement_declaration.go index c1667588fd..dda3085459 100644 --- a/ast/entitlement_declaration.go +++ b/ast/entitlement_declaration.go @@ -30,8 +30,8 @@ import ( type EntitlementDeclaration struct { Access Access - DocString string Identifier Identifier + Comments Comments Range } @@ -43,16 +43,16 @@ func NewEntitlementDeclaration( gauge common.MemoryGauge, access Access, identifier Identifier, - docString string, declRange Range, + comments Comments, ) *EntitlementDeclaration { common.UseMemory(gauge, common.EntitlementDeclarationMemoryUsage) return &EntitlementDeclaration{ Access: access, Identifier: identifier, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -85,17 +85,19 @@ func (d *EntitlementDeclaration) DeclarationMembers() *Members { } func (d *EntitlementDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -165,9 +167,9 @@ func (d *EntitlementMapRelation) Doc() prettier.Doc { // EntitlementMappingDeclaration type EntitlementMappingDeclaration struct { Access Access - DocString string Identifier Identifier Elements []EntitlementMapElement + Comments Comments Range } @@ -180,8 +182,8 @@ func NewEntitlementMappingDeclaration( access Access, identifier Identifier, elements []EntitlementMapElement, - docString string, declRange Range, + comments Comments, ) *EntitlementMappingDeclaration { common.UseMemory(gauge, common.EntitlementMappingDeclarationMemoryUsage) @@ -189,8 +191,8 @@ func NewEntitlementMappingDeclaration( Access: access, Identifier: identifier, Elements: elements, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -241,17 +243,19 @@ func (d *EntitlementMappingDeclaration) DeclarationMembers() *Members { } func (d *EntitlementMappingDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementMappingDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementMappingDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementMappingDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementMappingDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/ast/entitlement_declaration_test.go b/ast/entitlement_declaration_test.go index 70c6a040a8..85d72e114c 100644 --- a/ast/entitlement_declaration_test.go +++ b/ast/entitlement_declaration_test.go @@ -37,7 +37,11 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -52,6 +56,9 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "Identifier": { "Identifier": "AB", @@ -131,7 +138,11 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -168,6 +179,9 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementMappingDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "Identifier": { "Identifier": "AB", @@ -177,6 +191,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { "Elements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -188,6 +203,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { { "Input": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -198,6 +214,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { }, "Output": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Y", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -230,7 +247,11 @@ func TestEntitlementMappingDeclaration_Doc(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -277,7 +298,6 @@ func TestEntitlementMappingDeclaration_Doc(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -351,7 +371,6 @@ func TestEntitlementMappingDeclaration_String(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -383,7 +402,11 @@ entitlement mapping AB { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/ast/expression.go b/ast/expression.go index 7189be445e..084f2de7f4 100644 --- a/ast/expression.go +++ b/ast/expression.go @@ -301,7 +301,8 @@ type IntegerExpression struct { Value *big.Int `json:"-"` PositiveLiteral []byte Range - Base int + Base int + Comments Comments } var _ Element = &IntegerExpression{} @@ -313,6 +314,7 @@ func NewIntegerExpression( value *big.Int, base int, tokenRange Range, + comments Comments, ) *IntegerExpression { common.UseMemory(gauge, common.IntegerExpressionMemoryUsage) @@ -321,6 +323,7 @@ func NewIntegerExpression( Value: value, Base: base, Range: tokenRange, + Comments: comments, } } @@ -674,6 +677,7 @@ func (e DictionaryEntry) Doc() prettier.Doc { type IdentifierExpression struct { Identifier Identifier + Comments Comments } var _ Element = &IdentifierExpression{} @@ -682,11 +686,13 @@ var _ Expression = &IdentifierExpression{} func NewIdentifierExpression( gauge common.MemoryGauge, identifier Identifier, + comments Comments, ) *IdentifierExpression { common.UseMemory(gauge, common.IdentifierExpressionMemoryUsage) return &IdentifierExpression{ Identifier: identifier, + Comments: comments, } } diff --git a/ast/expression_test.go b/ast/expression_test.go index 8ddd609c82..fcb33f98fd 100644 --- a/ast/expression_test.go +++ b/ast/expression_test.go @@ -202,6 +202,7 @@ func TestIntegerExpression_MarshalJSON(t *testing.T) { ` { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "4_2", "Value": "42", "Base": 10, @@ -929,6 +930,7 @@ func TestIdentifierExpression_MarshalJSON(t *testing.T) { ` { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1965,6 +1967,7 @@ func TestUnaryExpression_MarshalJSON(t *testing.T) { "Operation": "OperationNegate", "Expression": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -2229,6 +2232,7 @@ func TestBinaryExpression_MarshalJSON(t *testing.T) { "Operation": "OperationPlus", "Left": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -2237,6 +2241,7 @@ func TestBinaryExpression_MarshalJSON(t *testing.T) { }, "Right": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "99", "Value": "99", "Base": 10, @@ -2761,6 +2766,7 @@ func TestDestroyExpression_MarshalJSON(t *testing.T) { "Type": "DestroyExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -3005,6 +3011,7 @@ func TestForceExpression_MarshalJSON(t *testing.T) { "Type": "ForceExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -3251,6 +3258,7 @@ func TestConditionalExpression_MarshalJSON(t *testing.T) { }, "Then": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -3259,6 +3267,7 @@ func TestConditionalExpression_MarshalJSON(t *testing.T) { }, "Else": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "99", "Value": "99", "Base": 10, @@ -3834,6 +3843,7 @@ func TestInvocationExpression_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -3847,6 +3857,7 @@ func TestInvocationExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -4311,6 +4322,7 @@ func TestCastingExpression_MarshalJSON(t *testing.T) { "Type": "CastingExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -4324,6 +4336,7 @@ func TestCastingExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -4709,6 +4722,7 @@ func TestCreateExpression_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -4722,6 +4736,7 @@ func TestCreateExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -4865,6 +4880,7 @@ func TestReferenceExpression_MarshalJSON(t *testing.T) { "Type": "ReferenceExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -5162,8 +5178,10 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { { "Type": "FunctionExpression", "ParameterList": { + "Comments": {}, "Parameters": [ { + "Comments": {}, "Label": "ok", "Identifier": { "Identifier": "foobar", @@ -5174,6 +5192,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -5198,6 +5217,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -5213,6 +5233,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} diff --git a/ast/function_declaration.go b/ast/function_declaration.go index 1e334cebba..3824986cb2 100644 --- a/ast/function_declaration.go +++ b/ast/function_declaration.go @@ -69,11 +69,11 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access Flags FunctionDeclarationFlags + Comments Comments } var _ Element = &FunctionDeclaration{} @@ -92,7 +92,7 @@ func NewFunctionDeclaration( returnTypeAnnotation *TypeAnnotation, functionBlock *FunctionBlock, startPos Position, - docString string, + comments Comments, ) *FunctionDeclaration { common.UseMemory(gauge, common.FunctionDeclarationMemoryUsage) @@ -114,7 +114,7 @@ func NewFunctionDeclaration( ReturnTypeAnnotation: returnTypeAnnotation, FunctionBlock: functionBlock, StartPos: startPos, - DocString: docString, + Comments: comments, } } @@ -176,7 +176,7 @@ func (d *FunctionDeclaration) DeclarationMembers() *Members { } func (d *FunctionDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *FunctionDeclaration) Doc() prettier.Doc { @@ -200,16 +200,18 @@ func (d *FunctionDeclaration) MarshalJSON() ([]byte, error) { *Alias Type string Range - IsStatic bool - IsNative bool - Flags FunctionDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + Flags FunctionDeclarationFlags `json:",omitempty"` + DocString string }{ - Type: "FunctionDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Alias: (*Alias)(d), - Flags: 0, + Type: "FunctionDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Alias: (*Alias)(d), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/ast/function_declaration_test.go b/ast/function_declaration_test.go index d2d0e6db90..64ad3cf666 100644 --- a/ast/function_declaration_test.go +++ b/ast/function_declaration_test.go @@ -113,8 +113,10 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []*Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, } actual, err := json.Marshal(decl) @@ -126,6 +128,9 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { { "Type": "FunctionDeclaration", "Access": "AccessAll", + "Comments": { + "Leading": ["///test"] + }, "IsStatic": true, "IsNative": true, "Identifier": { @@ -179,6 +184,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": { "Offset": 46, "Line": 47, @@ -218,9 +224,11 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { } }, "ParameterList": { + "Comments": {}, "Parameters": [ { "Label": "ok", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -230,6 +238,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -254,6 +263,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -269,6 +279,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} @@ -585,8 +596,10 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []*Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, }, } @@ -602,6 +615,9 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "FunctionDeclaration": { "Type": "FunctionDeclaration", "Access": "AccessNotSpecified", + "Comments": { + "Leading": ["///test"] + }, "IsStatic": false, "IsNative": true, "Identifier": { @@ -655,6 +671,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": { "Offset": 46, "Line": 47, @@ -694,8 +711,10 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { } }, "ParameterList": { + "Comments": {}, "Parameters": [ { + "Comments": {}, "Label": "ok", "Identifier": { "Identifier": "foobar", @@ -706,6 +725,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -730,6 +750,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -745,6 +766,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} diff --git a/ast/identifier.go b/ast/identifier.go index bf7a804c56..8113562f9a 100644 --- a/ast/identifier.go +++ b/ast/identifier.go @@ -31,7 +31,11 @@ type Identifier struct { Pos Position } -func NewIdentifier(memoryGauge common.MemoryGauge, identifier string, pos Position) Identifier { +func NewIdentifier( + memoryGauge common.MemoryGauge, + identifier string, + pos Position, +) Identifier { common.UseMemory(memoryGauge, common.IdentifierMemoryUsage) return Identifier{ Identifier: identifier, diff --git a/ast/import.go b/ast/import.go index dd3022e278..2499a8be16 100644 --- a/ast/import.go +++ b/ast/import.go @@ -42,6 +42,7 @@ type ImportDeclaration struct { Imports []Import Range LocationPos Position + Comments Comments } var _ Element = &ImportDeclaration{} @@ -53,6 +54,7 @@ func NewImportDeclaration( location common.Location, declRange Range, locationPos Position, + comments Comments, ) *ImportDeclaration { common.UseMemory(gauge, common.ImportDeclarationMemoryUsage) @@ -61,6 +63,7 @@ func NewImportDeclaration( Location: location, Range: declRange, LocationPos: locationPos, + Comments: comments, } } diff --git a/ast/import_test.go b/ast/import_test.go index eb9869ae25..8db6cbdfe5 100644 --- a/ast/import_test.go +++ b/ast/import_test.go @@ -68,6 +68,7 @@ func TestImportDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "ImportDeclaration", + "Comments": {}, "Imports": [ { "Identifier": { diff --git a/ast/interface.go b/ast/interface.go index b551abe582..b7e98b87a3 100644 --- a/ast/interface.go +++ b/ast/interface.go @@ -30,12 +30,12 @@ import ( type InterfaceDeclaration struct { Members *Members - DocString string Identifier Identifier Conformances []*NominalType Range Access Access CompositeKind common.CompositeKind + Comments Comments } var _ Element = &InterfaceDeclaration{} @@ -49,8 +49,8 @@ func NewInterfaceDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declRange Range, + comments Comments, ) *InterfaceDeclaration { common.UseMemory(gauge, common.InterfaceDeclarationMemoryUsage) @@ -60,8 +60,8 @@ func NewInterfaceDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -96,17 +96,19 @@ func (d *InterfaceDeclaration) DeclarationMembers() *Members { } func (d *InterfaceDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *InterfaceDeclaration) MarshalJSON() ([]byte, error) { type Alias InterfaceDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "InterfaceDeclaration", - Alias: (*Alias)(d), + Type: "InterfaceDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/ast/interface_test.go b/ast/interface_test.go index 0fc268236d..41f2e50d89 100644 --- a/ast/interface_test.go +++ b/ast/interface_test.go @@ -42,8 +42,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -58,6 +62,9 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -95,8 +102,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -110,6 +121,9 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -120,6 +134,7 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, diff --git a/ast/parameter.go b/ast/parameter.go index de377e66c5..74ed9433d0 100644 --- a/ast/parameter.go +++ b/ast/parameter.go @@ -32,6 +32,7 @@ type Parameter struct { Label string Identifier Identifier StartPos Position `json:"-"` + Comments Comments } func NewParameter( @@ -41,6 +42,7 @@ func NewParameter( typeAnnotation *TypeAnnotation, defaultArgument Expression, startPos Position, + comments Comments, ) *Parameter { common.UseMemory(gauge, common.ParameterMemoryUsage) return &Parameter{ @@ -49,6 +51,7 @@ func NewParameter( TypeAnnotation: typeAnnotation, DefaultArgument: defaultArgument, StartPos: startPos, + Comments: comments, } } diff --git a/ast/parameterlist.go b/ast/parameterlist.go index 1dfcb7f26f..4131f27b38 100644 --- a/ast/parameterlist.go +++ b/ast/parameterlist.go @@ -29,6 +29,7 @@ import ( type ParameterList struct { parametersByIdentifier atomic.Pointer[map[string]*Parameter] Parameters []*Parameter + Comments Comments Range } @@ -36,11 +37,13 @@ func NewParameterList( gauge common.MemoryGauge, parameters []*Parameter, astRange Range, + comments Comments, ) *ParameterList { common.UseMemory(gauge, common.ParameterListMemoryUsage) return &ParameterList{ Parameters: parameters, Range: astRange, + Comments: comments, } } diff --git a/ast/position.go b/ast/position.go index 0594f2df3e..6d4b86f4b5 100644 --- a/ast/position.go +++ b/ast/position.go @@ -207,6 +207,12 @@ func (r Range) EndPosition(common.MemoryGauge) Position { return r.EndPos } +func (r Range) Source(input []byte) []byte { + startOffset := r.StartPos.Offset + endOffset := r.EndPos.Offset + 1 + return input[startOffset:endOffset] +} + // NewRangeFromPositioned func NewRangeFromPositioned(memoryGauge common.MemoryGauge, hasPosition HasPosition) Range { diff --git a/ast/statement.go b/ast/statement.go index 872adf8b0e..b328ade3e0 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -38,17 +38,24 @@ type Statement interface { type ReturnStatement struct { Expression Expression + Comments Comments Range } var _ Element = &ReturnStatement{} var _ Statement = &ReturnStatement{} -func NewReturnStatement(gauge common.MemoryGauge, expression Expression, stmtRange Range) *ReturnStatement { +func NewReturnStatement( + gauge common.MemoryGauge, + expression Expression, + stmtRange Range, + comments Comments, +) *ReturnStatement { common.UseMemory(gauge, common.ReturnStatementMemoryUsage) return &ReturnStatement{ Expression: expression, Range: stmtRange, + Comments: comments, } } @@ -96,16 +103,18 @@ func (s *ReturnStatement) MarshalJSON() ([]byte, error) { // BreakStatement type BreakStatement struct { + Comments Comments Range } var _ Element = &BreakStatement{} var _ Statement = &BreakStatement{} -func NewBreakStatement(gauge common.MemoryGauge, tokenRange Range) *BreakStatement { +func NewBreakStatement(gauge common.MemoryGauge, tokenRange Range, comments Comments) *BreakStatement { common.UseMemory(gauge, common.BreakStatementMemoryUsage) return &BreakStatement{ - Range: tokenRange, + Range: tokenRange, + Comments: comments, } } @@ -143,16 +152,18 @@ func (s *BreakStatement) MarshalJSON() ([]byte, error) { // ContinueStatement type ContinueStatement struct { + Comments Comments Range } var _ Element = &ContinueStatement{} var _ Statement = &ContinueStatement{} -func NewContinueStatement(gauge common.MemoryGauge, tokenRange Range) *ContinueStatement { +func NewContinueStatement(gauge common.MemoryGauge, tokenRange Range, comments Comments) *ContinueStatement { common.UseMemory(gauge, common.ContinueStatementMemoryUsage) return &ContinueStatement{ - Range: tokenRange, + Range: tokenRange, + Comments: comments, } } @@ -202,6 +213,8 @@ type IfStatement struct { Then *Block Else *Block StartPos Position `json:"-"` + // Comments.Leading comments that appear before `if` keyword + Comments Comments `json:"-"` } var _ Element = &IfStatement{} @@ -213,6 +226,7 @@ func NewIfStatement( thenBlock *Block, elseBlock *Block, startPos Position, + comments Comments, ) *IfStatement { common.UseMemory(gauge, common.IfStatementMemoryUsage) return &IfStatement{ @@ -220,6 +234,7 @@ func NewIfStatement( Then: thenBlock, Else: elseBlock, StartPos: startPos, + Comments: comments, } } @@ -308,6 +323,7 @@ type WhileStatement struct { Test Expression Block *Block StartPos Position `json:"-"` + Comments Comments } var _ Element = &WhileStatement{} @@ -318,12 +334,14 @@ func NewWhileStatement( expression Expression, block *Block, startPos Position, + comments Comments, ) *WhileStatement { common.UseMemory(gauge, common.WhileStatementMemoryUsage) return &WhileStatement{ Test: expression, Block: block, StartPos: startPos, + Comments: comments, } } @@ -384,6 +402,7 @@ type ForStatement struct { Block *Block Identifier Identifier StartPos Position `json:"-"` + Comments Comments } var _ Element = &ForStatement{} @@ -396,6 +415,7 @@ func NewForStatement( block *Block, expression Expression, startPos Position, + comments Comments, ) *ForStatement { common.UseMemory(gauge, common.ForStatementMemoryUsage) @@ -405,6 +425,7 @@ func NewForStatement( Block: block, Value: expression, StartPos: startPos, + Comments: comments, } } @@ -479,6 +500,7 @@ func (s *ForStatement) MarshalJSON() ([]byte, error) { type EmitStatement struct { InvocationExpression *InvocationExpression StartPos Position `json:"-"` + Comments Comments } var _ Element = &EmitStatement{} @@ -488,11 +510,13 @@ func NewEmitStatement( gauge common.MemoryGauge, invocation *InvocationExpression, startPos Position, + comments Comments, ) *EmitStatement { common.UseMemory(gauge, common.EmitStatementMemoryUsage) return &EmitStatement{ InvocationExpression: invocation, StartPos: startPos, + Comments: comments, } } @@ -744,6 +768,7 @@ func (s *ExpressionStatement) String() string { type SwitchStatement struct { Expression Expression Cases []*SwitchCase + Comments Comments Range } @@ -755,12 +780,14 @@ func NewSwitchStatement( expression Expression, cases []*SwitchCase, stmtRange Range, + comments Comments, ) *SwitchStatement { common.UseMemory(gauge, common.SwitchStatementMemoryUsage) return &SwitchStatement{ Expression: expression, Cases: cases, Range: stmtRange, + Comments: comments, } } @@ -840,6 +867,7 @@ func (s *SwitchStatement) MarshalJSON() ([]byte, error) { type SwitchCase struct { Expression Expression Statements []Statement + Comments Comments Range } @@ -848,12 +876,14 @@ func NewSwitchCase( expression Expression, statements []Statement, astRange Range, + comments Comments, ) *SwitchCase { common.UseMemory(gauge, common.SwitchCaseMemoryUsage) return &SwitchCase{ Expression: expression, Statements: statements, Range: astRange, + Comments: comments, } } diff --git a/ast/statement_test.go b/ast/statement_test.go index ebbd681b17..39b36b306e 100644 --- a/ast/statement_test.go +++ b/ast/statement_test.go @@ -156,6 +156,7 @@ func TestReturnStatement_MarshalJSON(t *testing.T) { ` { "Type": "ReturnStatement", + "Comments": {}, "Expression": { "Type": "BoolExpression", "Value": false, @@ -258,6 +259,7 @@ func TestBreakStatement_MarshalJSON(t *testing.T) { ` { "Type": "BreakStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} } @@ -305,6 +307,7 @@ func TestContinueStatement_MarshalJSON(t *testing.T) { ` { "Type": "ContinueStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} } @@ -378,12 +381,14 @@ func TestIfStatement_MarshalJSON(t *testing.T) { }, "Then": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, "EndPos": {"Offset": 10, "Line": 11, "Column": 12} }, "Else": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 13, "Line": 14, "Column": 15}, "EndPos": {"Offset": 16, "Line": 17, "Column": 18} @@ -598,6 +603,7 @@ func TestWhileStatement_MarshalJSON(t *testing.T) { ` { "Type": "WhileStatement", + "Comments": {}, "Test": { "Type": "BoolExpression", "Value": false, @@ -606,6 +612,7 @@ func TestWhileStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, "EndPos": {"Offset": 10, "Line": 11, "Column": 12} @@ -738,6 +745,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { ` { "Type": "ForStatement", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -752,6 +760,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, "EndPos": {"Offset": 13, "Line": 14, "Column": 15} @@ -802,6 +811,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { ` { "Type": "ForStatement", + "Comments": {}, "Index": { "Identifier": "i", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -820,6 +830,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos":{"Offset": 13, "Line": 14, "Column": 15}, "EndPos": {"Offset": 16, "Line": 17, "Column": 18} @@ -1025,6 +1036,7 @@ func TestAssignmentStatement_MarshalJSON(t *testing.T) { "Type": "AssignmentStatement", "Target": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1185,6 +1197,7 @@ func TestSwapStatement_MarshalJSON(t *testing.T) { "Type": "SwapStatement", "Left": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1344,10 +1357,12 @@ func TestEmitStatement_MarshalJSON(t *testing.T) { ` { "Type": "EmitStatement", + "Comments": {}, "InvocationExpression": { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1361,6 +1376,7 @@ func TestEmitStatement_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -1546,8 +1562,10 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { ` { "Type": "SwitchStatement", + "Comments": {}, "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foo", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1559,6 +1577,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "Cases": [ { "Type": "SwitchCase", + "Comments": {}, "Expression": { "Type": "BoolExpression", "Value": false, @@ -1572,6 +1591,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "EndPos": {"Offset": 12, "Line": 11, "Column": 14}, "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "bar", "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, @@ -1587,6 +1607,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { }, { "Type": "SwitchCase", + "Comments": {}, "Expression": null, "Statements": [ { @@ -1595,6 +1616,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "EndPos": {"Offset": 21, "Line": 20, "Column": 23}, "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "baz", "StartPos": {"Offset": 19, "Line": 20, "Column": 21}, diff --git a/ast/transaction_declaration.go b/ast/transaction_declaration.go index aad73b308f..7355ef57cb 100644 --- a/ast/transaction_declaration.go +++ b/ast/transaction_declaration.go @@ -34,6 +34,7 @@ type TransactionDeclaration struct { PostConditions *Conditions DocString string Fields []*FieldDeclaration + Comments Comments Range } @@ -49,8 +50,8 @@ func NewTransactionDeclaration( preConditions *Conditions, postConditions *Conditions, execute *SpecialFunctionDeclaration, - docString string, declRange Range, + comments Comments, ) *TransactionDeclaration { common.UseMemory(gauge, common.TransactionDeclarationMemoryUsage) @@ -61,8 +62,8 @@ func NewTransactionDeclaration( PreConditions: preConditions, PostConditions: postConditions, Execute: execute, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -104,17 +105,19 @@ func (d *TransactionDeclaration) DeclarationMembers() *Members { } func (d *TransactionDeclaration) DeclarationDocString() string { - return "" + return d.Comments.LeadingDocString() } func (d *TransactionDeclaration) MarshalJSON() ([]byte, error) { type Alias TransactionDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "TransactionDeclaration", - Alias: (*Alias)(d), + Type: "TransactionDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/ast/transaction_declaration_test.go b/ast/transaction_declaration_test.go index 15d0a91f91..061eb71730 100644 --- a/ast/transaction_declaration_test.go +++ b/ast/transaction_declaration_test.go @@ -55,8 +55,12 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { EndPos: Position{Offset: 22, Line: 23, Column: 24}, }, }, - DocString: "test", - Execute: nil, + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, + Execute: nil, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -71,7 +75,11 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "TransactionDeclaration", + "Comments": { + "Leading": ["///test"] + }, "ParameterList": { + "Comments": {}, "Parameters": [], "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} diff --git a/ast/type.go b/ast/type.go index 991c9f51e5..0e54c7d8c0 100644 --- a/ast/type.go +++ b/ast/type.go @@ -111,6 +111,7 @@ func IsEmptyType(t Type) bool { type NominalType struct { NestedIdentifiers []Identifier `json:",omitempty"` Identifier Identifier + Comments Comments } var _ Type = &NominalType{} @@ -119,11 +120,13 @@ func NewNominalType( memoryGauge common.MemoryGauge, identifier Identifier, nestedIdentifiers []Identifier, + comments Comments, ) *NominalType { common.UseMemory(memoryGauge, common.NominalTypeMemoryUsage) return &NominalType{ Identifier: identifier, NestedIdentifiers: nestedIdentifiers, + Comments: comments, } } diff --git a/ast/type_test.go b/ast/type_test.go index a0cc161586..5f3c748409 100644 --- a/ast/type_test.go +++ b/ast/type_test.go @@ -187,6 +187,7 @@ func TestTypeAnnotation_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -330,6 +331,7 @@ func TestNominalType_MarshalJSON(t *testing.T) { ` { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -446,6 +448,7 @@ func TestOptionalType_MarshalJSON(t *testing.T) { "Type": "OptionalType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -579,6 +582,7 @@ func TestVariableSizedType_MarshalJSON(t *testing.T) { "Type": "VariableSizedType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -831,6 +835,7 @@ func TestConstantSizedType_MarshalJSON(t *testing.T) { "Type": "ConstantSizedType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -841,6 +846,7 @@ func TestConstantSizedType_MarshalJSON(t *testing.T) { }, "Size": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -1090,6 +1096,7 @@ func TestDictionaryType_MarshalJSON(t *testing.T) { "Type": "DictionaryType", "KeyType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1100,6 +1107,7 @@ func TestDictionaryType_MarshalJSON(t *testing.T) { }, "ValueType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -1331,6 +1339,7 @@ func TestFunctionType_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1348,6 +1357,7 @@ func TestFunctionType_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, @@ -1858,8 +1868,9 @@ func TestReferenceType_MarshalJSON(t *testing.T) { "LegacyAuthorized": false, "Authorization": { "ConjunctiveElements": [ - { + { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -1868,8 +1879,9 @@ func TestReferenceType_MarshalJSON(t *testing.T) { "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, "EndPos": {"Offset": 0, "Line": 0, "Column": 0} }, - { + { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Y", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -1882,6 +1894,7 @@ func TestReferenceType_MarshalJSON(t *testing.T) { }, "ReferencedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -2088,6 +2101,7 @@ func TestIntersectionType_MarshalJSON(t *testing.T) { "Types": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -2098,6 +2112,7 @@ func TestIntersectionType_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "EF", "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, @@ -2402,6 +2417,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "Type": "InstantiationType", "InstantiatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -2415,6 +2431,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -2430,6 +2447,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "EF", "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, diff --git a/ast/variable_declaration.go b/ast/variable_declaration.go index 5412584614..ce14eea8fe 100644 --- a/ast/variable_declaration.go +++ b/ast/variable_declaration.go @@ -33,11 +33,11 @@ type VariableDeclaration struct { Transfer *Transfer SecondTransfer *Transfer ParentIfStatement *IfStatement `json:"-"` - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access IsConstant bool + Comments Comments } var _ Element = &VariableDeclaration{} @@ -55,7 +55,7 @@ func NewVariableDeclaration( startPos Position, secondTransfer *Transfer, secondValue Expression, - docString string, + comments Comments, ) *VariableDeclaration { common.UseMemory(gauge, common.VariableDeclarationMemoryUsage) @@ -69,7 +69,7 @@ func NewVariableDeclaration( StartPos: startPos, SecondTransfer: secondTransfer, SecondValue: secondValue, - DocString: docString, + Comments: comments, } } @@ -134,7 +134,7 @@ func (d *VariableDeclaration) DeclarationMembers() *Members { } func (d *VariableDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } var varKeywordDoc prettier.Doc = prettier.Text("var") @@ -240,12 +240,14 @@ func (d *VariableDeclaration) MarshalJSON() ([]byte, error) { type Alias VariableDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string Range }{ - Type: "VariableDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - Alias: (*Alias)(d), + Type: "VariableDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/ast/variable_declaration_test.go b/ast/variable_declaration_test.go index 902632a3b4..a971a91dbc 100644 --- a/ast/variable_declaration_test.go +++ b/ast/variable_declaration_test.go @@ -71,7 +71,11 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { EndPos: Position{Offset: 28, Line: 29, Column: 30}, }, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, } actual, err := json.Marshal(decl) @@ -83,6 +87,9 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { { "Type": "VariableDeclaration", "Access": "AccessAll", + "Comments": { + "Leading": ["///test"] + }, "IsConstant": true, "Identifier": { "Identifier": "foo", @@ -95,6 +102,7 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { diff --git a/bbq/compiler/desugar.go b/bbq/compiler/desugar.go index 9d295489e8..2d78e59fb8 100644 --- a/bbq/compiler/desugar.go +++ b/bbq/compiler/desugar.go @@ -217,7 +217,7 @@ func (d *Desugar) VisitFunctionDeclaration(declaration *ast.FunctionDeclaration, declaration.ReturnTypeAnnotation, modifiedFuncBlock, declaration.StartPos, - declaration.DocString, + declaration.Comments, ) d.elaboration.SetFunctionDeclarationFunctionType(modifiedDecl, functionType) @@ -297,6 +297,7 @@ func (d *Desugar) desugarFunctionBlock( d.memoryGauge, modifiedStatements, ast.NewRangeFromPositioned(d.memoryGauge, hasPosition), + ast.EmptyComments, ), nil, nil, @@ -329,7 +330,7 @@ func (d *Desugar) tempResultVariable( pos, nil, nil, - "", + ast.EmptyComments, ) d.elaboration.SetVariableDeclarationTypes( @@ -392,7 +393,7 @@ func (d *Desugar) declareResultVariable( pos, nil, nil, - "", + ast.EmptyComments, ) d.elaboration.SetVariableDeclarationTypes( @@ -416,6 +417,7 @@ func (d *Desugar) tempResultIdentifierExpr(pos ast.Position) *ast.IdentifierExpr tempResultVariableName, pos, ), + ast.EmptyComments, ) } @@ -717,6 +719,7 @@ func (d *Desugar) desugarCondition( functionName, startPos, ), + ast.EmptyComments, ), nil, []*ast.Argument{ @@ -748,9 +751,11 @@ func (d *Desugar) desugarCondition( d.memoryGauge, condition.Test, ), + ast.EmptyComments, ), nil, startPos, + ast.EmptyComments, ) return ifStmt @@ -798,6 +803,7 @@ func (d *Desugar) desugarCondition( declaredContract.GetIdentifier(), pos, ), + ast.EmptyComments, ), false, pos, @@ -821,6 +827,7 @@ func (d *Desugar) desugarCondition( d.memoryGauge, newEventConstructorInvocation, emitStmt.StartPos, + ast.EmptyComments, ) //Inject a static import so the compiler can link the functions. @@ -982,8 +989,8 @@ func (d *Desugar) VisitAttachmentDeclaration(declaration *ast.AttachmentDeclarat declaration.BaseType, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1104,8 +1111,8 @@ func (d *Desugar) VisitCompositeDeclaration(declaration *ast.CompositeDeclaratio declaration.Identifier, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1270,7 +1277,7 @@ func (d *Desugar) inheritedDefaultFunctions( ) funcReturnType := inheritedFuncType.ReturnTypeAnnotation.Type - returnStmt := ast.NewReturnStatement(d.memoryGauge, invocation, declRange) + returnStmt := ast.NewReturnStatement(d.memoryGauge, invocation, declRange, ast.EmptyComments) d.elaboration.SetReturnStatementTypes( returnStmt, sema.ReturnStatementTypes{ @@ -1305,12 +1312,13 @@ func (d *Desugar) inheritedDefaultFunctions( returnStmt, }, declRange, + ast.EmptyComments, ), nil, nil, ), inheritedFunc.StartPos, - inheritedFunc.DocString, + inheritedFunc.Comments, ) d.elaboration.SetFunctionDeclarationFunctionType(defaultFuncDelegator, inheritedFuncType) @@ -1380,6 +1388,7 @@ func (d *Desugar) interfaceDelegationMethodCall( param.Identifier, pos, ), + ast.EmptyComments, ), ) } else { @@ -1402,6 +1411,7 @@ func (d *Desugar) interfaceDelegationMethodCall( param.Identifier, pos, ), + ast.EmptyComments, ), ) } @@ -1427,12 +1437,12 @@ func (d *Desugar) interfaceDelegationMethodCall( d.memoryGauge, ast.NewIdentifierExpression( d.memoryGauge, - ast.NewIdentifier( d.memoryGauge, "self", pos, ), + ast.EmptyComments, ), false, pos, @@ -1526,6 +1536,7 @@ func (d *Desugar) addImport(location common.Location) { location, ast.EmptyRange, ast.EmptyPosition, + ast.EmptyComments, ) d.newImports = append( @@ -1564,8 +1575,8 @@ func (d *Desugar) VisitInterfaceDeclaration(declaration *ast.InterfaceDeclaratio declaration.Identifier, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1606,7 +1617,7 @@ func (d *Desugar) VisitTransactionDeclaration(transaction *ast.TransactionDeclar parameter.StartPos, nil, nil, - "", + ast.EmptyComments, ) varDeclarations = append(varDeclarations, variableDecl) @@ -1672,8 +1683,8 @@ func (d *Desugar) VisitTransactionDeclaration(transaction *ast.TransactionDeclar ), nil, ast.NewMembers(d.memoryGauge, members), - "", ast.EmptyRange, + ast.EmptyComments, ) compositeType := d.transactionCompositeType() @@ -1786,12 +1797,12 @@ var emptyInitializer = func() *ast.SpecialFunctionDeclaration { nil, ast.NewFunctionBlock( nil, - ast.NewBlock(nil, nil, ast.EmptyRange), + ast.NewBlock(nil, nil, ast.EmptyRange, ast.EmptyComments), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) return ast.NewSpecialFunctionDeclaration( @@ -1844,11 +1855,13 @@ func newEnumInitializer( ast.EmptyPosition, ), nil, + ast.EmptyComments, ), ast.EmptyPosition, ), nil, ast.EmptyPosition, + ast.EmptyComments, ), } @@ -1861,6 +1874,7 @@ func newEnumInitializer( sema.SelfIdentifier, ast.EmptyPosition, ), + ast.EmptyComments, ), false, ast.EmptyPosition, @@ -1895,6 +1909,7 @@ func newEnumInitializer( ast.NewIdentifierExpression( gauge, rawValueIdentifier, + ast.EmptyComments, ), ) @@ -1918,7 +1933,7 @@ func newEnumInitializer( ast.EmptyPosition, ), nil, - ast.NewParameterList(gauge, parameters, ast.EmptyRange), + ast.NewParameterList(gauge, parameters, ast.EmptyRange, ast.EmptyComments), nil, ast.NewFunctionBlock( gauge, @@ -1928,12 +1943,13 @@ func newEnumInitializer( assignmentStatement, }, ast.EmptyRange, + ast.EmptyComments, ), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) return ast.NewSpecialFunctionDeclaration( @@ -1994,11 +2010,13 @@ func newEnumLookup( ast.EmptyPosition, ), nil, + ast.EmptyComments, ), ast.EmptyPosition, ), nil, ast.EmptyPosition, + ast.EmptyComments, ), } @@ -2017,6 +2035,7 @@ func newEnumLookup( big.NewInt(int64(index)), 10, ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetIntegerExpressionType( integerExpression, @@ -2028,6 +2047,7 @@ func newEnumLookup( ast.NewIdentifierExpression( gauge, typeIdentifier, + ast.EmptyComments, ), false, ast.EmptyPosition, @@ -2049,6 +2069,7 @@ func newEnumLookup( gauge, memberExpression, ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetReturnStatementTypes( @@ -2076,6 +2097,7 @@ func newEnumLookup( gauge, ast.NewNilExpression(gauge, ast.EmptyPosition), ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetReturnStatementTypes( @@ -2097,9 +2119,10 @@ func newEnumLookup( switchStatement := ast.NewSwitchStatement( gauge, - ast.NewIdentifierExpression(gauge, rawValueIdentifier), + ast.NewIdentifierExpression(gauge, rawValueIdentifier, ast.EmptyComments), switchCases, ast.EmptyRange, + ast.EmptyComments, ) return ast.NewFunctionDeclaration( @@ -2110,7 +2133,7 @@ func newEnumLookup( false, typeIdentifier, nil, - ast.NewParameterList(gauge, parameters, ast.EmptyRange), + ast.NewParameterList(gauge, parameters, ast.EmptyRange, ast.EmptyComments), nil, ast.NewFunctionBlock( gauge, @@ -2120,12 +2143,13 @@ func newEnumLookup( switchStatement, }, ast.EmptyRange, + ast.EmptyComments, ), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) } @@ -2162,6 +2186,7 @@ func simpleFunctionDeclaration( memoryGauge, parameters, astRange, + ast.EmptyComments, ) } @@ -2185,12 +2210,13 @@ func simpleFunctionDeclaration( memoryGauge, statements, astRange, + ast.EmptyComments, ), nil, nil, ), startPos, - "", + ast.EmptyComments, ) } @@ -2266,6 +2292,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( enclosingType.QualifiedString(), startPos, ), + ast.EmptyComments, ), false, startPos, @@ -2381,6 +2408,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( commons.CollectEventsParamName, startPos, ), + ast.EmptyComments, ), nil, eventConstructorInvocations, @@ -2428,6 +2456,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( ), nil, ast.EmptyPosition, + ast.EmptyComments, ) eventEmittingFunction := ast.NewFunctionDeclaration( @@ -2446,6 +2475,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( d.memoryGauge, []*ast.Parameter{parameter}, astRange, + ast.EmptyComments, ), nil, ast.NewFunctionBlock( @@ -2454,12 +2484,13 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( d.memoryGauge, []ast.Statement{expressionStmt}, astRange, + ast.EmptyComments, ), nil, nil, ), startPos, - "", + ast.EmptyComments, ) d.elaboration.SetFunctionDeclarationFunctionType(eventEmittingFunction, eventEmittingFunctionType) @@ -2495,6 +2526,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( sema.SelfIdentifier, pos, ), + ast.EmptyComments, ), false, pos, @@ -2542,6 +2574,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( parameterName, pos, ), + ast.EmptyComments, ), ) @@ -2560,6 +2593,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( d.memoryGauge, statements, ast.NewRangeFromPositioned(d.memoryGauge, initializer), + ast.EmptyComments, ), nil, nil, @@ -2577,7 +2611,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( initializer.ReturnTypeAnnotation, modifiedFuncBlock, initializer.StartPos, - initializer.DocString, + initializer.Comments, ) // Desugared function's type is same as the original function's type. diff --git a/cmd/errors/errors.go b/cmd/errors/errors.go index b1d2ce0a56..473ad47b11 100644 --- a/cmd/errors/errors.go +++ b/cmd/errors/errors.go @@ -589,11 +589,6 @@ func generateErrors() []namedError { Token: placeholderToken, }, }, - {"parser.UnexpectedTokenInBlockCommentError", - &parser.UnexpectedTokenInBlockCommentError{ - GotToken: placeholderToken, - }, - }, {"parser.UnexpectedTokenInExpressionError", &parser.UnexpectedTokenInExpressionError{ GotToken: placeholderToken, diff --git a/interpreter/interpreter_statement.go b/interpreter/interpreter_statement.go index 573470bf7f..910109f87a 100644 --- a/interpreter/interpreter_statement.go +++ b/interpreter/interpreter_statement.go @@ -182,6 +182,7 @@ func (interpreter *Interpreter) VisitSwitchStatement(switchStatement *ast.Switch interpreter, switchCase.Statements, ast.EmptyRange, + ast.EmptyComments, ) result := interpreter.visitBlock(block) diff --git a/interpreter/memory_metering_test.go b/interpreter/memory_metering_test.go index 716b8bc7e4..05a303716e 100644 --- a/interpreter/memory_metering_test.go +++ b/interpreter/memory_metering_test.go @@ -8883,7 +8883,7 @@ func TestInterpretMemoryMeteringToken(t *testing.T) { _, err = inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(10), meter.getMemory(common.MemoryKindTypeToken)) + assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindTypeToken)) assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindSpaceToken)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindRawString)) }) diff --git a/old_parser/declaration.go b/old_parser/declaration.go index ce460447f9..c4ca0c5d82 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -415,7 +415,7 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - docString, + ast.EmptyComments, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) @@ -710,6 +710,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { endPos, ), locationPos, + ast.EmptyComments, ), nil } @@ -816,7 +817,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - "", + ast.EmptyComments, ), ) @@ -834,12 +835,12 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, startPos, parameterList.EndPos, ), + ast.EmptyComments, ), nil } @@ -925,12 +926,12 @@ func parseFieldWithVariableKind( variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.EmptyComments, ), nil } @@ -1063,8 +1064,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, []*ast.NominalType{}, members, - docString, declarationRange, + ast.EmptyComments, ), nil } else { return ast.NewCompositeDeclaration( @@ -1074,8 +1075,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + ast.EmptyComments, ), nil } } @@ -1166,8 +1167,8 @@ func parseAttachmentDeclaration( baseNominalType, conformances, members, - docString, declarationRange, + ast.EmptyComments, ), nil } @@ -1480,12 +1481,12 @@ func parseFieldDeclarationWithoutVariableKind( ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.EmptyComments, ), nil } @@ -1550,7 +1551,7 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - docString, + ast.EmptyComments, ), ), nil } @@ -1587,7 +1588,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - docString, + ast.EmptyComments, startPos, ), nil } diff --git a/old_parser/declaration_test.go b/old_parser/declaration_test.go index 583b00bb8f..c7466c1957 100644 --- a/old_parser/declaration_test.go +++ b/old_parser/declaration_test.go @@ -820,8 +820,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -858,8 +857,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -896,8 +894,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -2962,6 +2959,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 24, Offset: 24}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ @@ -3004,6 +3002,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 24, Offset: 24}, }, nil, + ast.EmptyComments, ), ast.NewNominalType( nil, @@ -3012,6 +3011,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 28, Offset: 28}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ @@ -6880,6 +6880,7 @@ func TestParseNestedPragma(t *testing.T) { } +// Doc strings are not tracked anymore in the old parser, and can be safely ignored. func TestParseMemberDocStrings(t *testing.T) { t.Parallel() @@ -6916,8 +6917,7 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -6931,8 +6931,7 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -6956,8 +6955,7 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -7034,8 +7032,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7052,8 +7049,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7070,8 +7066,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " destroyWithBlock", + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/old_parser/expression.go b/old_parser/expression.go index 06e7d9d672..ee13be3816 100644 --- a/old_parser/expression.go +++ b/old_parser/expression.go @@ -821,6 +821,7 @@ func defineIdentifierExpression() { return ast.NewIdentifierExpression( p.memoryGauge, p.tokenToIdentifier(token), + ast.EmptyComments, ), nil } }, @@ -1771,7 +1772,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange, ast.EmptyComments) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/old_parser/function.go b/old_parser/function.go index ceb8a39f63..f6db3a30a9 100644 --- a/old_parser/function.go +++ b/old_parser/function.go @@ -108,6 +108,7 @@ func parseParameterList(p *parser) (*ast.ParameterList, error) { startPos, endPos, ), + ast.EmptyComments, ), nil } @@ -167,6 +168,7 @@ func parseParameter(p *parser) (*ast.Parameter, error) { typeAnnotation, nil, startPos, + ast.EmptyComments, ), nil } @@ -341,7 +343,7 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - docString, + ast.EmptyComments, ), nil } diff --git a/old_parser/statement.go b/old_parser/statement.go index 131a76feaa..ae8f6d5980 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -194,7 +194,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State returnTypeAnnotation, functionBlock, startPos, - "", + ast.EmptyComments, ), nil } else { parameterList, returnTypeAnnotation, functionBlock, err := @@ -250,6 +250,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + ast.EmptyComments, ), nil } @@ -257,14 +258,14 @@ func parseBreakStatement(p *parser) *ast.BreakStatement { tokenRange := p.current.Range p.next() - return ast.NewBreakStatement(p.memoryGauge, tokenRange) + return ast.NewBreakStatement(p.memoryGauge, tokenRange, ast.EmptyComments) } func parseContinueStatement(p *parser) *ast.ContinueStatement { tokenRange := p.current.Range p.next() - return ast.NewContinueStatement(p.memoryGauge, tokenRange) + return ast.NewContinueStatement(p.memoryGauge, tokenRange, ast.EmptyComments) } func parseIfStatement(p *parser) (*ast.IfStatement, error) { @@ -336,6 +337,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { thenBlock, elseBlock, startPos, + ast.EmptyComments, ) if variableDeclaration != nil { @@ -359,6 +361,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.EmptyComments, ) result = outer } @@ -381,7 +384,7 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { return nil, err } - return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos), nil + return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos, ast.EmptyComments), nil } func parseForStatement(p *parser) (*ast.ForStatement, error) { @@ -447,6 +450,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { block, expression, startPos, + ast.EmptyComments, ), nil } @@ -476,6 +480,7 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.EmptyComments, ), nil } @@ -531,6 +536,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + ast.EmptyComments, ), preConditions, postConditions, @@ -612,7 +618,7 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { return nil, err } - return ast.NewEmitStatement(p.memoryGauge, invocation, startPos), nil + return ast.NewEmitStatement(p.memoryGauge, invocation, startPos, ast.EmptyComments), nil } func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { @@ -651,6 +657,7 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { startPos, endToken.EndPos, ), + ast.EmptyComments, ), nil } @@ -820,5 +827,6 @@ func parseRemoveStatement( attachmentNominalType, attached, startPos, + ast.EmptyComments, ), nil } diff --git a/old_parser/transaction.go b/old_parser/transaction.go index 07c9eafbfd..03f37db257 100644 --- a/old_parser/transaction.go +++ b/old_parser/transaction.go @@ -198,12 +198,12 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, startPos, endPos, ), + ast.EmptyComments, ), nil } @@ -282,7 +282,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - "", + ast.EmptyComments, ), ), nil } diff --git a/old_parser/type.go b/old_parser/type.go index c75ad6875e..e7f97c6949 100644 --- a/old_parser/type.go +++ b/old_parser/type.go @@ -219,6 +219,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, + ast.EmptyComments, ), nil } @@ -883,6 +884,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, var invokedExpression ast.Expression = ast.NewIdentifierExpression( p.memoryGauge, ty.Identifier, + ast.EmptyComments, ) for _, nestedIdentifier := range ty.NestedIdentifiers { diff --git a/parser/comment.go b/parser/comment.go deleted file mode 100644 index a121151275..0000000000 --- a/parser/comment.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package parser - -import ( - "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - progress := p.newProgress() - - for p.checkProgress(&progress) { - - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.report(&MissingCommentEndError{ - Pos: p.current.StartPos, - }) - ok = false - return - - default: - p.report(&UnexpectedTokenInBlockCommentError{ - GotToken: p.current, - }) - ok = false - return - } - } - - panic(errors.NewUnreachableError()) -} diff --git a/parser/comment_test.go b/parser/comment_test.go index ea99409e42..4f1230d4fd 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -19,14 +19,16 @@ package parser import ( + "encoding/json" "math/big" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/cadence/ast" "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/parser/lexer" + + "github.com/onflow/cadence/ast" . "github.com/onflow/cadence/test_utils/common_utils" ) @@ -83,6 +85,7 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() + // Extracting comments attached to the infix operator is more difficult and also an edge case, so ignore that for now. result, errs := testParseExpression(" 1/*test foo*/+/* bar */ 2 ") require.Empty(t, errs) @@ -97,6 +100,11 @@ func TestParseBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/*test foo*/")), + }, + }, }, Right: &ast.IntegerExpression{ PositiveLiteral: []byte("2"), @@ -193,55 +201,47 @@ func TestParseBlockComment(t *testing.T) { errs, ) }) +} - t.Run("invalid content", func(t *testing.T) { +func TestCommentsJSONSerialization(t *testing.T) { + t.Parallel() + t.Run("comments with leading and trailing", func(t *testing.T) { t.Parallel() - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Offset: 0, Column: 0}, - EndPos: ast.Position{Line: 1, Offset: 1, Column: 1}, - }, - }, - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Offset: 2, Column: 2}, - EndPos: ast.Position{Line: 1, Offset: 4, Column: 4}, - }, - }, - {Type: lexer.TokenEOF}, + comments := ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// leading comment 1")), + ast.NewComment(nil, []byte("/* leading comment 2 */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// trailing comment")), }, - input: []byte(`/*foo`), } - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, + jsonBytes, err := json.Marshal(comments) + require.NoError(t, err) + + assert.JSONEq(t, + `{ + "Leading": ["// leading comment 1", "/* leading comment 2 */"], + "Trailing": ["// trailing comment"] + }`, + string(jsonBytes), ) - AssertEqualWithDiff(t, - []error{ - &UnexpectedTokenInBlockCommentError{ - GotToken: lexer.Token{ - Range: ast.Range{ - StartPos: ast.Position{Offset: 2, Line: 1, Column: 2}, - EndPos: ast.Position{Offset: 4, Line: 1, Column: 4}, - }, - Type: lexer.TokenIdentifier, - }, - }, - }, - errs, + }) + + t.Run("empty comments", func(t *testing.T) { + t.Parallel() + + comments := ast.Comments{} + + jsonBytes, err := json.Marshal(comments) + require.NoError(t, err) + + assert.JSONEq(t, + `{}`, + string(jsonBytes), ) }) } diff --git a/parser/declaration.go b/parser/declaration.go index 447769cec2..d29d91dccc 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -34,9 +34,8 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] for p.checkProgress(&progress) { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -50,7 +49,7 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] default: var declaration ast.Declaration - declaration, err = parseDeclaration(p, docString) + declaration, err = parseDeclaration(p) if err != nil { return } @@ -66,16 +65,15 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] panic(errors.NewUnreachableError()) } -func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { - +func parseDeclaration(p *parser) (ast.Declaration, error) { var access ast.Access = ast.AccessNotSpecified - var accessPos *ast.Position + var accessToken *lexer.Token purity := ast.FunctionPurityUnspecified - var purityPos *ast.Position + var purityToken *lexer.Token - var staticPos *ast.Position - var nativePos *ast.Position + var staticToken *lexer.Token + var nativeToken *lexer.Token staticModifierEnabled := p.config.StaticModifierEnabled nativeModifierEnabled := p.config.NativeModifierEnabled @@ -84,11 +82,11 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenPragma: - rejectAllModifiers(p, access, accessPos, staticPos, nativePos, purityPos, common.DeclarationKindPragma) + rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, purityToken, common.DeclarationKindPragma) return parsePragmaDeclaration(p) case lexer.TokenIdentifier: @@ -96,62 +94,61 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { case KeywordLet: const isLet = true - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindVariable) - return parseVariableDeclaration(p, access, accessPos, isLet, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindVariable) + return parseVariableDeclaration(p, access, accessToken, isLet) case KeywordVar: const isLet = false - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindVariable) - return parseVariableDeclaration(p, access, accessPos, isLet, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindVariable) + return parseVariableDeclaration(p, access, accessToken, isLet) case KeywordFun: return parseFunctionDeclaration( p, false, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, - docString, + purityToken, + staticToken, + nativeToken, ) case KeywordImport: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindImport) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindImport) return parseImportDeclaration(p) case KeywordEvent: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEvent) - return parseEventDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEvent) + return parseEventDeclaration(p, access, accessToken) case KeywordStruct: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindStructure) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindStructure) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordResource: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindResource) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindResource) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEntitlement: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEntitlement) - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEntitlement) + return parseEntitlementOrMappingDeclaration(p, access, accessToken) case KeywordAttachment: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindAttachment) - return parseAttachmentDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindAttachment) + return parseAttachmentDeclaration(p, access, accessToken) case KeywordContract: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindContract) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindContract) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEnum: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEnum) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEnum) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordTransaction: - rejectAllModifiers(p, access, accessPos, staticPos, nativePos, purityPos, common.DeclarationKindTransaction) - return parseTransactionDeclaration(p, docString) + rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, purityToken, common.DeclarationKindTransaction) + return parseTransactionDeclaration(p) case KeywordView: if purity != ast.FunctionPurityUnspecified { @@ -160,8 +157,8 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { }) } - pos := p.current.StartPos - purityPos = &pos + tok := p.current + purityToken = &tok purity = parsePurityAnnotation(p) continue @@ -175,14 +172,14 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { case KeywordAccess: previousAccess := access - if staticModifierEnabled && staticPos != nil { + if staticModifierEnabled && staticToken != nil { p.reportSyntaxError("invalid access modifier after `static` modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { p.reportSyntaxError("invalid access modifier after `native` modifier") } - pos := p.current.StartPos - accessPos = &pos + tok := p.current + accessToken = &tok var ( accessRange ast.Range err error @@ -204,14 +201,14 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { break } - if staticPos != nil { + if staticToken != nil { p.reportSyntaxError("invalid second `static` modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { p.reportSyntaxError("invalid `static` modifier after `native` modifier") } - pos := p.current.StartPos - staticPos = &pos + tok := p.current + staticToken = &tok p.next() continue @@ -220,11 +217,11 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { break } - if nativePos != nil { + if nativeToken != nil { p.reportSyntaxError("invalid second `native` modifier") } - pos := p.current.StartPos - nativePos = &pos + tok := p.current + nativeToken = &tok p.next() continue } @@ -273,7 +270,7 @@ func handlePub(p *parser) { p.next() } - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenParenClose) { endToken = p.current @@ -316,7 +313,7 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { } rejectAccessKeywordEntitlementType(p, firstType) - p.skipSpaceAndComments() + p.skipSpace() entitlements := []*ast.NominalType{firstType} var separator lexer.TokenType @@ -423,7 +420,7 @@ func parseAccess(p *parser) (ast.Access, ast.Range, error) { } access = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) @@ -457,19 +454,20 @@ func parseAccess(p *parser) (ast.Access, ast.Range, error) { func parseVariableDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, isLet bool, - docString string, ) (*ast.VariableDeclaration, error) { - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `let` or `var` keyword p.nextSemanticToken() + identifierToken := p.current + identifier, err := p.nonReservedIdentifier("after start of variable declaration") if err != nil { return nil, err @@ -489,8 +487,9 @@ func parseVariableDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() + transferToken := p.current transfer := parseTransfer(p) if transfer == nil { p.report(&MissingTransferError{ @@ -503,7 +502,7 @@ func parseVariableDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() secondTransfer := parseTransfer(p) var secondValue ast.Expression @@ -514,6 +513,11 @@ func parseVariableDeclaration( } } + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startToken.Comments.All()...) + leadingComments = append(leadingComments, identifierToken.Comments.All()...) + leadingComments = append(leadingComments, transferToken.Comments.All()...) + variableDeclaration := ast.NewVariableDeclaration( p.memoryGauge, access, @@ -522,10 +526,12 @@ func parseVariableDeclaration( typeAnnotation, value, transfer, - startPos, + startToken.StartPos, secondTransfer, secondValue, - docString, + ast.Comments{ + Leading: leadingComments, + }, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) @@ -596,17 +602,19 @@ func parsePragmaDeclaration(p *parser) (*ast.PragmaDeclaration, error) { // ( string | hexadecimalLiteral | identifier ) func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { - startPosition := p.current.StartPos + startToken := p.current var imports []ast.Import var location common.Location var locationPos ast.Position var endPos ast.Position + var trailingComments []*ast.Comment parseStringOrAddressLocation := func() { locationPos = p.current.StartPos endPos = p.current.EndPos + trailingComments = p.current.Comments.Trailing switch p.current.Type { case lexer.TokenString: @@ -625,10 +633,11 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { p.next() } - setIdentifierLocation := func(identifier ast.Identifier) { + setIdentifierLocation := func(identifier ast.Identifier, identifierToken lexer.Token) { location = common.IdentifierLocation(identifier.Identifier) locationPos = identifier.Pos endPos = identifier.EndPosition(p.memoryGauge) + trailingComments = identifierToken.Comments.Trailing } parseLocation := func() error { @@ -638,7 +647,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { case lexer.TokenIdentifier: identifier := p.tokenToIdentifier(p.current) - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, p.current) p.next() default: @@ -703,7 +712,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { // Parse optional alias alias := parseOptionalImportAlias(p) - p.skipSpaceAndComments() + p.skipSpace() imports = append( imports, @@ -738,6 +747,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { parseStringOrAddressLocation() case lexer.TokenIdentifier: + identifierToken := p.current identifier := p.tokenToIdentifier(p.current) // Skip the identifier p.nextSemanticToken() @@ -745,7 +755,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { // Parse optional alias alias := parseOptionalImportAlias(p) - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -789,12 +799,12 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return nil, err } } else { - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) } case lexer.TokenEOF: // The previous identifier is the identifier location - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) default: return nil, &InvalidImportContinuationError{ @@ -819,10 +829,14 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { location, ast.NewRange( p.memoryGauge, - startPosition, + startToken.StartPos, endPos, ), locationPos, + ast.Comments{ + Leading: startToken.Comments.All(), + Trailing: trailingComments, + }, ), nil } @@ -910,13 +924,11 @@ func parseHexadecimalLocation(p *parser) common.AddressLocation { func parseEventDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - docString string, + accessToken *lexer.Token, ) (*ast.CompositeDeclaration, error) { - - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `event` keyword @@ -951,7 +963,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - "", + ast.EmptyComments, ), ) @@ -969,12 +981,14 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, parameterList.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -1044,13 +1058,11 @@ func checkAndReportFieldInitialization(p *parser) { func parseFieldWithVariableKind( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - docString string, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, ) (*ast.FieldDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) + startToken := lexer.EarliestToken(p.current, accessToken, staticToken, nativeToken) var variableKind ast.VariableKind switch string(p.currentTokenSource()) { @@ -1094,17 +1106,19 @@ func parseFieldWithVariableKind( return ast.NewFieldDeclaration( p.memoryGauge, access, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -1123,7 +1137,7 @@ func parseEntitlementMapping(p *parser) (*ast.EntitlementMapRelation, error) { }) } - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenRightArrow) { p.nextSemanticToken() @@ -1145,7 +1159,7 @@ func parseEntitlementMapping(p *parser) (*ast.EntitlementMapRelation, error) { }) } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewEntitlementMapRelation( p.memoryGauge, @@ -1162,7 +1176,7 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { @@ -1214,7 +1228,7 @@ func parseDeclarationBraces[T any]( return } - p.skipSpaceAndComments() + p.skipSpace() endToken = p.current @@ -1254,12 +1268,11 @@ func parseDeclarationClosingBrace(p *parser, kind common.DeclarationKind) { func parseEntitlementOrMappingDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - docString string, + accessToken *lexer.Token, ) (ast.Declaration, error) { - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `entitlement` keyword @@ -1293,6 +1306,10 @@ func parseEntitlementOrMappingDeclaration( } p.nextSemanticToken() + comments := ast.Comments{ + Leading: startToken.Comments.Leading, + } + if isMapping { elements, endToken, err := parseDeclarationBraces( @@ -1308,7 +1325,7 @@ func parseEntitlementOrMappingDeclaration( declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) @@ -1317,13 +1334,13 @@ func parseEntitlementOrMappingDeclaration( access, identifier, elements, - docString, declarationRange, + comments, ), nil } else { declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, identifier.EndPosition(p.memoryGauge), ) @@ -1331,8 +1348,8 @@ func parseEntitlementOrMappingDeclaration( p.memoryGauge, access, identifier, - docString, declarationRange, + comments, ), nil } } @@ -1371,13 +1388,12 @@ func parseConformances(p *parser) ([]*ast.NominalType, error) { func parseCompositeOrInterfaceDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - docString string, + accessToken *lexer.Token, ) (ast.Declaration, error) { - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } compositeKind := parseCompositeKind(p) @@ -1392,7 +1408,7 @@ func parseCompositeOrInterfaceDeclaration( for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, p.newSyntaxError( @@ -1428,7 +1444,7 @@ func parseCompositeOrInterfaceDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1448,10 +1464,14 @@ func parseCompositeOrInterfaceDeclaration( declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) + comments := ast.Comments{ + Leading: startToken.Comments.Leading, + } + if isInterface { return ast.NewInterfaceDeclaration( p.memoryGauge, @@ -1460,8 +1480,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + comments, ), nil } else { return ast.NewCompositeDeclaration( @@ -1471,8 +1491,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + comments, ), nil } } @@ -1480,12 +1500,12 @@ func parseCompositeOrInterfaceDeclaration( func parseAttachmentDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - docString string, + accessToken *lexer.Token, ) (ast.Declaration, error) { - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `attachment` keyword @@ -1496,7 +1516,7 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFor) { // Skip the `for` keyword @@ -1519,7 +1539,7 @@ func parseAttachmentDeclaration( }) } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1539,7 +1559,7 @@ func parseAttachmentDeclaration( declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) @@ -1550,8 +1570,10 @@ func parseAttachmentDeclaration( baseNominalType, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -1567,9 +1589,8 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) for p.checkProgress(&progress) { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -1582,7 +1603,7 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) return ast.NewMembers(p.memoryGauge, declarations), nil default: - memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p, docString) + memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p) if err != nil { return nil, err } @@ -1609,18 +1630,17 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | eventDeclaration // | enumCase // | pragmaDeclaration -func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaration, error) { - +func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { const functionBlockIsOptional = true var access ast.Access = ast.AccessNotSpecified - var accessPos *ast.Position + var accessToken *lexer.Token purity := ast.FunctionPurityUnspecified - var purityPos *ast.Position + var purityToken *lexer.Token - var staticPos *ast.Position - var nativePos *ast.Position + var staticToken *lexer.Token + var nativeToken *lexer.Token var previousIdentifierToken *lexer.Token @@ -1631,7 +1651,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -1648,59 +1668,57 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: - rejectPurityModifier(p, purityPos, common.DeclarationKindField) + rejectPurityModifier(p, purityToken, common.DeclarationKindField) return parseFieldWithVariableKind( p, access, - accessPos, - staticPos, - nativePos, - docString, + accessToken, + staticToken, + nativeToken, ) case KeywordCase: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEnumCase) - return parseEnumCase(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEnumCase) + return parseEnumCase(p, access, accessToken) case KeywordFun: return parseFunctionDeclaration( p, functionBlockIsOptional, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, - docString, + purityToken, + staticToken, + nativeToken, ) case KeywordEvent: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEvent) - return parseEventDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEvent) + return parseEventDeclaration(p, access, accessToken) case KeywordStruct: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindStructure) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindStructure) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordResource: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindResource) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindResource) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordContract: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindContract) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindContract) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEntitlement: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEntitlement) - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEntitlement) + return parseEntitlementOrMappingDeclaration(p, access, accessToken) case KeywordEnum: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEnum) - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEnum) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordAttachment: - return parseAttachmentDeclaration(p, access, accessPos, docString) + return parseAttachmentDeclaration(p, access, accessToken) case KeywordView: if purity != ast.FunctionPurityUnspecified { @@ -1708,8 +1726,8 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio Range: p.current.Range, }) } - pos := p.current.StartPos - purityPos = &pos + tok := p.current + purityToken = &tok purity = parsePurityAnnotation(p) continue @@ -1723,14 +1741,14 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio case KeywordAccess: previousAccess := access - if staticModifierEnabled && staticPos != nil { + if staticModifierEnabled && staticToken != nil { p.reportSyntaxError("invalid access modifier after `static` modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { p.reportSyntaxError("invalid access modifier after `native` modifier") } - pos := p.current.StartPos - accessPos = &pos + tok := p.current + accessToken = &tok var ( accessRange ast.Range err error @@ -1751,14 +1769,14 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio break } - if staticPos != nil { + if staticToken != nil { p.reportSyntaxError("invalid second `static` modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { p.reportSyntaxError("invalid `static` modifier after `native` modifier") } - pos := p.current.StartPos - staticPos = &pos + tok := p.current + staticToken = &tok p.next() continue @@ -1767,11 +1785,11 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio break } - if nativePos != nil { + if nativeToken != nil { p.reportSyntaxError("invalid second `native` modifier") } - pos := p.current.StartPos - nativePos = &pos + tok := p.current + nativeToken = &tok p.next() continue } @@ -1798,7 +1816,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio ). WithSecondary("remove the identifier before the pragma declaration") } - rejectAllModifiers(p, access, accessPos, staticPos, nativePos, purityPos, common.DeclarationKindPragma) + rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, purityToken, common.DeclarationKindPragma) return parsePragmaDeclaration(p) case lexer.TokenColon: @@ -1807,16 +1825,16 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio WithSecondary("expected an identifier before the colon"). WithDocumentation("https://cadence-lang.org/docs/language/glossary#-colon") } - rejectPurityModifier(p, purityPos, common.DeclarationKindField) + rejectPurityModifier(p, purityToken, common.DeclarationKindField) identifier := p.tokenToIdentifier(*previousIdentifierToken) return parseFieldDeclarationWithoutVariableKind( p, access, - accessPos, - staticPos, - nativePos, + accessToken, + staticToken, + nativeToken, + *previousIdentifierToken, identifier, - docString, ) case lexer.TokenParenOpen: @@ -1826,18 +1844,16 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio WithDocumentation("https://cadence-lang.org/docs/language/types-and-type-system/composite-types") } - identifier := p.tokenToIdentifier(*previousIdentifierToken) return parseSpecialFunctionDeclaration( p, functionBlockIsOptional, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, - identifier, - docString, + purityToken, + staticToken, + nativeToken, + *previousIdentifierToken, ) } @@ -1850,57 +1866,57 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio func rejectAllModifiers( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - purityPos *ast.Position, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, + purityToken *lexer.Token, kind common.DeclarationKind, ) { if access != ast.AccessNotSpecified { p.report(&InvalidAccessModifierError{ - Pos: *accessPos, + Pos: accessToken.StartPos, DeclarationKind: kind, }) } rejectNonAccessModifiers( p, - staticPos, - nativePos, - purityPos, + staticToken, + nativeToken, + purityToken, kind, ) } func rejectNonAccessModifiers( p *parser, - staticPos *ast.Position, - nativePos *ast.Position, - purityPos *ast.Position, + staticToken *lexer.Token, + nativeToken *lexer.Token, + purityToken *lexer.Token, kind common.DeclarationKind, ) { - if p.config.StaticModifierEnabled && staticPos != nil { + if p.config.StaticModifierEnabled && staticToken != nil { p.report(&InvalidStaticModifierError{ - Pos: *staticPos, + Pos: staticToken.StartPos, DeclarationKind: kind, }) } - if p.config.NativeModifierEnabled && nativePos != nil { + if p.config.NativeModifierEnabled && nativeToken != nil { p.report(&InvalidNativeModifierError{ - Pos: *nativePos, + Pos: nativeToken.StartPos, DeclarationKind: kind, }) } - rejectPurityModifier(p, purityPos, kind) + rejectPurityModifier(p, purityToken, kind) } func rejectPurityModifier( p *parser, - purityPos *ast.Position, + purityToken *lexer.Token, kind common.DeclarationKind, ) { - if purityPos != nil { + if purityToken != nil { p.report(&InvalidViewModifierError{ - Pos: *purityPos, + Pos: purityToken.StartPos, DeclarationKind: kind, }) } @@ -1909,14 +1925,13 @@ func rejectPurityModifier( func parseFieldDeclarationWithoutVariableKind( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, + identifierToken lexer.Token, identifier ast.Identifier, - docString string, ) (*ast.FieldDeclaration, error) { - - startPos := ast.EarliestPosition(identifier.Pos, accessPos, staticPos, nativePos) + startToken := lexer.EarliestToken(identifierToken, accessToken, staticToken, nativeToken) if p.current.Is(lexer.TokenColon) { p.nextSemanticToken() @@ -1938,17 +1953,19 @@ func parseFieldDeclarationWithoutVariableKind( return ast.NewFieldDeclaration( p.memoryGauge, access, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -1956,16 +1973,16 @@ func parseSpecialFunctionDeclaration( p *parser, functionBlockIsOptional bool, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, purity ast.FunctionPurity, - purityPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - identifier ast.Identifier, - docString string, + purityToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, + identifierToken lexer.Token, ) (*ast.SpecialFunctionDeclaration, error) { - startPos := ast.EarliestPosition(identifier.Pos, accessPos, purityPos, staticPos, nativePos) + identifier := p.tokenToIdentifier(identifierToken) + startToken := lexer.EarliestToken(identifierToken, accessToken, purityToken, staticToken, nativeToken) parameterList, returnTypeAnnotation, functionBlock, err := parseFunctionParameterListAndRest(p, functionBlockIsOptional) @@ -1988,7 +2005,7 @@ func parseSpecialFunctionDeclaration( } p.report(&CustomDestructorError{ Pos: identifier.Pos, - DestructorRange: ast.NewRange(p.memoryGauge, startPos, endPos), + DestructorRange: ast.NewRange(p.memoryGauge, startToken.StartPos, endPos), }) case KeywordPrepare: @@ -2009,15 +2026,17 @@ func parseSpecialFunctionDeclaration( p.memoryGauge, access, purity, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, identifier, nil, parameterList, nil, functionBlock, - startPos, - docString, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), ), nil } @@ -2028,13 +2047,11 @@ func parseSpecialFunctionDeclaration( func parseEnumCase( p *parser, access ast.Access, - accessPos *ast.Position, - docString string, + accessToken *lexer.Token, ) (*ast.EnumCaseDeclaration, error) { - - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `enum` keyword @@ -2054,7 +2071,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - docString, - startPos, + ast.Comments{Leading: startToken.Comments.All()}, + startToken.StartPos, ), nil } diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 01552d1213..ad6d9bc6cc 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -74,6 +74,59 @@ func TestParseVariableDeclaration(t *testing.T) { ) }) + t.Run("var, no type annotation, copy, one value, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before x +var x = /* Before 1 */ 1 // After 1 +// Ignored +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.VariableDeclaration{ + Access: ast.AccessNotSpecified, + IsConstant: false, + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 4, Offset: 17}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before x")), + ast.NewComment(nil, []byte("/* Before 1 */")), + }, + Trailing: []*ast.Comment{}, + }, + Value: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + EndPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After 1")), + }, + }, + }, + Transfer: &ast.Transfer{ + Operation: ast.TransferOperationCopy, + Pos: ast.Position{Line: 3, Column: 6, Offset: 19}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + }, + }, + result, + ) + }) + t.Run("var, no type annotation, copy, one value, access(all)", func(t *testing.T) { t.Parallel() @@ -269,6 +322,7 @@ func TestParseVariableDeclaration(t *testing.T) { ) }) + // TODO: Fix these cases t.Run("with purity", func(t *testing.T) { t.Parallel() @@ -491,9 +545,12 @@ func TestParseParameterList(t *testing.T) { nil, []byte(input), func(p *parser) (*ast.ParameterList, error) { - return parseParameterList(p, false) + parameters, err := parseParameterList(p, false) + return parameters, err + }, + Config{ + CommentTrackingEnabled: true, }, - Config{}, ) } @@ -571,6 +628,62 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("one, resource type, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := parse(`( a : /* After colon */ +// Before type +@Int /* After type */ )`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: true, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 1, Offset: 41}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + // This comment should be attached to Type instead of TypeAnnotation + // (even tho it's technically attached to the @ resource symbol) for simplicity. + ast.NewComment(nil, []byte("// Before type")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After type */")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 40}, + }, + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After colon */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 3, Column: 22, Offset: 62}, + }, + }, + result, + ) + }) + t.Run("one, with argument label", func(t *testing.T) { t.Parallel() @@ -665,6 +778,91 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("two, with comments", func(t *testing.T) { + + t.Parallel() + + // Note: Some edge case comments are currently ignored (not mapped to the AST) for simplicity. + result, errs := parse(`( /* Before param b */ a b : /* Before b type annotation */ Int /* After param b */, +// Before param c +c /* After c identifier */ : Int // After param c +)`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "a", + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Line: 1, Column: 25, Offset: 25}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After param b */")), + }, + }, + }, + StartPos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + StartPos: ast.Position{Line: 1, Column: 23, Offset: 23}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* Before b type annotation */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "c", + Pos: ast.Position{Line: 3, Column: 0, Offset: 104}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After param c ")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 104}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before param c")), + ast.NewComment(nil, []byte("/* After c identifier */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 4, Column: 0, Offset: 155}, + }, + }, + result, + ) + }) + t.Run("two, with and without argument label, missing comma", func(t *testing.T) { t.Parallel() @@ -1220,8 +1418,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// Test")), + }, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -1258,8 +1460,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -1296,8 +1503,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -1742,7 +1953,10 @@ func TestParseFunctionDeclaration(t *testing.T) { PreConditions: (*ast.Conditions)(nil), PostConditions: (*ast.Conditions)(nil), }, - DocString: "", + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{Offset: 30, Line: 1, Column: 30}, @@ -3084,6 +3298,38 @@ func TestParseImportDeclaration(t *testing.T) { ) }) + t.Run("no identifiers, string location, with leading/trailing comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before foo +import "foo" /* After foo */`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.ImportDeclaration{ + Location: common.StringLocation("foo"), + LocationPos: ast.Position{Line: 3, Column: 7, Offset: 22}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 15}, + EndPos: ast.Position{Line: 3, Column: 11, Offset: 26}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After foo */")), + }, + }, + }, + }, + result, + ) + }) + t.Run("no identifiers, address location", func(t *testing.T) { t.Parallel() @@ -3231,11 +3477,11 @@ func TestParseImportDeclaration(t *testing.T) { ) }) - t.Run("three identifiers, address location", func(t *testing.T) { + t.Run("three identifiers, address location, with trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo , bar , baz from 0x42`) + result, errs := testParseDeclarations(` import foo , bar , baz from 0x42 // After address`) require.Empty(t, errs) AssertEqualWithDiff(t, @@ -3269,6 +3515,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 32, Offset: 32}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After address")), + }, + }, }, }, result, @@ -3326,11 +3577,11 @@ func TestParseImportDeclaration(t *testing.T) { AssertEqualWithDiff(t, expected, result) }) - t.Run("no identifiers, identifier location", func(t *testing.T) { + t.Run("no identifiers, identifier location, with trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo`) + result, errs := testParseDeclarations(` import foo // After foo`) require.Empty(t, errs) AssertEqualWithDiff(t, @@ -3343,6 +3594,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 10, Offset: 10}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After foo")), + }, + }, }, }, result, @@ -3745,6 +4001,63 @@ func TestParseEvent(t *testing.T) { ) }) + t.Run("no parameters, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before E +event E() // After E +// Ignored +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Offset: 19, Line: 3, Column: 6}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + }, + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + Kind: common.DeclarationKindInitializer, + FunctionDeclaration: &ast.FunctionDeclaration{ + Access: ast.AccessNotSpecified, + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After E")), + }, + }, + }, + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + }, + }, + }, + ), + Range: ast.Range{ + StartPos: ast.Position{Offset: 13, Line: 3, Column: 0}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + }, + }, + result, + ) + }) + t.Run("two parameters, private", func(t *testing.T) { t.Parallel() @@ -3822,6 +4135,120 @@ func TestParseEvent(t *testing.T) { ) }) + t.Run("one parameter with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(`event E2 ( + // Before a + a: Int // After a +)`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 28, + Line: 3, + Column: 4, + }, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After a")), + }, + }, + }, + StartPos: ast.Position{ + Offset: 28, + Line: 3, + Column: 4, + }, + }, + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{ + Offset: 25, + Line: 3, + Column: 1, + }, + }, + StartPos: ast.Position{ + Offset: 25, + Line: 3, + Column: 1, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before a")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 9, + Line: 1, + Column: 9, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 4, + Column: 0, + }, + }, + }, + StartPos: ast.Position{ + Offset: 9, + Line: 1, + Column: 9, + }, + Access: ast.AccessNotSpecified, + }, + Kind: common.DeclarationKindInitializer, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "E2", + Pos: ast.Position{ + Offset: 6, + Line: 1, + Column: 6, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 4, + Column: 0, + }, + }, + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + }, + }, + result, + ) + }) + t.Run("default event", func(t *testing.T) { t.Parallel() @@ -3982,7 +4409,6 @@ func TestParseFieldWithVariableKind(t *testing.T) { nil, nil, nil, - "", ) }, Config{}, @@ -4196,12 +4622,7 @@ func TestParseField(t *testing.T) { return Parse( nil, []byte(input), - func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) - }, + parseMemberOrNestedDeclaration, config, ) } @@ -4663,20 +5084,247 @@ func TestParseCompositeDeclaration(t *testing.T) { Pos: ast.Position{Offset: 60, Line: 3, Column: 35}, }, }, - StartPos: ast.Position{Offset: 60, Line: 3, Column: 35}, + StartPos: ast.Position{Offset: 60, Line: 3, Column: 35}, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{Offset: 55, Line: 3, Column: 30}, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 39, Line: 3, Column: 14}, + EndPos: ast.Position{Offset: 62, Line: 3, Column: 37}, + }, + Access: ast.AccessAll, + VariableKind: ast.VariableKindVariable, + Flags: 0, + }, + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Offset: 89, Line: 5, Column: 24}, + }, + }, + StartPos: ast.Position{Offset: 89, Line: 5, Column: 24}, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{Offset: 84, Line: 5, Column: 19}, + }, + StartPos: ast.Position{Offset: 84, Line: 5, Column: 19}, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 83, Line: 5, Column: 18}, + EndPos: ast.Position{Offset: 92, Line: 5, Column: 27}, + }, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.AssignmentStatement{ + Target: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{Offset: 114, Line: 6, Column: 18}, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{Offset: 119, Line: 6, Column: 23}, + }, + AccessEndPos: ast.Position{Offset: 118, Line: 6, Column: 22}, + Optional: false, + }, + Transfer: &ast.Transfer{ + Operation: ast.TransferOperationCopy, + Pos: ast.Position{Offset: 123, Line: 6, Column: 27}, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{Offset: 125, Line: 6, Column: 29}, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 94, Line: 5, Column: 29}, + EndPos: ast.Position{Offset: 143, Line: 7, Column: 14}, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "init", + Pos: ast.Position{Offset: 79, Line: 5, Column: 14}, + }, + StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, + Access: ast.AccessNotSpecified, + Flags: 0, + }, + Kind: common.DeclarationKindInitializer, + }, + &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 182, Line: 9, Column: 36}, + EndPos: ast.Position{Offset: 183, Line: 9, Column: 37}, + }, + }, + ReturnTypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Offset: 186, Line: 9, Column: 40}, + }, + }, + StartPos: ast.Position{Offset: 186, Line: 9, Column: 40}, + IsResource: false, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{Offset: 217, Line: 10, Column: 25}, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{Offset: 222, Line: 10, Column: 30}, + }, + AccessEndPos: ast.Position{Offset: 221, Line: 10, Column: 29}, + Optional: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 210, Line: 10, Column: 18}, + EndPos: ast.Position{Offset: 224, Line: 10, Column: 32}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 190, Line: 9, Column: 44}, + EndPos: ast.Position{Offset: 240, Line: 11, Column: 14}, + }, + }, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, + }, + Identifier: ast.Identifier{ + Identifier: "getFoo", + Pos: ast.Position{Offset: 176, Line: 9, Column: 30}, + }, + StartPos: ast.Position{Offset: 160, Line: 9, Column: 14}, + Access: ast.AccessAll, + Flags: 0, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "Test", + Pos: ast.Position{Offset: 18, Line: 2, Column: 17}, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 11, Line: 2, Column: 10}, + EndPos: ast.Position{Offset: 252, Line: 12, Column: 10}, + }, + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindStructure, + }, + }, + result, + ) + }) + + t.Run("struct, with fields, functions, special functions, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before Test + struct Test { + // Before foo + access(all) var foo: Int + + // Before init + init(foo: Int) { + self.foo = foo + } + + // Before getFoo + access(all) fun getFoo(): Int { + return self.foo + } + } + `) + + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.FieldDeclaration{ + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, + }, + }, + StartPos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, IsResource: false, }, Identifier: ast.Identifier{ Identifier: "foo", - Pos: ast.Position{Offset: 55, Line: 3, Column: 30}, + Pos: ast.Position{ + Offset: 92, + Line: 5, + Column: 26, + }, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 39, Line: 3, Column: 14}, - EndPos: ast.Position{Offset: 62, Line: 3, Column: 37}, + StartPos: ast.Position{ + Offset: 76, + Line: 5, + Column: 10, + }, + EndPos: ast.Position{ + Offset: 99, + Line: 5, + Column: 33, + }, }, Access: ast.AccessAll, - VariableKind: ast.VariableKindVariable, - Flags: 0, + VariableKind: 0x1, + Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + }, }, &ast.SpecialFunctionDeclaration{ FunctionDeclaration: &ast.FunctionDeclaration{ @@ -4687,22 +5335,46 @@ func TestParseCompositeDeclaration(t *testing.T) { Type: &ast.NominalType{ Identifier: ast.Identifier{ Identifier: "Int", - Pos: ast.Position{Offset: 89, Line: 5, Column: 24}, + Pos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, }, }, - StartPos: ast.Position{Offset: 89, Line: 5, Column: 24}, + StartPos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, IsResource: false, }, Identifier: ast.Identifier{ Identifier: "foo", - Pos: ast.Position{Offset: 84, Line: 5, Column: 19}, + Pos: ast.Position{ + Offset: 142, + Line: 8, + Column: 15, + }, + }, + StartPos: ast.Position{ + Offset: 142, + Line: 8, + Column: 15, }, - StartPos: ast.Position{Offset: 84, Line: 5, Column: 19}, }, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 83, Line: 5, Column: 18}, - EndPos: ast.Position{Offset: 92, Line: 5, Column: 27}, + StartPos: ast.Position{ + Offset: 141, + Line: 8, + Column: 14, + }, + EndPos: ast.Position{ + Offset: 150, + Line: 8, + Column: 23, + }, }, }, FunctionBlock: &ast.FunctionBlock{ @@ -4713,59 +5385,116 @@ func TestParseCompositeDeclaration(t *testing.T) { Expression: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "self", - Pos: ast.Position{Offset: 114, Line: 6, Column: 18}, + Pos: ast.Position{ + Offset: 168, + Line: 9, + Column: 14, + }, }, }, Identifier: ast.Identifier{ Identifier: "foo", - Pos: ast.Position{Offset: 119, Line: 6, Column: 23}, + Pos: ast.Position{ + Offset: 173, + Line: 9, + Column: 19, + }, }, - AccessEndPos: ast.Position{Offset: 118, Line: 6, Column: 22}, - Optional: false, + AccessEndPos: ast.Position{ + Offset: 172, + Line: 9, + Column: 18, + }, + Optional: false, }, Transfer: &ast.Transfer{ - Operation: ast.TransferOperationCopy, - Pos: ast.Position{Offset: 123, Line: 6, Column: 27}, + Operation: 0x1, + Pos: ast.Position{ + Offset: 177, + Line: 9, + Column: 23, + }, }, Value: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "foo", - Pos: ast.Position{Offset: 125, Line: 6, Column: 29}, + Pos: ast.Position{ + Offset: 179, + Line: 9, + Column: 25, + }, }, }, }, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 94, Line: 5, Column: 29}, - EndPos: ast.Position{Offset: 143, Line: 7, Column: 14}, + StartPos: ast.Position{ + Offset: 152, + Line: 8, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 193, + Line: 10, + Column: 10, + }, }, }, }, Identifier: ast.Identifier{ Identifier: "init", - Pos: ast.Position{Offset: 79, Line: 5, Column: 14}, + Pos: ast.Position{ + Offset: 137, + Line: 8, + Column: 10, + }, + }, + StartPos: ast.Position{ + Offset: 137, + Line: 8, + Column: 10, + }, + Access: ast.AccessNotSpecified, + Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before init")), + }, }, - StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, - Access: ast.AccessNotSpecified, - Flags: 0, }, - Kind: common.DeclarationKindInitializer, + Kind: 0xd, }, &ast.FunctionDeclaration{ ParameterList: &ast.ParameterList{ Range: ast.Range{ - StartPos: ast.Position{Offset: 182, Line: 9, Column: 36}, - EndPos: ast.Position{Offset: 183, Line: 9, Column: 37}, + StartPos: ast.Position{ + Offset: 255, + Line: 13, + Column: 32, + }, + EndPos: ast.Position{ + Offset: 256, + Line: 13, + Column: 33, + }, }, }, ReturnTypeAnnotation: &ast.TypeAnnotation{ Type: &ast.NominalType{ Identifier: ast.Identifier{ Identifier: "Int", - Pos: ast.Position{Offset: 186, Line: 9, Column: 40}, + Pos: ast.Position{ + Offset: 259, + Line: 13, + Column: 36, + }, }, }, - StartPos: ast.Position{Offset: 186, Line: 9, Column: 40}, + StartPos: ast.Position{ + Offset: 259, + Line: 13, + Column: 36, + }, IsResource: false, }, FunctionBlock: &ast.FunctionBlock{ @@ -4776,49 +5505,106 @@ func TestParseCompositeDeclaration(t *testing.T) { Expression: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "self", - Pos: ast.Position{Offset: 217, Line: 10, Column: 25}, + Pos: ast.Position{ + Offset: 286, + Line: 14, + Column: 21, + }, }, }, Identifier: ast.Identifier{ Identifier: "foo", - Pos: ast.Position{Offset: 222, Line: 10, Column: 30}, + Pos: ast.Position{ + Offset: 291, + Line: 14, + Column: 26, + }, }, - AccessEndPos: ast.Position{Offset: 221, Line: 10, Column: 29}, - Optional: false, + AccessEndPos: ast.Position{ + Offset: 290, + Line: 14, + Column: 25, + }, + Optional: false, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 210, Line: 10, Column: 18}, - EndPos: ast.Position{Offset: 224, Line: 10, Column: 32}, + StartPos: ast.Position{ + Offset: 279, + Line: 14, + Column: 14, + }, + EndPos: ast.Position{ + Offset: 293, + Line: 14, + Column: 28, + }, }, }, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 190, Line: 9, Column: 44}, - EndPos: ast.Position{Offset: 240, Line: 11, Column: 14}, + StartPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 40, + }, + EndPos: ast.Position{ + Offset: 305, + Line: 15, + Column: 10, + }, }, }, }, - DocString: "", + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before getFoo")), + }, + }, Identifier: ast.Identifier{ Identifier: "getFoo", - Pos: ast.Position{Offset: 176, Line: 9, Column: 30}, + Pos: ast.Position{ + Offset: 249, + Line: 13, + Column: 26, + }, }, - StartPos: ast.Position{Offset: 160, Line: 9, Column: 14}, - Access: ast.AccessAll, - Flags: 0, + StartPos: ast.Position{ + Offset: 233, + Line: 13, + Column: 10, + }, + Access: ast.AccessAll, + Flags: 0x00, }, }, ), Identifier: ast.Identifier{ Identifier: "Test", - Pos: ast.Position{Offset: 18, Line: 2, Column: 17}, + Pos: ast.Position{ + Offset: 35, + Line: 3, + Column: 13, + }, }, Range: ast.Range{ - StartPos: ast.Position{Offset: 11, Line: 2, Column: 10}, - EndPos: ast.Position{Offset: 252, Line: 12, Column: 10}, + StartPos: ast.Position{ + Offset: 28, + Line: 3, + Column: 6, + }, + EndPos: ast.Position{ + Offset: 313, + Line: 16, + Column: 6, + }, }, Access: ast.AccessNotSpecified, CompositeKind: common.CompositeKindStructure, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before Test")), + }, + }, }, }, result, @@ -5075,6 +5861,45 @@ func TestParseAttachmentDeclaration(t *testing.T) { ) }) + t.Run("no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before E + access(all) attachment E for S {}`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.AttachmentDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Line: 3, Column: 27, Offset: 44}, + }, + BaseType: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 33, Offset: 50}, + }, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 4, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 36, Offset: 53}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested in contract", func(t *testing.T) { t.Parallel() @@ -5301,6 +6126,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 32, Offset: 32}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ @@ -5608,6 +6434,40 @@ func TestParseInterfaceDeclaration(t *testing.T) { ) }) + t.Run("struct, no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before S +access(all) struct interface S { }`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.InterfaceDeclaration{ + Access: ast.AccessAll, + CompositeKind: common.CompositeKindStructure, + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 29, Offset: 42}, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + EndPos: ast.Position{Line: 3, Column: 33, Offset: 46}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before S")), + }, + }, + }, + }, + result, + ) + }) + t.Run("struct, interface keyword as name", func(t *testing.T) { t.Parallel() @@ -6197,6 +7057,38 @@ func TestParseTransactionDeclaration(t *testing.T) { ) }) + t.Run("EmptyTransaction, comments", func(t *testing.T) { + + const code = ` + // Before tx + transaction {} + ` + result, errs := testParseProgram(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.TransactionDeclaration{ + Fields: nil, + Prepare: nil, + PreConditions: nil, + PostConditions: nil, + Execute: nil, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before tx")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 22, Line: 3, Column: 4}, + EndPos: ast.Position{Offset: 35, Line: 3, Column: 17}, + }, + }, + }, + result.Declarations(), + ) + }) + t.Run("SimpleTransaction", func(t *testing.T) { t.Parallel() @@ -7119,7 +8011,6 @@ func TestParseStructure(t *testing.T) { StartPos: ast.Position{Offset: 56, Line: 3, Column: 33}, IsResource: false, }, - DocString: "", Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{Offset: 51, Line: 3, Column: 28}, @@ -9911,12 +10802,7 @@ func TestParseNestedPragma(t *testing.T) { return Parse( nil, []byte(input), - func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) - }, + parseMemberOrNestedDeclaration, config, ) } @@ -10172,6 +11058,38 @@ func TestParseEntitlementDeclaration(t *testing.T) { ) }) + t.Run("basic, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before ABC + access(all) entitlement ABC`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "ABC", + Pos: ast.Position{Line: 3, Column: 27, Offset: 45}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 29, Offset: 47}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before ABC")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested entitlement", func(t *testing.T) { t.Parallel() @@ -10288,8 +11206,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -10303,8 +11225,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -10328,8 +11254,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -10403,8 +11333,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -10421,8 +11355,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -10479,6 +11417,38 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { ) }) + t.Run("empty, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before M + access(all) entitlement mapping M { }`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementMappingDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "M", + Pos: ast.Position{Line: 3, Column: 35, Offset: 51}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 19}, + EndPos: ast.Position{Line: 3, Column: 39, Offset: 55}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before M")), + }, + }, + }, + }, + result, + ) + }) + t.Run("mappings", func(t *testing.T) { t.Parallel() @@ -10494,8 +11464,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{Offset: 43, Line: 2, Column: 42}, @@ -10558,8 +11527,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{Offset: 43, Line: 2, Column: 42}, @@ -11541,3 +12509,96 @@ func TestParseTransactionDeclarationMissingOpeningBraceEOF(t *testing.T) { fixes[0].TextEdits[0].ApplyTo(code), ) } + +func TestParseDeclarationComments(t *testing.T) { + + t.Parallel() + + t.Run("event declaration", func(t *testing.T) { + res, errs := ParseDeclarations( + nil, + []byte(` +/// Before MyEvent +event MyEvent() // After MyEvent +/// Ignored +`), + Config{CommentTrackingEnabled: true}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + event, ok := res[0].(*ast.CompositeDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, + }, event.Comments) + assert.Equal(t, " Before MyEvent", event.DeclarationDocString()) + + decl, ok := event.Members.Declarations()[0].(*ast.SpecialFunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, + }, decl.FunctionDeclaration.ParameterList.Comments) + assert.Equal(t, "", decl.DeclarationDocString()) + + }) + + t.Run("function declaration", func(t *testing.T) { + res, errs := ParseProgram( + nil, + []byte(` +/// Inline doc 1 of first +/// Inline doc 2 of first +fun first() {} // Trailing inline comment of first + +/** +Multi-line doc 1 of second +*/ +/** +Multi-line doc 2 of second +*/ +fun second() {} /** +Trailing multi-line comment of second +*/ +`), + Config{ + CommentTrackingEnabled: true, + }, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// Inline doc 1 of first")), + ast.NewComment(nil, []byte("/// Inline doc 2 of first")), + }, + }, first.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// Trailing inline comment of first")), + }, + }, first.FunctionBlock.Block.Comments) + + second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), + ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), + }, + }, second.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), + }, + }, second.FunctionBlock.Block.Comments) + }) + +} diff --git a/parser/errors.go b/parser/errors.go index d18cda2aba..8cba3d8749 100644 --- a/parser/errors.go +++ b/parser/errors.go @@ -3976,10 +3976,7 @@ func (e *MissingCommentEndError) EndPosition(_ common.MemoryGauge) ast.Position } func (e *MissingCommentEndError) Error() string { - return fmt.Sprintf( - "missing comment end (%#q)", - lexer.TokenBlockCommentEnd, - ) + return "missing comment end (`*/`)" } func (*MissingCommentEndError) SecondaryError() string { @@ -4007,43 +4004,6 @@ func (e *MissingCommentEndError) SuggestFixes(_ string) []errors.SuggestedFix[as } } -// UnexpectedTokenInBlockCommentError is reported when an unexpected token is found in a block comment. -type UnexpectedTokenInBlockCommentError struct { - GotToken lexer.Token -} - -var _ ParseError = &UnexpectedTokenInBlockCommentError{} -var _ errors.UserError = &UnexpectedTokenInBlockCommentError{} -var _ errors.SecondaryError = &UnexpectedTokenInBlockCommentError{} -var _ errors.HasDocumentationLink = &UnexpectedTokenInBlockCommentError{} - -func (*UnexpectedTokenInBlockCommentError) isParseError() {} - -func (*UnexpectedTokenInBlockCommentError) IsUserError() {} - -func (e *UnexpectedTokenInBlockCommentError) StartPosition() ast.Position { - return e.GotToken.StartPos -} - -func (e *UnexpectedTokenInBlockCommentError) EndPosition(_ common.MemoryGauge) ast.Position { - return e.GotToken.EndPos -} - -func (e *UnexpectedTokenInBlockCommentError) Error() string { - return fmt.Sprintf( - "unexpected token %s in block comment", - e.GotToken.Type, - ) -} - -func (*UnexpectedTokenInBlockCommentError) SecondaryError() string { - return "only text is allowed in a block comment" -} - -func (*UnexpectedTokenInBlockCommentError) DocumentationLink() string { - return "https://cadence-lang.org/docs/language/syntax#comments" -} - // SpecialFunctionReturnTypeError is reported when a special function has a return type. type SpecialFunctionReturnTypeError struct { DeclarationKind common.DeclarationKind diff --git a/parser/expression.go b/parser/expression.go index 15438d3ff9..c37e235c1f 100644 --- a/parser/expression.go +++ b/parser/expression.go @@ -360,7 +360,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindBinary, - token.Range, + token, ), nil }, }) @@ -374,7 +374,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindOctal, - token.Range, + token, ), nil }, }) @@ -388,7 +388,7 @@ func init() { literal, literal, common.IntegerLiteralKindDecimal, - token.Range, + token, ), nil }, }) @@ -402,7 +402,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindHexadecimal, - token.Range, + token, ), nil }, }) @@ -416,7 +416,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindUnknown, - token.Range, + token, ), nil }, }) @@ -590,7 +590,7 @@ func defineLessThanOrTypeArgumentsExpression() { }) } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken, err := p.mustOne(lexer.TokenParenOpen) if err != nil { @@ -826,7 +826,7 @@ func defineIdentifierExpression() { current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // Skip the `fun` keyword @@ -847,6 +847,7 @@ func defineIdentifierExpression() { return ast.NewIdentifierExpression( p.memoryGauge, p.tokenToIdentifier(token), + token.Comments, ), nil }, }) @@ -960,7 +961,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordTo) { // Skip the `to` keyword @@ -976,7 +977,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewAttachExpression(p.memoryGauge, base, attachment, token.StartPos), nil } @@ -1016,7 +1017,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -1054,7 +1055,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as return nil, ast.EmptyPosition, err } - p.skipSpaceAndComments() + p.skipSpace() argument.TrailingSeparatorPos = p.current.StartPos @@ -1078,7 +1079,7 @@ func parseArgument(p *parser) (*ast.Argument, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() // If a colon follows the expression, the expression was our label. if p.current.Is(lexer.TokenColon) { @@ -1119,7 +1120,7 @@ func defineNestedExpression() { setExprNullDenotation( lexer.TokenParenOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() // special case: parse a Void literal `()` if p.current.Is(lexer.TokenParenClose) { @@ -1189,7 +1190,7 @@ func defineStringExpression() { literal = p.tokenSource(curToken) // remove quotation marks if they exist - if curToken == startToken { + if curToken.Equal(startToken) { literal = literal[1:] } @@ -1270,7 +1271,7 @@ func defineArrayExpression() { setExprNullDenotation( lexer.TokenBracketOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var values []ast.Expression @@ -1279,7 +1280,7 @@ func defineArrayExpression() { for !p.current.Is(lexer.TokenBracketClose) && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() if len(values) > 0 { if !p.current.Is(lexer.TokenComma) { @@ -1321,7 +1322,7 @@ func defineDictionaryExpression() { setExprNullDenotation( lexer.TokenBraceOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var entries []ast.DictionaryEntry @@ -1330,7 +1331,7 @@ func defineDictionaryExpression() { for !p.current.Is(lexer.TokenBraceClose) && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() if len(entries) > 0 { if !p.current.Is(lexer.TokenComma) { @@ -1525,7 +1526,7 @@ func parseMemberAccess(p *parser, token lexer.Token, left ast.Expression, option if p.current.Is(lexer.TokenSpace) { whitespaceToken := p.current - p.skipSpaceAndComments() + p.skipSpace() p.report(&WhitespaceAfterMemberAccessError{ OperatorTokenType: token.Type, WhitespaceRange: whitespaceToken.Range, @@ -1591,7 +1592,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { p.expressionDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -1609,7 +1610,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { // Some left denotations do not support newlines before them, // to avoid ambiguities and potential underhanded code - p.parseTrivia(triviaOptions{ + p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) @@ -1889,7 +1890,7 @@ func parseHex(r rune) rune { return -1 } -func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, tokenRange ast.Range) *ast.IntegerExpression { +func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, token lexer.Token) *ast.IntegerExpression { report := func(invalidKind InvalidNumberLiteralKind) { p.report( @@ -1898,7 +1899,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit InvalidIntegerLiteralKind: invalidKind, // NOTE: not using text, because it has the base-prefix stripped Literal: string(literal), - Range: tokenRange, + Range: token.Range, }, ) } @@ -1948,7 +1949,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, token.Range, token.Comments) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/parser/expression_test.go b/parser/expression_test.go index 52d0466ba7..3ddad824b2 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -31,10 +31,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/parser/lexer" + "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) @@ -3550,6 +3551,12 @@ func TestParseLineComment(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 1, Offset: 28}, EndPos: ast.Position{Line: 2, Column: 1, Offset: 28}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("//// // this is a comment")), + }, + Trailing: nil, + }, }, Right: &ast.IntegerExpression{ PositiveLiteral: []byte("2"), diff --git a/parser/function.go b/parser/function.go index cc8d8b3c6d..68cda67b5a 100644 --- a/parser/function.go +++ b/parser/function.go @@ -37,8 +37,9 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { var parameters []*ast.Parameter + var endToken lexer.Token - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenParenOpen) { return nil, &MissingStartOfParameterListError{ @@ -46,20 +47,17 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } } - startPos := p.current.StartPos + startToken := p.current // Skip the opening paren p.next() - var endPos ast.Position - expectParameter := true - var atEnd bool progress := p.newProgress() for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -87,7 +85,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL expectParameter = true case lexer.TokenParenClose: - endPos = p.current.EndPos + endToken = p.current // Skip the closing paren p.next() atEnd = true @@ -115,16 +113,19 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL parameters, ast.NewRange( p.memoryGauge, - startPos, - endPos, + startToken.StartPos, + endToken.EndPos, ), + ast.Comments{ + Trailing: endToken.Comments.Trailing, + }, ), nil } func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, error) { - p.skipSpaceAndComments() + p.skipSpace() - startPos := p.current.StartPos + startToken := p.current argumentLabel := "" identifier, err := p.nonReservedIdentifier("for argument label or parameter name") @@ -151,9 +152,10 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro p.nextSemanticToken() } - if !p.current.Is(lexer.TokenColon) { + colonToken := p.current + if !colonToken.Is(lexer.TokenColon) { return nil, &MissingColonAfterParameterNameError{ - GotToken: p.current, + GotToken: colonToken, } } @@ -166,7 +168,7 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var defaultArgument ast.Expression @@ -197,14 +199,20 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro identifier, typeAnnotation, defaultArgument, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: append( + startToken.Comments.All(), + colonToken.Comments.All()..., + ), + }, ), nil } func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { var typeParameters []*ast.TypeParameter - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenLess) { return nil, nil @@ -223,7 +231,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -285,7 +293,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { } func parseTypeParameter(p *parser) (*ast.TypeParameter, error) { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, &InvalidTypeParameterNameError{ @@ -319,15 +327,13 @@ func parseFunctionDeclaration( p *parser, functionBlockIsOptional bool, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, purity ast.FunctionPurity, - purityPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - docString string, + purityToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, purityPos, staticPos, nativePos) + startToken := lexer.EarliestToken(p.current, accessToken, purityToken, staticToken, nativeToken) // Skip the `fun` keyword p.nextSemanticToken() @@ -362,15 +368,17 @@ func parseFunctionDeclaration( p.memoryGauge, access, purity, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, identifier, typeParameterList, parameterList, returnTypeAnnotation, functionBlock, - startPos, - docString, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -395,7 +403,7 @@ func parseFunctionParameterListAndRest( current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenColon) { p.nextSemanticToken() @@ -414,7 +422,7 @@ func parseFunctionParameterListAndRest( if functionBlockIsOptional { current = p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenBraceOpen) { p.tokens.Revert(cursor) p.current = current diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index da0997ebd5..1d0b67ae51 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -56,6 +56,9 @@ const ( lexerModeStringInterpolation ) +// trailingCommentsEndMarker is a sentinel value for determining the end of trailing comments (before the first newline). +var trailingCommentsEndMarker *ast.Comment = nil + type lexer struct { // memoryGauge is used for metering memory usage memoryGauge common.MemoryGauge @@ -85,6 +88,13 @@ type lexer struct { mode lexerMode // counts the number of unclosed brackets for string templates \((())) openBrackets int + // indicates if tracking comments is enabled + trackComments bool + // currentComments stores the leading and/or trailing comments, + // waiting to be consumed and attached to respective tokens + // nil is used as sentinel value to track newlines + // Note: currentComments is only initialized when trackComments=true + currentComments []*ast.Comment } var _ TokenStream = &lexer{} @@ -103,6 +113,13 @@ func (l *lexer) Next() Token { endPos.column, ) + var comments ast.Comments + if len(l.currentComments) > 0 { + comments = ast.Comments{ + Leading: l.currentComments, + } + } + return Token{ Type: TokenEOF, Range: ast.NewRange( @@ -110,6 +127,7 @@ func (l *lexer) Next() Token { pos, pos, ), + Comments: comments, } } @@ -143,6 +161,8 @@ func (l *lexer) clear() { l.tokenCount = 0 l.mode = lexerModeNormal l.openBrackets = 0 + l.currentComments = l.currentComments[:0] + l.trackComments = false } func (l *lexer) Reclaim() { @@ -166,6 +186,17 @@ func Lex(input []byte, memoryGauge common.MemoryGauge) (TokenStream, error) { return l, err } +// LexWithComments is equivalent to Lex, but it enables comments tracking. +func LexWithComments(input []byte, memoryGauge common.MemoryGauge) (TokenStream, error) { + l := pool.Get().(*lexer) + l.clear() + l.memoryGauge = memoryGauge + l.input = input + l.trackComments = true + err := l.run(rootState) + return l, err +} + // run executes the stateFn, which will scan the runes in the input // and emit tokens. // @@ -197,6 +228,8 @@ func (l *lexer) run(state stateFn) (err error) { state = state(l) } + l.setPrevTokenTrailingComments() + return } @@ -277,23 +310,131 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), + Comments: l.tokenComments(ty), } l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) if consume { - l.startOffset = l.endOffset + l.consume(endPos) + } +} + +// tokenComments creates initial ast.Comments with leading comments, but empty trailing comments. +// Trailing comments for the current token are determined when the next non-space token is consumed (in setPrevTokenTrailingComments). +func (l *lexer) tokenComments(ty TokenType) ast.Comments { + // Do not attach comments to space tokens, since those are usually ignored during parsing, + // so mapping token comments to AST nodes would require extra effort. + if ty == TokenSpace || !l.trackComments { + return ast.EmptyComments + } - l.startPos = endPos - r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) + l.setPrevTokenTrailingComments() - if r == '\n' { - l.startPos.line++ - l.startPos.column = 0 + leadingComments := l.currentComments + defer (func() { + // Mark as fully consumed (clear by reallocating). + l.currentComments = []*ast.Comment{} + })() + + return ast.Comments{ + Leading: leadingComments, + // Trailing comments can't be determined yet, since we haven't consumed them. + Trailing: nil, + } +} + +// setPrevTokenTrailingComments sets the trailing comments of the latest non-space token, +// since they were not known yet at the time of calling tokenComments. +func (l *lexer) setPrevTokenTrailingComments() { + if l.tokenCount == 0 || !l.trackComments { + return + } + + latestNonSpaceTokenIndex := -1 + for i := l.tokenCount - 1; i >= 0; i-- { + if l.tokens[i].Type != TokenSpace { + latestNonSpaceTokenIndex = i + break + } + } + + // Split the current comments into two sets: + // - trailing comments of the previous token + // - leading comments of the next token + var trailing, leading []*ast.Comment + // If all previously seen tokens are spaces, + // treat all comments as the leading set for the current token, + // since comments shouldn't be attached to space tokens. + isLeadingSet := latestNonSpaceTokenIndex == -1 + for _, comment := range l.currentComments { + if comment == trailingCommentsEndMarker { + isLeadingSet = true + continue + } + if isLeadingSet { + leading = append(leading, comment) } else { - l.startPos.column++ + trailing = append(trailing, comment) + } + } + + l.currentComments = leading + + // If there is a trailing set, attach it to the latest non-space token (if any), + // since comments shouldn't be attached to space tokens. + if latestNonSpaceTokenIndex != -1 && len(trailing) > 0 { + lastNonSpaceToken := &l.tokens[latestNonSpaceTokenIndex] + if lastNonSpaceToken.Comments.Trailing == nil { + lastNonSpaceToken.Comments.Trailing = []*ast.Comment{} } + + lastNonSpaceToken.Comments.Trailing = append(lastNonSpaceToken.Comments.Trailing, trailing...) + } +} + +func (l *lexer) markTrailingCommentsEnd() { + l.currentComments = append(l.currentComments, trailingCommentsEndMarker) +} + +func (l *lexer) emitComment() { + endPos := l.endPos() + + if l.trackComments { + if l.currentComments == nil { + l.currentComments = []*ast.Comment{} + } + + commentRange := ast.NewRange( + l.memoryGauge, + l.startPosition(), + ast.NewPosition( + l.memoryGauge, + l.endOffset-1, + endPos.line, + endPos.column, + ), + ) + + l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, commentRange.Source(l.input))) + } + + l.consume(endPos) +} + +// endPos pre-computed end-position by calling l.endPos() +func (l *lexer) consume(endPos position) { + l.startOffset = l.endOffset + + l.startPos = endPos + r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) + + if r == '\n' { + l.startPos.line++ + l.startPos.column = 0 + } else { + l.startPos.column++ } } @@ -384,16 +525,6 @@ func IsIdentifierRune(r rune) bool { r == '_' } -func IsValidIdentifier(s string) bool { - for _, r := range s { - if !IsIdentifierRune(r) { - return false - } - } - - return true -} - func (l *lexer) scanLineComment() { // lookahead is already lexed. // parse more, if any @@ -493,3 +624,15 @@ func (l *lexer) scanFixedPointRemainder() { func isDecimalDigitOrUnderscore(r rune) bool { return (r >= '0' && r <= '9') || r == '_' } + +func IsValidIdentifier(s string) bool { + // Note: func is used by downstream dependants, do not remove. + + for _, r := range s { + if !IsIdentifierRune(r) { + return false + } + } + + return true +} diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 18d7ace5fc..9bdd9c3b23 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -63,7 +63,7 @@ func testLex(t *testing.T, input string, expected []token) { bytes := []byte(input) - tokenStream, err := Lex(bytes, nil) + tokenStream, err := LexWithComments(bytes, nil) require.NoError(t, err) withTokens(tokenStream, func(actualTokens []Token) { @@ -876,6 +876,1019 @@ func TestLexBasic(t *testing.T) { }, ) }) + + t.Run("trivia", func(t *testing.T) { + testLex(t, + "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", + []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 24, Offset: 24}, + EndPos: ast.Position{Line: 1, Column: 24, Offset: 24}, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 18, Offset: 43}, + EndPos: ast.Position{Line: 2, Column: 18, Offset: 43}, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, + EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in next line ")), + ast.NewComment(nil, []byte("/* test is here */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in the same line")), + }, + }, + }, + Source: "test", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + EndPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 51, Offset: 76}, + EndPos: ast.Position{Line: 2, Column: 51, Offset: 76}, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in previous line")), + }, + }, + }, + }, + }, + ) + }) +} + +func TestLexComments(t *testing.T) { + + t.Parallel() + + t.Run("simple", func(t *testing.T) { + testLex(t, + `/* before brace open */ { /* after brace open */ // after brace open 2 +// before brace close 1 +// before brace close 2 +}`, + []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* before brace open */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* after brace open */")), + ast.NewComment(nil, []byte("// after brace open 2")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + EndPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before brace close 1")), + ast.NewComment(nil, []byte("// before brace close 2")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + }, + }, + }, + }, + ) + }) + + t.Run("complex", func(t *testing.T) { + testLex(t, ` +// Before transaction identifier +transaction /* After transaction identifier */ ( + // Before first arg + a: Int, // After first arg + /* + Before second arg + */ + b: Int /* After second arg + + */ // After second arg 2 + // Before paren close +) /* After paren close */ { /* After brace open */ } // After brace close + +// Before EOF +`, []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + EndPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before transaction identifier")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After transaction identifier */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 34, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 44, + Line: 3, + Column: 10, + }, + }, + }, + Source: "transaction", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + EndPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + EndPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenParenOpen, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + EndPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + }, + }, + Source: "(", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 82, + Line: 3, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 83, + Line: 4, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 103, + Line: 4, + Column: 20, + }, + EndPos: ast.Position{ + Offset: 104, + Line: 5, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before first arg")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + }, + }, + Source: "a", + }, + { + Token: Token{ + Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + }, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 108, + Line: 5, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 110, + Line: 5, + Column: 6, + }, + }, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenComma, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After first arg ")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, + }, + }, + }, + Source: ",", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + EndPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 132, + Line: 5, + Column: 28, + }, + EndPos: ast.Position{ + Offset: 133, + Line: 6, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 159, + Line: 8, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 160, + Line: 9, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/*\n\tBefore second arg\n\t*/")), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + }, + }, + Source: "b", + }, + { + Token: Token{ + Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + }, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After second arg\n\n\t*/")), + ast.NewComment(nil, []byte("// After second arg 2")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 164, + Line: 9, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 166, + Line: 9, + Column: 6, + }, + }, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 214, + Line: 11, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 215, + Line: 12, + Column: 0, + }, + }, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + EndPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenParenClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before paren close")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After paren close */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, + }, + }, + }, + Source: ")", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After brace open */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, + }, + EndPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + EndPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + EndPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After brace close")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, + }, + EndPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + EndPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 311, + Line: 13, + Column: 73, + }, + EndPos: ast.Position{ + Offset: 312, + Line: 14, + Column: 0, + }, + }, + }, + Source: "\n\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 326, + Line: 15, + Column: 13, + }, + EndPos: ast.Position{ + Offset: 326, + Line: 15, + Column: 13, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenEOF, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before EOF")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 327, + Line: 16, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 327, + Line: 16, + Column: 0, + }, + }, + }, + }, + }) + }) } func TestLexString(t *testing.T) { @@ -1644,53 +2657,15 @@ func TestLexBlockComment(t *testing.T) { []token{ { Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Source: ` // *X `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 10, Offset: 10}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 17, Offset: 17}, - }, - }, - Source: ` \\* `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, + Type: TokenError, + SpaceOrError: MissingCommentEndError{}, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - EndPos: ast.Position{Line: 1, Column: 19, Offset: 19}, + StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, + EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, }, }, - Source: "*/", + // Error occurs at the EOF, which doesn't have a position within the source. + Source: "\000", }, { Token: Token{ @@ -1710,76 +2685,7 @@ func TestLexBlockComment(t *testing.T) { `/* test foo /* bar */ asd */ `, []token{ { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: ` test foo `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 14, Offset: 14}, - EndPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - }, - }, - Source: ` bar `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 19, Offset: 19}, - EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 21, Offset: 21}, - EndPos: ast.Position{Line: 1, Column: 25, Offset: 25}, - }, - }, - Source: ` asd `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 26, Offset: 26}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - }, - Source: "*/", - }, - { + Source: " ", Token: Token{ Type: TokenSpace, SpaceOrError: Space{ @@ -1790,7 +2696,6 @@ func TestLexBlockComment(t *testing.T) { EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, }, }, - Source: " ", }, { Token: Token{ @@ -1799,6 +2704,11 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, EndPos: ast.Position{Line: 1, Column: 30, Offset: 30}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* test foo /* bar */ asd */")), + }, + }, }, }, }, @@ -1809,26 +2719,6 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/**/`, []token{ - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, - Source: "*/", - }, { Token: Token{ Type: TokenEOF, @@ -1836,6 +2726,11 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/**/")), + }, + }, }, }, }, @@ -2692,6 +3587,11 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// bar ")), + }, + }, }, Source: "foo", }, @@ -2708,16 +3608,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenEOF, @@ -2757,6 +3647,11 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// bar ")), + }, + }, }, Source: "foo", }, @@ -2773,16 +3668,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenSpace, diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 96b20bec05..d143c4368b 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -146,8 +146,7 @@ func rootState(l *lexer) stateFn { case '/': return lineCommentState case '*': - l.emitType(TokenBlockCommentStart) - return blockCommentState(0) + return blockCommentState(l, 0) default: l.backupOne() l.emitType(TokenSlash) @@ -266,6 +265,8 @@ func numberState(l *lexer) stateFn { return rootState } +// Space must be preserved alongside the new comment structs, +// since some parsing code depends on it. type Space struct { ContainsNewline bool } @@ -275,16 +276,22 @@ func spaceState(startIsNewline bool) stateFn { containsNewline := l.scanSpace() containsNewline = containsNewline || startIsNewline + l.scanSpace() + common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) + if containsNewline { + // Trailing comments end before the first newline. + l.markTrailingCommentsEnd() + } + l.emit( TokenSpace, - Space{ - ContainsNewline: containsNewline, - }, + Space{ContainsNewline: containsNewline}, l.startPosition(), true, ) + return rootState } } @@ -317,12 +324,13 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitType(TokenLineComment) + l.emitComment() return rootState } -func blockCommentState(nesting int) stateFn { +func blockCommentState(l *lexer, nesting int) stateFn { if nesting < 0 { + l.emitComment() return rootState } @@ -330,6 +338,7 @@ func blockCommentState(nesting int) stateFn { r := l.next() switch r { case EOF: + l.emitError(MissingCommentEndError{}) return nil case '/': beforeSlashOffset := l.prevEndOffset @@ -337,11 +346,9 @@ func blockCommentState(nesting int) stateFn { if beforeSlashOffset-l.startOffset > 0 { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitType(TokenBlockCommentContent) l.endOffset = starOffset } - l.emitType(TokenBlockCommentStart) - return blockCommentState(nesting + 1) + return blockCommentState(l, nesting+1) } case '*': @@ -350,14 +357,22 @@ func blockCommentState(nesting int) stateFn { if beforeStarOffset-l.startOffset > 0 { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitType(TokenBlockCommentContent) l.endOffset = slashOffset } - l.emitType(TokenBlockCommentEnd) - return blockCommentState(nesting - 1) + return blockCommentState(l, nesting-1) } + + return blockCommentState(l, nesting) } - return blockCommentState(nesting) + return blockCommentState(l, nesting) } } + +type MissingCommentEndError struct{} + +var _ error = MissingCommentEndError{} + +func (MissingCommentEndError) Error() string { + return "missing comment end (`*/`)" +} diff --git a/parser/lexer/token.go b/parser/lexer/token.go index ccd5d25d9a..4f8e842138 100644 --- a/parser/lexer/token.go +++ b/parser/lexer/token.go @@ -24,8 +24,12 @@ import ( type Token struct { SpaceOrError any + Type TokenType ast.Range - Type TokenType + // Leading comments span from the end of the trailing comments of the last (non-space) token to the start of the next (non-space) token. + // Trailing comments span up to, but not including, the next newline character after the current (non-space) token. + // Important: comments are not tracked for space token, since those are usually ignored in the parser. + Comments ast.Comments } func (t Token) Is(ty TokenType) bool { @@ -33,7 +37,20 @@ func (t Token) Is(ty TokenType) bool { } func (t Token) Source(input []byte) []byte { - startOffset := t.StartPos.Offset - endOffset := t.EndPos.Offset + 1 - return input[startOffset:endOffset] + return t.Range.Source(input) +} + +func (t Token) Equal(other Token) bool { + // ignore comments, since they should not be treated as source code + return t.Type == other.Type && t.Range == other.Range && t.SpaceOrError == other.SpaceOrError +} + +func EarliestToken(t Token, ts ...*Token) (earliest Token) { + earliest = t + for _, tok := range ts { + if tok != nil && tok.StartPos.Compare(earliest.StartPos) < 0 { + earliest = *tok + } + } + return } diff --git a/parser/lexer/tokentype.go b/parser/lexer/tokentype.go index db3a446411..df807d0a9d 100644 --- a/parser/lexer/tokentype.go +++ b/parser/lexer/tokentype.go @@ -69,10 +69,6 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual - TokenBlockCommentStart - TokenBlockCommentEnd - TokenBlockCommentContent - TokenLineComment TokenAmpersand TokenAmpersandAmpersand TokenCaret @@ -180,14 +176,6 @@ func (t TokenType) String() string { return "`!`" case TokenNotEqual: return "`!=`" - case TokenBlockCommentStart: - return "`/*`" - case TokenBlockCommentContent: - return "block comment" - case TokenLineComment: - return "line comment" - case TokenBlockCommentEnd: - return "`*/`" case TokenAmpersand: return "`&`" case TokenAmpersandAmpersand: diff --git a/parser/parser.go b/parser/parser.go index 47f792592e..43445ed76e 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -19,9 +19,7 @@ package parser import ( - "bytes" "os" - "strings" "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" @@ -57,6 +55,8 @@ type Config struct { IgnoreLeadingIdentifierEnabled bool // TypeParametersEnabled determines if type parameters are enabled TypeParametersEnabled bool + // CommentTrackingEnabled determines if comment should be parsed and tracked in lexer and parser. + CommentTrackingEnabled bool } type parser struct { @@ -100,7 +100,7 @@ func Parse[T any]( config Config, ) (result T, errors []error) { // create a lexer, which turns the input string into tokens - tokens, err := lexer.Lex(input, memoryGauge) + tokens, err := lex(input, memoryGauge, config.CommentTrackingEnabled) if err != nil { errors = append(errors, err) return @@ -192,7 +192,7 @@ func ParseTokenStream[T any]( return result, p.errors } - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenEOF) { p.report(&UnexpectedTokenAtEndError{ @@ -203,6 +203,14 @@ func ParseTokenStream[T any]( return result, p.errors } +func lex(input []byte, memoryGauge common.MemoryGauge, trackComments bool) (lexer.TokenStream, error) { + if trackComments { + return lexer.LexWithComments(input, memoryGauge) + } else { + return lexer.Lex(input, memoryGauge) + } +} + func (p *parser) newSyntaxError(message string, params ...any) *SyntaxError { return NewSyntaxError(p.current.StartPos, message, params...) } @@ -256,13 +264,26 @@ func (p *parser) next() { if !ok { panic(errors.NewUnreachableError()) } - parseError, ok := err.(ParseError) - if !ok { + + var parseError ParseError + switch err := err.(type) { + case ParseError: + // already a parse error, do nothing + parseError = err + + case lexer.MissingCommentEndError: + parseError = &MissingCommentEndError{ + Pos: token.StartPos, + } + + default: + // wrap other errors as syntax errors parseError = &SyntaxError{ Pos: token.StartPos, Message: err.Error(), } } + p.report(parseError) continue } @@ -277,7 +298,7 @@ func (p *parser) next() { // It skips whitespace, including newlines, and comments func (p *parser) nextSemanticToken() { p.next() - p.skipSpaceAndComments() + p.skipSpace() } func (p *parser) mustOne(tokenType lexer.TokenType) (lexer.Token, error) { @@ -420,32 +441,19 @@ func (p *parser) replayBuffered() error { return nil } -type triviaOptions struct { - skipNewlines bool - parseDocStrings bool +type skipSpaceOptions struct { + skipNewlines bool } -// skipSpaceAndComments skips whitespace, including newlines, and comments -func (p *parser) skipSpaceAndComments() (containsNewline bool) { - containsNewline, _ = p.parseTrivia(triviaOptions{ +// skipSpace skips whitespace, including newlines, and comments +func (p *parser) skipSpace() (containsNewline bool) { + containsNewline = p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) return } -var blockCommentDocStringPrefix = []byte("/**") -var lineCommentDocStringPrefix = []byte("///") - -func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { - var docStringBuilder strings.Builder - defer func() { - if options.parseDocStrings { - docString = docStringBuilder.String() - } - }() - - var insideLineDocString bool - +func (p *parser) skipSpaceWithOptions(options skipSpaceOptions) (containsNewline bool) { var atEnd bool progress := p.newProgress() @@ -469,43 +477,6 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() - case lexer.TokenBlockCommentStart: - commentStartOffset := p.current.StartPos.Offset - endToken, ok := p.parseBlockComment() - - if ok && options.parseDocStrings { - commentEndOffset := endToken.EndPos.Offset - - contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] - - insideLineDocString = false - docStringBuilder.Reset() - if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { - // Strip prefix (`/**`) - docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) - } - } - - case lexer.TokenLineComment: - if options.parseDocStrings { - comment := p.currentTokenSource() - if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { - if insideLineDocString { - docStringBuilder.WriteByte('\n') - } else { - insideLineDocString = true - docStringBuilder.Reset() - } - // Strip prefix - docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) - } else { - insideLineDocString = false - docStringBuilder.Reset() - } - } - - p.next() - default: atEnd = true } @@ -688,7 +659,7 @@ func ParseArgumentList( memoryGauge, input, func(p *parser) (ast.Arguments, error) { - p.skipSpaceAndComments() + p.skipSpace() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { @@ -703,7 +674,7 @@ func ParseArgumentList( } func ParseProgram(memoryGauge common.MemoryGauge, code []byte, config Config) (program *ast.Program, err error) { - tokens, err := lexer.Lex(code, memoryGauge) + tokens, err := lex(code, memoryGauge, config.CommentTrackingEnabled) if err != nil { return } diff --git a/parser/parser_test.go b/parser/parser_test.go index 8cf0c433bd..08f5d1bb4c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -105,7 +105,7 @@ func testParseStatements(s string) ([]ast.Statement, []error) { } func testParseStatementsWithConfig(s string, config Config) ([]ast.Statement, []error) { - statements, errs := ParseStatements(nil, []byte(s), config) + statements, errs := ParseStatements(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return statements, errs } @@ -115,7 +115,7 @@ func testParseDeclarations(s string) ([]ast.Declaration, []error) { } func testParseDeclarationsWithConfig(s string, config Config) ([]ast.Declaration, []error) { - declarations, errs := ParseDeclarations(nil, []byte(s), config) + declarations, errs := ParseDeclarations(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return declarations, errs } @@ -125,7 +125,7 @@ func testParseProgram(s string) (*ast.Program, error) { } func testParseProgramWithConfig(s string, config Config) (*ast.Program, error) { - program, err := ParseProgram(nil, []byte(s), config) + program, err := ParseProgram(nil, []byte(s), configWithCommentTracking(config)) if err != nil { _ = err.Error() } @@ -137,7 +137,7 @@ func testParseExpression(s string) (ast.Expression, []error) { } func testParseExpressionWithConfig(s string, config Config) (ast.Expression, []error) { - expression, errs := ParseExpression(nil, []byte(s), config) + expression, errs := ParseExpression(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return expression, errs } @@ -147,7 +147,7 @@ func testParseArgumentList(s string) (ast.Arguments, []error) { } func testParseArgumentListWithConfig(s string, config Config) (ast.Arguments, []error) { - arguments, errs := ParseArgumentList(nil, []byte(s), config) + arguments, errs := ParseArgumentList(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return arguments, errs } @@ -157,11 +157,16 @@ func testParseType(s string) (ast.Type, []error) { } func testParseTypeWithConfig(s string, config Config) (ast.Type, []error) { - ty, errs := ParseType(nil, []byte(s), config) + ty, errs := ParseType(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return ty, errs } +func configWithCommentTracking(config Config) Config { + config.CommentTrackingEnabled = true + return config +} + func TestParseInvalid(t *testing.T) { t.Parallel() @@ -600,8 +605,8 @@ func TestParseBuffering(t *testing.T) { []error{ &RestrictedTypeError{ Range: ast.Range{ - StartPos: ast.Position{Offset: 138, Line: 4, Column: 55}, - EndPos: ast.Position{Offset: 139, Line: 4, Column: 56}, + StartPos: ast.Position{Offset: 145, Line: 4, Column: 62}, + EndPos: ast.Position{Offset: 145, Line: 4, Column: 62}, }, }, }, @@ -687,7 +692,7 @@ func TestParseEOF(t *testing.T) { if err != nil { return struct{}{}, err } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustToken(lexer.TokenIdentifier, "b") if err != nil { return struct{}{}, err diff --git a/parser/statement.go b/parser/statement.go index 219bf9b275..4b5569a4b8 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -31,7 +31,7 @@ func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statem for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: @@ -77,7 +77,7 @@ func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statem } func parseStatement(p *parser) (ast.Statement, error) { - p.skipSpaceAndComments() + p.skipSpace() // Flag for cases where we can tell early-on that the current token isn't being used as a keyword // e.g. soft keywords like `view` @@ -109,17 +109,16 @@ func parseStatement(p *parser) (ast.Statement, error) { case KeywordView: // save current stream state before looking ahead for the `fun` keyword cursor := p.tokens.Cursor() - current := p.current - purityPos := current.StartPos + purityToken := p.current p.nextSemanticToken() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { - return parseFunctionDeclarationOrFunctionExpressionStatement(p, ast.FunctionPurityView, &purityPos) + return parseFunctionDeclarationOrFunctionExpressionStatement(p, ast.FunctionPurityView, &purityToken) } // no `fun` :( revert back to previous lexer state and treat it as an identifier p.tokens.Revert(cursor) - p.current = current + p.current = purityToken tokenIsIdentifier = true case KeywordFun: @@ -133,7 +132,7 @@ func parseStatement(p *parser) (ast.Statement, error) { if !tokenIsIdentifier { // If it is not a keyword for a statement, // it might start with a keyword for a declaration - declaration, err := parseDeclaration(p, "") + declaration, err := parseDeclaration(p) if err != nil { return nil, err } @@ -154,7 +153,7 @@ func parseStatement(p *parser) (ast.Statement, error) { // If the expression is followed by a transfer, // it is actually the target of an assignment or swap statement - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenEqual, lexer.TokenLeftArrow, lexer.TokenLeftArrowExclamation: transfer := parseTransfer(p) @@ -184,10 +183,20 @@ func parseStatement(p *parser) (ast.Statement, error) { func parseFunctionDeclarationOrFunctionExpressionStatement( p *parser, purity ast.FunctionPurity, - purityPos *ast.Position, + purityToken *lexer.Token, ) (ast.Statement, error) { + var leadingComments []*ast.Comment + var startPos ast.Position - startPos := *ast.EarlierPosition(&p.current.StartPos, purityPos) + funToken := p.current + if purityToken == nil { + startPos = funToken.StartPos + } else { + startPos = *ast.EarlierPosition(&funToken.StartPos, &purityToken.StartPos) + leadingComments = append(leadingComments, purityToken.Comments.All()...) + } + + leadingComments = append(leadingComments, funToken.Comments.All()...) // Skip the `fun` keyword p.nextSemanticToken() @@ -229,7 +238,9 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( returnTypeAnnotation, functionBlock, startPos, - "", + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { parameterList, returnTypeAnnotation, functionBlock, err := @@ -253,19 +264,25 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( } func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { - tokenRange := p.current.Range + startToken := p.current + tokenRange := startToken.Range endPosition := tokenRange.EndPos p.next() - sawNewLine, _ := p.parseTrivia(triviaOptions{ + sawNewLine := p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) var expression ast.Expression var err error + var endToken *lexer.Token + var comments ast.Comments switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: - break + tok := p.current + endToken = &tok + comments.Leading = startToken.Comments.All() + comments.Trailing = endToken.Comments.All() default: if !sawNewLine { expression, err = parseExpression(p, lowestBindingPower) @@ -275,6 +292,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { endPosition = expression.EndPosition(p.memoryGauge) } + comments = startToken.Comments } return ast.NewReturnStatement( @@ -285,21 +303,22 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + comments, ), nil } func parseBreakStatement(p *parser) *ast.BreakStatement { - tokenRange := p.current.Range + breakToken := p.current p.next() - return ast.NewBreakStatement(p.memoryGauge, tokenRange) + return ast.NewBreakStatement(p.memoryGauge, breakToken.Range, breakToken.Comments) } func parseContinueStatement(p *parser) *ast.ContinueStatement { - tokenRange := p.current.Range + continueToken := p.current p.next() - return ast.NewContinueStatement(p.memoryGauge, tokenRange) + return ast.NewContinueStatement(p.memoryGauge, continueToken.Range, continueToken.Comments) } func parseIfStatement(p *parser) (*ast.IfStatement, error) { @@ -310,7 +329,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { for p.checkProgress(&progress) { - startPos := p.current.StartPos + startToken := p.current p.nextSemanticToken() @@ -322,7 +341,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { case KeywordLet: const isLet = true variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil, isLet, "") + parseVariableDeclaration(p, ast.AccessNotSpecified, nil, isLet) if err != nil { return nil, err } @@ -330,7 +349,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { case KeywordVar: const isLet = false variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil, isLet, "") + parseVariableDeclaration(p, ast.AccessNotSpecified, nil, isLet) if err != nil { return nil, err } @@ -355,11 +374,19 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { parseNested := false - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordElse) { + elseToken := p.current // Skip the `else` keyword p.nextSemanticToken() + // The parser ignores the `else` token, + // so to preserve potential comments associated with else token, + // we attach the comments to the (next) `if` token. + leadingComments := elseToken.Comments.All() + leadingComments = append(leadingComments, p.current.Comments.Leading...) + p.current.Comments.Leading = leadingComments + if p.isToken(p.current, lexer.TokenIdentifier, KeywordIf) { parseNested = true } else { @@ -385,7 +412,10 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { test, thenBlock, elseBlock, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.All(), + }, ) if variableDeclaration != nil { @@ -409,6 +439,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.EmptyComments, ) result = outer } @@ -418,7 +449,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() expression, err := parseExpression(p, lowestBindingPower) @@ -431,12 +462,20 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { return nil, err } - return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos), nil + return ast.NewWhileStatement( + p.memoryGauge, + expression, + block, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.All(), + }, + ), nil } func parseForStatement(p *parser) (*ast.ForStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.nextSemanticToken() firstValue, err := p.mustIdentifier() @@ -450,7 +489,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { }) } - p.skipSpaceAndComments() + p.skipSpace() var index *ast.Identifier var identifier ast.Identifier @@ -464,12 +503,14 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() } else { identifier = firstValue } + var inToken lexer.Token if p.isToken(p.current, lexer.TokenIdentifier, KeywordIn) { + inToken = p.current // Skip the `in` keyword p.nextSemanticToken() } else { @@ -494,7 +535,11 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { index, block, expression, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.All(), + Trailing: inToken.Comments.All(), + }, ), nil } @@ -524,7 +569,7 @@ func parseBraces[T any]( return } - p.skipSpaceAndComments() + p.skipSpace() endToken = p.current @@ -562,11 +607,15 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.All(), + Trailing: endToken.Comments.All(), + }, ), nil } func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { - p.skipSpaceAndComments() + p.skipSpace() type bracesResult struct { preConditions *ast.Conditions @@ -579,7 +628,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { "function block", func() (res bracesResult, err error) { - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { prePos := p.current.StartPos @@ -593,7 +642,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { } } - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordPost) { startPos := p.current.StartPos @@ -628,6 +677,10 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.Leading, + Trailing: endToken.Comments.Trailing, + }, ), res.preConditions, res.postConditions, @@ -637,7 +690,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { // parseConditions parses conditions (pre/post) func parseConditions(p *parser, startPos ast.Position) (*ast.Conditions, error) { - p.skipSpaceAndComments() + p.skipSpace() conditions, _, endToken, err := parseBraces( p, @@ -649,7 +702,7 @@ func parseConditions(p *parser, startPos ast.Position) (*ast.Conditions, error) for !done && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: @@ -709,7 +762,7 @@ func parseCondition(p *parser) (ast.Condition, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var message ast.Expression if p.current.Is(lexer.TokenColon) { @@ -728,7 +781,7 @@ func parseCondition(p *parser) (ast.Condition, error) { } func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() invocation, err := parseNominalTypeInvocationRemainder(p) @@ -736,12 +789,19 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { return nil, err } - return ast.NewEmitStatement(p.memoryGauge, invocation, startPos), nil + return ast.NewEmitStatement( + p.memoryGauge, + invocation, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.All(), + }, + ), nil } func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the `switch` keyword p.next() @@ -768,9 +828,13 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { cases, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.All(), + Trailing: endToken.Comments.All(), + }, ), nil } @@ -783,7 +847,7 @@ func parseSwitchCases(p *parser) (cases []*ast.SwitchCase, err error) { for p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -833,7 +897,7 @@ func parseSwitchCases(p *parser) (cases []*ast.SwitchCase, err error) { // | `default` `:` statements func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the keyword p.next() @@ -847,10 +911,10 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { return nil, err } } else { - p.skipSpaceAndComments() + p.skipSpace() } - colonPos := p.current.StartPos + colonToken := p.current if p.current.Is(lexer.TokenColon) { p.next() @@ -881,7 +945,7 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { return nil, err } - endPos := colonPos + endPos := colonToken.StartPos if len(statements) > 0 { lastStatementIndex := len(statements) - 1 @@ -894,9 +958,13 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { statements, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endPos, ), + ast.Comments{ + Leading: startToken.Comments.All(), + Trailing: colonToken.Comments.All(), + }, ), nil } @@ -904,9 +972,9 @@ func parseRemoveStatement( p *parser, ) (*ast.RemoveStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() - p.skipSpaceAndComments() + p.skipSpace() ty, err := parseType(p, lowestBindingPower) if err != nil { @@ -920,9 +988,11 @@ func parseRemoveStatement( }) } - p.skipSpaceAndComments() + p.skipSpace() + var fromToken lexer.Token if p.isToken(p.current, lexer.TokenIdentifier, KeywordFrom) { + fromToken = p.current // Skip the `from` keyword p.nextSemanticToken() } else { @@ -936,10 +1006,16 @@ func parseRemoveStatement( return nil, err } + leadingComments := startToken.Comments.All() + leadingComments = append(leadingComments, fromToken.Comments.All()...) + return ast.NewRemoveStatement( p.memoryGauge, attachmentNominalType, attached, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: leadingComments, + }, ), nil } diff --git a/parser/statement_test.go b/parser/statement_test.go index d85cfbd648..717c9e2c7e 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -80,6 +80,68 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("no expression, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return // After return +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + + t.Run("no expression, semicolon, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return ; // After return +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on same line", func(t *testing.T) { t.Parallel() @@ -109,6 +171,49 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("expression on same line, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return 1 // After return +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + // TODO(preserve-comments): Should we attach this to expression or statement node? + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on next line, no semicolon", func(t *testing.T) { t.Parallel() @@ -577,42 +682,6 @@ func TestParseIfStatement(t *testing.T) { }) } -func TestParseWhileStatement(t *testing.T) { - - t.Parallel() - - t.Run("empty block", func(t *testing.T) { - - t.Parallel() - - result, errs := testParseStatements("while true { }") - require.Empty(t, errs) - - AssertEqualWithDiff(t, - []ast.Statement{ - &ast.WhileStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Block: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - result, - ) - }) -} - func TestParseAssignmentStatement(t *testing.T) { t.Parallel() @@ -828,6 +897,58 @@ func TestParseForStatement(t *testing.T) { result, ) }) + + t.Run("empty block, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before for +for /* after for */ x /* before in */ in /* after in */ y { } +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ForStatement{ + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 20, Offset: 35}, + // TODO(preserve-comments): Handle identifier comments + //Comments: ast.Comments{ + // Trailing: []*ast.Comment{ + // ast.NewComment(nil, []byte("/* before in */")), + // }, + //}, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "y", + Pos: ast.Position{Line: 3, Column: 56, Offset: 71}, + }, + }, + Block: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 58, Offset: 73}, + EndPos: ast.Position{Line: 3, Column: 60, Offset: 75}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 15}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before for")), + ast.NewComment(nil, []byte("/* after for */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* after in */")), + }, + }, + }, + }, + result, + ) + }) } func TestParseForStatementIndexBinding(t *testing.T) { @@ -1027,6 +1148,42 @@ func TestParseEmit(t *testing.T) { fixes[0].TextEdits[0].ApplyTo(code), ) }) + + t.Run("simple, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before emit 1 +/* Before emit 2 */ emit T() +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.EmitStatement{ + InvocationExpression: &ast.InvocationExpression{ + InvokedExpression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "T", + Pos: ast.Position{Line: 3, Column: 25, Offset: 43}, + }, + }, + ArgumentsStartPos: ast.Position{Line: 3, Column: 26, Offset: 44}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 45}, + }, + StartPos: ast.Position{Line: 3, Column: 20, Offset: 38}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before emit 1")), + ast.NewComment(nil, []byte("/* Before emit 2 */")), + }, + }, + }, + }, + result, + ) + }) } func TestParseFunctionStatementOrExpression(t *testing.T) { @@ -1378,6 +1535,46 @@ func TestParseRemoveAttachmentStatement(t *testing.T) { ) }) + t.Run("basic, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before remove +remove A +// Before from +from b +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.RemoveStatement{ + Attachment: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "A", + Pos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Line: 5, Column: 5, Offset: 47}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before remove")), + ast.NewComment(nil, []byte("// Before from")), + }, + }, + }, + }, + result, + ) + }) + t.Run("namespaced attachment", func(t *testing.T) { t.Parallel() @@ -1702,6 +1899,132 @@ func TestParseSwitchStatement(t *testing.T) { ) }) + t.Run("two cases, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` + // Before switch + switch x { + // Before first case + case true : // After first case + true + // Before default case + default : /* After default case */ false + } // After switch +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.SwitchStatement{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 10, Offset: 31}, + }, + }, + Cases: []*ast.SwitchCase{ + { + Expression: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 70, + Line: 5, + Column: 9, + }, + EndPos: ast.Position{ + Offset: 73, + Line: 5, + Column: 12, + }, + }, + }, + Statements: []ast.Statement{ + &ast.ExpressionStatement{ + Expression: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 102, + Line: 6, + Column: 5, + }, + EndPos: ast.Position{ + Offset: 105, + Line: 6, + Column: 8, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 5, Column: 4, Offset: 65}, + EndPos: ast.Position{Line: 6, Column: 8, Offset: 105}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before first case")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After first case")), + }, + }, + }, + { + Statements: []ast.Statement{ + &ast.ExpressionStatement{ + Expression: &ast.BoolExpression{ + Value: false, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 173, + Line: 8, + Column: 39, + }, + EndPos: ast.Position{ + Offset: 177, + Line: 8, + Column: 43, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 8, Column: 4, Offset: 138}, + EndPos: ast.Position{Line: 8, Column: 43, Offset: 177}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before default case")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After default case */")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 24}, + EndPos: ast.Position{Line: 9, Column: 3, Offset: 182}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before switch")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After switch")), + }, + }, + }, + }, + result, + ) + }) + t.Run("invalid identifiers in switch cases", func(t *testing.T) { t.Parallel() @@ -2122,87 +2445,359 @@ func TestParseIfStatementNoElse(t *testing.T) { ) } -func TestParseWhileStatementInFunctionDeclaration(t *testing.T) { +func TestParseIfStatementWithComments(t *testing.T) { t.Parallel() - const code = ` - fun test() { - while true { - return - break - continue - } - } - ` - result, errs := testParseProgram(code) + result, errs := testParseStatements(` +// before if +if true { + // noop +} // after if +// before else-if +else if true { + // noop +} +// before second else-if +else if true { + // noop +} /* after else-if */ else { + // noop +} // after else +`) require.Empty(t, errs) AssertEqualWithDiff(t, - []ast.Declaration{ - &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Identifier: ast.Identifier{ - Identifier: "test", - Pos: ast.Position{Offset: 10, Line: 2, Column: 9}, + []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, + EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, + }, }, - ParameterList: &ast.ParameterList{ + Then: &ast.Block{ + Statements: nil, Range: ast.Range{ - StartPos: ast.Position{Offset: 14, Line: 2, Column: 13}, - EndPos: ast.Position{Offset: 15, Line: 2, Column: 14}, + StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, + EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after if")), + }, }, }, - FunctionBlock: &ast.FunctionBlock{ - Block: &ast.Block{ - Statements: []ast.Statement{ - &ast.WhileStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Offset: 37, Line: 3, Column: 18}, - EndPos: ast.Position{Offset: 40, Line: 3, Column: 21}, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, + EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, + EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), }, }, - Block: &ast.Block{ - Statements: []ast.Statement{ - &ast.ReturnStatement{ - Expression: nil, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, Range: ast.Range{ - StartPos: ast.Position{Offset: 58, Line: 4, Column: 14}, - EndPos: ast.Position{Offset: 63, Line: 4, Column: 19}, + StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, + EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, }, }, - &ast.BreakStatement{ + Then: &ast.Block{ + Statements: nil, Range: ast.Range{ - StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, - EndPos: ast.Position{Offset: 83, Line: 5, Column: 18}, + StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, + EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("/* after else-if */")), + }, }, }, - &ast.ContinueStatement{ + Else: &ast.Block{ + Statements: nil, Range: ast.Range{ - StartPos: ast.Position{Offset: 99, Line: 6, Column: 14}, - EndPos: ast.Position{Offset: 106, Line: 6, Column: 21}, + StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after else")), + }, + }, + }, + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before second else-if")), }, }, }, - Range: ast.Range{ - StartPos: ast.Position{Offset: 42, Line: 3, Column: 23}, - EndPos: ast.Position{Offset: 120, Line: 7, Column: 12}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before if")), + }, + }, + }, + }, + result, + ) +} + +func TestParseWhileStatement(t *testing.T) { + + t.Parallel() + + t.Run("while in function", func(t *testing.T) { + const code = ` + fun test() { + while true { + return + break + continue + } + } + ` + result, errs := testParseProgram(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.FunctionDeclaration{ + Access: ast.AccessNotSpecified, + Identifier: ast.Identifier{ + Identifier: "test", + Pos: ast.Position{Offset: 10, Line: 2, Column: 9}, + }, + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 14, Line: 2, Column: 13}, + EndPos: ast.Position{Offset: 15, Line: 2, Column: 14}, + }, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.WhileStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Offset: 37, Line: 3, Column: 18}, + EndPos: ast.Position{Offset: 40, Line: 3, Column: 21}, + }, }, + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 58, Line: 4, Column: 14}, + EndPos: ast.Position{Offset: 63, Line: 4, Column: 19}, + }, + }, + &ast.BreakStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, + EndPos: ast.Position{Offset: 83, Line: 5, Column: 18}, + }, + }, + &ast.ContinueStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 99, Line: 6, Column: 14}, + EndPos: ast.Position{Offset: 106, Line: 6, Column: 21}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 42, Line: 3, Column: 23}, + EndPos: ast.Position{Offset: 120, Line: 7, Column: 12}, + }, + }, + StartPos: ast.Position{Offset: 31, Line: 3, Column: 12}, }, - StartPos: ast.Position{Offset: 31, Line: 3, Column: 12}, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 17, Line: 2, Column: 16}, + EndPos: ast.Position{Offset: 130, Line: 8, Column: 8}, }, }, + }, + StartPos: ast.Position{Offset: 6, Line: 2, Column: 5}, + }, + }, + result.Declarations(), + ) + }) + + t.Run("while, comments", func(t *testing.T) { + const code = ` + // before while + while true { + // ignore + } // after while + ` + result, errs := testParseStatements(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.WhileStatement{ + Test: &ast.BoolExpression{ + Value: true, Range: ast.Range{ - StartPos: ast.Position{Offset: 17, Line: 2, Column: 16}, - EndPos: ast.Position{Offset: 130, Line: 8, Column: 8}, + StartPos: ast.Position{Offset: 30, Line: 3, Column: 11}, + EndPos: ast.Position{Offset: 33, Line: 3, Column: 14}, + }, + }, + Block: &ast.Block{ + Statements: []ast.Statement{}, + Range: ast.Range{ + StartPos: ast.Position{Offset: 35, Line: 3, Column: 16}, + EndPos: ast.Position{Offset: 53, Line: 5, Column: 2}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// ignore")), + ast.NewComment(nil, []byte("// after while")), + }, + }, + }, + StartPos: ast.Position{Offset: 24, Line: 3, Column: 5}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before while")), }, }, }, - StartPos: ast.Position{Offset: 6, Line: 2, Column: 5}, + }, + result, + ) + }) + +} + +func TestParseReturnBreakContinueWithComments(t *testing.T) { + + t.Parallel() + + const code = ` + // Before return + return // After return + // Before return semicolon + return /* After return, before semicolon */ ; // After semicolon + // Before break + break // After break + // Before continue + continue // After continue + ` + result, errs := testParseStatements(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 31, Line: 3, Column: 8}, + EndPos: ast.Position{Offset: 36, Line: 3, Column: 13}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 91, Line: 5, Column: 8}, + EndPos: ast.Position{Offset: 96, Line: 5, Column: 13}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return semicolon")), + ast.NewComment(nil, []byte("/* After return, before semicolon */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After semicolon")), + }, + }, + }, + &ast.BreakStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 182, Line: 7, Column: 8}, + EndPos: ast.Position{Offset: 186, Line: 7, Column: 12}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before break")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After break")), + }, + }, + }, + &ast.ContinueStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 232, Line: 9, Column: 8}, + EndPos: ast.Position{Offset: 239, Line: 9, Column: 15}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before continue")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After continue")), + }, + }, }, }, - result.Declarations(), + result, ) } diff --git a/parser/transaction.go b/parser/transaction.go index de70af87f1..878f9d2c43 100644 --- a/parser/transaction.go +++ b/parser/transaction.go @@ -40,9 +40,9 @@ import ( // | /* no execute or postConditions */ // ) // '}' -func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionDeclaration, error) { +func parseTransactionDeclaration(p *parser) (*ast.TransactionDeclaration, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the `transaction` keyword p.nextSemanticToken() @@ -59,7 +59,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD } } - p.skipSpaceAndComments() + p.skipSpace() parseDeclarationOpeningBrace(p, common.DeclarationKindTransaction) @@ -75,7 +75,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var prepare *ast.SpecialFunctionDeclaration var execute *ast.SpecialFunctionDeclaration - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenIdentifier) { @@ -83,7 +83,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD switch string(keyword) { case KeywordPrepare: - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current // Skip the `prepare` keyword p.next() prepare, err = parseSpecialFunctionDeclaration( @@ -95,8 +95,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD nil, nil, nil, - identifier, - "", + identifierToken, ) if err != nil { return nil, err @@ -121,7 +120,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var preConditions *ast.Conditions if execute == nil { - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { preStartPos := p.current.StartPos @@ -149,7 +148,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -211,12 +210,14 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endPos, ), + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -225,9 +226,8 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro for p.checkProgress(&progress) { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -248,7 +248,6 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro nil, nil, nil, - docString, ) if err != nil { return nil, err @@ -270,7 +269,8 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro } func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) { - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current + identifier := p.tokenToIdentifier(identifierToken) // Skip the `execute` keyword p.nextSemanticToken() @@ -300,7 +300,9 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - "", + ast.Comments{ + Leading: identifierToken.Comments.Leading, + }, ), ), nil } diff --git a/parser/type.go b/parser/type.go index e38c6dac09..81b409ab1b 100644 --- a/parser/type.go +++ b/parser/type.go @@ -148,13 +148,13 @@ func init() { func defineParenthesizedTypes() { setTypeNullDenotation(lexer.TokenParenOpen, func(p *parser, token lexer.Token) (ast.Type, error) { - p.skipSpaceAndComments() + p.skipSpace() innerType, err := parseType(p, lowestBindingPower) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenParenClose) { p.next() @@ -170,6 +170,8 @@ func defineParenthesizedTypes() { func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, error) { var nestedIdentifiers []ast.Identifier + // Stores the last token at the end of the loop + var nestedToken *lexer.Token progress := p.newProgress() @@ -179,15 +181,16 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, // Skip the dot p.next() - nestedToken := p.current + tok := p.current + nestedToken = &tok if !nestedToken.Is(lexer.TokenIdentifier) { return nil, &NestedTypeMissingNameError{ - GotToken: nestedToken, + GotToken: *nestedToken, } } - nestedIdentifier := p.tokenToIdentifier(nestedToken) + nestedIdentifier := p.tokenToIdentifier(*nestedToken) // Skip the identifier p.next() @@ -198,10 +201,21 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, ) } + var trailingComments []*ast.Comment + if nestedToken == nil { + trailingComments = token.Comments.Trailing + } else { + trailingComments = nestedToken.Comments.All() + } + return ast.NewNominalType( p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, + ast.Comments{ + Leading: token.Comments.Leading, + Trailing: trailingComments, + }, ), nil } @@ -215,7 +229,7 @@ func defineArrayType() { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var size *ast.IntegerExpression @@ -237,7 +251,7 @@ func defineArrayType() { } } - p.skipSpaceAndComments() + p.skipSpace() endToken := p.current if p.current.Is(lexer.TokenBracketClose) { @@ -343,7 +357,7 @@ func defineIntersectionOrDictionaryType() { for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -573,7 +587,7 @@ func parseNominalTypes( for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case separator: @@ -642,7 +656,7 @@ func parseType(p *parser, rightBindingPower int) (ast.Type, error) { p.typeDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -718,14 +732,21 @@ func defaultTypeMetaLeftDenotation( } func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { - p.skipSpaceAndComments() + p.skipSpace() - startPos := p.current.StartPos + startToken := p.current isResource := false if p.current.Is(lexer.TokenAt) { p.next() isResource = true + + // Append @ token comments to type token, + // so that we don't need to store them in TypeAnnotation node. + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startToken.Comments.Leading...) + leadingComments = append(leadingComments, p.current.Comments.Leading...) + p.current.Comments.Leading = leadingComments } ty, err := parseType(p, lowestBindingPower) @@ -737,7 +758,7 @@ func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { p.memoryGauge, isResource, ty, - startPos, + startToken.StartPos, ), nil } @@ -762,7 +783,7 @@ func applyTypeLeftDenotation(p *parser, token lexer.Token, left ast.Type) (ast.T } func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, error) { - p.skipSpaceAndComments() + p.skipSpace() identifier, err := p.mustOne(lexer.TokenIdentifier) if err != nil { return nil, err @@ -773,7 +794,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, return nil, err } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken := p.current if p.current.Is(lexer.TokenParenOpen) { @@ -793,6 +814,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, var invokedExpression ast.Expression = ast.NewIdentifierExpression( p.memoryGauge, ty.Identifier, + ty.Comments, ) for _, nestedIdentifier := range ty.NestedIdentifiers { @@ -830,7 +852,7 @@ func parseCommaSeparatedTypeAnnotations( for !atEnd && p.checkProgress(&progress) { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -922,14 +944,14 @@ func defineIdentifierTypes() { func(p *parser, token lexer.Token) (ast.Type, error) { switch string(p.tokenSource(token)) { case KeywordAuth: - p.skipSpaceAndComments() + p.skipSpace() var authorization ast.Authorization if p.current.Is(lexer.TokenParenOpen) { p.next() - p.skipSpaceAndComments() + p.skipSpace() var err error authorization, err = parseAuthorization(p) @@ -942,7 +964,7 @@ func defineIdentifierTypes() { }) } - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenAmpersand) { p.next() @@ -965,7 +987,7 @@ func defineIdentifierTypes() { ), nil case KeywordFun: - p.skipSpaceAndComments() + p.skipSpace() return parseFunctionType(p, token.StartPos, ast.FunctionPurityUnspecified) case KeywordView: @@ -974,7 +996,7 @@ func defineIdentifierTypes() { cursor := p.tokens.Cursor() // Look ahead for the `fun` keyword, if it exists - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // Skip the `fun` keyword @@ -1006,7 +1028,7 @@ func parseAuthorization(p *parser) (auth ast.Authorization, err error) { return nil, err } auth = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) @@ -1035,7 +1057,7 @@ func parseAuthorization(p *parser) (auth ast.Authorization, err error) { // ( ':' type )? func parseFunctionType(p *parser, startPos ast.Position, purity ast.FunctionPurity) (ast.Type, error) { - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenParenOpen) { p.nextSemanticToken() @@ -1078,6 +1100,7 @@ func parseFunctionType(p *parser, startPos ast.Position, purity ast.FunctionPuri endPos, ), nil, + ast.EmptyComments, ) returnTypeAnnotation = ast.NewTypeAnnotation( p.memoryGauge, diff --git a/sema/before_extractor.go b/sema/before_extractor.go index b99842f5fc..041e7b9e05 100644 --- a/sema/before_extractor.go +++ b/sema/before_extractor.go @@ -76,7 +76,7 @@ func (e *BeforeExtractor) ExtractInvocation( ast.EmptyPosition, ) - newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier) + newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier, ast.EmptyComments) extractedExpressions = append(extractedExpressions, ast.ExtractedExpression{ diff --git a/sema/check_composite_declaration.go b/sema/check_composite_declaration.go index daba52288f..e7b3ed95ef 100644 --- a/sema/check_composite_declaration.go +++ b/sema/check_composite_declaration.go @@ -1034,7 +1034,7 @@ func (checker *Checker) declareContractValue( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: compositeType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: contracts are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindContract, @@ -1088,7 +1088,7 @@ func (checker *Checker) declareEnumConstructor( TypeAnnotation: memberCaseTypeAnnotation, DeclarationKind: common.DeclarationKindField, VariableKind: ast.VariableKindConstant, - DocString: enumCase.DocString, + DocString: enumCase.DeclarationDocString(), }, ) @@ -1097,7 +1097,7 @@ func (checker *Checker) declareEnumConstructor( checker.recordFieldDeclarationOrigin( enumCase.Identifier, compositeType, - enumCase.DocString, + enumCase.DeclarationDocString(), ) } } @@ -1109,7 +1109,7 @@ func (checker *Checker) declareEnumConstructor( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: enumLookupFunctionType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: enums are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindEnum, @@ -1848,7 +1848,7 @@ func (checker *Checker) defaultMembersAndOrigins( DeclarationKind: declarationKind, TypeAnnotation: fieldTypeAnnotation, VariableKind: field.VariableKind, - DocString: field.DocString, + DocString: field.DeclarationDocString(), }) if checker.PositionInfo != nil && origins != nil { @@ -1856,7 +1856,7 @@ func (checker *Checker) defaultMembersAndOrigins( checker.recordFieldDeclarationOrigin( field.Identifier, fieldTypeAnnotation.Type, - field.DocString, + field.DeclarationDocString(), ) } @@ -1931,7 +1931,7 @@ func (checker *Checker) defaultMembersAndOrigins( TypeAnnotation: fieldTypeAnnotation, VariableKind: ast.VariableKindConstant, ArgumentLabels: argumentLabels, - DocString: function.DocString, + DocString: function.DeclarationDocString(), HasImplementation: hasImplementation, HasConditions: hasConditions, }) diff --git a/sema/check_function.go b/sema/check_function.go index dfc4477684..9a5fade0d4 100644 --- a/sema/check_function.go +++ b/sema/check_function.go @@ -146,7 +146,7 @@ func (checker *Checker) declareFunctionDeclaration( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: functionType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: common.DeclarationKindFunction, pos: declaration.Identifier.Pos, diff --git a/sema/check_interface_declaration.go b/sema/check_interface_declaration.go index 2628d387bb..e8c40cbb81 100644 --- a/sema/check_interface_declaration.go +++ b/sema/check_interface_declaration.go @@ -286,7 +286,7 @@ func (checker *Checker) declareInterfaceType(declaration *ast.InterfaceDeclarati ty: interfaceType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) checker.report(err) @@ -485,7 +485,7 @@ func (checker *Checker) declareEntitlementType(declaration *ast.EntitlementDecla ty: entitlementType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) @@ -532,7 +532,7 @@ func (checker *Checker) declareEntitlementMappingType(declaration *ast.Entitleme ty: entitlementMapType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) diff --git a/sema/check_switch.go b/sema/check_switch.go index 5cfd1cedc6..8f0e39adcf 100644 --- a/sema/check_switch.go +++ b/sema/check_switch.go @@ -185,6 +185,7 @@ func (checker *Checker) checkSwitchCaseStatements(switchCase *ast.SwitchCase) { switchCase.Statements[0].StartPosition(), switchCase.EndPos, ), + ast.EmptyComments, ) checker.checkBlock(block) } diff --git a/sema/check_variable_declaration.go b/sema/check_variable_declaration.go index e7b1c1cf47..49310d675f 100644 --- a/sema/check_variable_declaration.go +++ b/sema/check_variable_declaration.go @@ -225,7 +225,7 @@ func (checker *Checker) declareVariableDeclaration(declaration *ast.VariableDecl variable, err := checker.valueActivations.declare(variableDeclaration{ identifier: identifier, ty: declarationType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: declaration.DeclarationKind(), pos: declaration.Identifier.Pos, diff --git a/sema/checker.go b/sema/checker.go index 924c52b63d..676dfa266c 100644 --- a/sema/checker.go +++ b/sema/checker.go @@ -1244,6 +1244,7 @@ func (checker *Checker) convertNominalType(t *ast.NominalType) Type { checker.memoryGauge, t.Identifier, resolvedIdentifiers, + ast.EmptyComments, ), }, ) @@ -1259,6 +1260,7 @@ func (checker *Checker) convertNominalType(t *ast.NominalType) Type { checker.memoryGauge, t.Identifier, resolvedIdentifiers, + ast.EmptyComments, ) checker.report( &NotDeclaredError{ diff --git a/sema/gen/main.go b/sema/gen/main.go index ec3cbebc66..0bce2d1980 100644 --- a/sema/gen/main.go +++ b/sema/gen/main.go @@ -70,9 +70,10 @@ const headerTemplate = `// Code generated from {{ . }}. DO NOT EDIT. var parsedHeaderTemplate = template.Must(template.New("header").Parse(headerTemplate)) var parserConfig = parser.Config{ - StaticModifierEnabled: true, - NativeModifierEnabled: true, - TypeParametersEnabled: true, + StaticModifierEnabled: true, + NativeModifierEnabled: true, + TypeParametersEnabled: true, + CommentTrackingEnabled: true, } func initialUpper(s string) string { diff --git a/sema/gen/main_test.go b/sema/gen/main_test.go index d1ce9c277f..bce60f4353 100644 --- a/sema/gen/main_test.go +++ b/sema/gen/main_test.go @@ -51,6 +51,13 @@ func TestFiles(t *testing.T) { inputPath := filepath.Join(dirPath, "test.cdc") + defer func() { + err := recover() + if err != nil { + t.Errorf("%s should not have panicked: %s", testName, err) + } + }() + gen(inputPath, outFile, "github.com/onflow/cadence/sema/gen/"+dirPath) goldenPath := filepath.Join(dirPath, "test.golden.go") diff --git a/sema/positioninfo.go b/sema/positioninfo.go index 84d5f25c5e..de0bf26f79 100644 --- a/sema/positioninfo.go +++ b/sema/positioninfo.go @@ -111,7 +111,7 @@ func (i *PositionInfo) recordFunctionDeclarationOrigin( DeclarationKind: common.DeclarationKindFunction, StartPos: &startPosition, EndPos: &endPosition, - DocString: function.DocString, + DocString: function.DeclarationDocString(), } i.Occurrences.Put( @@ -246,7 +246,7 @@ func (i *PositionInfo) recordVariableDeclarationRange( Identifier: identifier, DeclarationKind: declaration.DeclarationKind(), Type: declarationType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), }, ) } diff --git a/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go b/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go index ab854202b0..2d8afb09d6 100644 --- a/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go +++ b/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go @@ -309,7 +309,7 @@ func (validator *CadenceV042ToV1ContractUpdateValidator) expectedAuthorizationOf } func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRepresentableComposite(newDecl *ast.CompositeDeclaration) { - dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil) + dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil, ast.EmptyComments) compositeType := validator.getCompositeType(dummyNominalType) supportedEntitlements := compositeType.SupportedEntitlements() @@ -323,7 +323,7 @@ func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRep } func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRepresentableInterface(newDecl *ast.InterfaceDeclaration) { - dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil) + dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil, ast.EmptyComments) interfaceType := validator.getInterfaceType(dummyNominalType) supportedEntitlements := interfaceType.SupportedEntitlements() @@ -865,7 +865,7 @@ func semaConformanceToASTNominalType(newConformance sema.Conformance) *ast.Nomin } if containerType == nil { - return ast.NewNominalType(nil, identifier, nil) + return ast.NewNominalType(nil, identifier, nil, ast.EmptyComments) } return ast.NewNominalType( @@ -874,6 +874,7 @@ func semaConformanceToASTNominalType(newConformance sema.Conformance) *ast.Nomin Identifier: containerType.String(), }, []ast.Identifier{identifier}, + ast.EmptyComments, ) } diff --git a/stdlib/test.go b/stdlib/test.go index fc403760ce..536298799c 100644 --- a/stdlib/test.go +++ b/stdlib/test.go @@ -463,7 +463,7 @@ func TestCheckerContractValueHandler( return StandardLibraryValue{ Name: declaration.Identifier.Identifier, Type: constructorType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), Kind: declaration.DeclarationKind(), Position: &declaration.Identifier.Pos, ArgumentLabels: constructorArgumentLabels,