From 88ed3a122a0af2c39622e12961638d11d788bfb0 Mon Sep 17 00:00:00 2001 From: Lucas Terra Date: Mon, 27 Jan 2020 15:07:26 +0100 Subject: [PATCH] fix: block comment parsing --- src/main/scanner.ts | 39 +- .../parser/fixtures/empty-comments.thrift | 14 + src/tests/parser/parser.spec.ts | 21 + .../solutions/comments-mapping.solution.json | 2004 ++++++++--------- .../solutions/complex-comments.solution.json | 36 +- .../solutions/empty-comments.solution.json | 193 ++ .../scanner/fixtures/comment-block-empty.txt | 18 + .../scanner/fixtures/comment-block-line.txt | 1 + src/tests/scanner/scanner.spec.ts | 28 + .../comment-block-empty.solution.json | 226 ++ .../comment-block-line.solution.json | 34 + .../solutions/comment-block.solution.json | 2 +- 12 files changed, 1571 insertions(+), 1045 deletions(-) create mode 100644 src/tests/parser/fixtures/empty-comments.thrift create mode 100644 src/tests/parser/solutions/empty-comments.solution.json create mode 100644 src/tests/scanner/fixtures/comment-block-empty.txt create mode 100644 src/tests/scanner/fixtures/comment-block-line.txt create mode 100644 src/tests/scanner/solutions/comment-block-empty.solution.json create mode 100644 src/tests/scanner/solutions/comment-block-line.solution.json diff --git a/src/main/scanner.ts b/src/main/scanner.ts index 1cafc69..63a07a6 100644 --- a/src/main/scanner.ts +++ b/src/main/scanner.ts @@ -308,45 +308,36 @@ export function createScanner( function multilineComment(): void { let comment: string = '' let cursor: number = 0 + let isNewLine = true - while (true) { - if ( - current() === '\n' || - isAtEnd() || - (current() !== '/' && current() !== '*' && current() !== ' ') - ) { - break - } else { - advance() - } - } + // Skip first two characters: '/' and '*' + advance() + advance() while (true) { - if (current() === '\n') { - nextLine() + // A comment goes until we find a comment terminator (*/). + if ((current() === '*' && peek() === '/') || isAtEnd()) { + advance() + break } - if ( - comment.charAt(cursor - 1) === '\n' && - (peek() === ' ' || peek() === '*') - ) { + if (isNewLine && (current() === ' ' || current() === '*')) { /** * We ignore stars and spaces after a new line to normalize comment formatting. * We're only keeping the text of the comment without the extranious formatting. */ } else { + isNewLine = false comment += current() cursor += 1 } - advance() - - // A comment goes until we find a comment terminator (*/). - if ((peek() === '*' && peekNext() === '/') || isAtEnd()) { - advance() - advance() - break + if (current() === '\n') { + isNewLine = true + nextLine() } + + advance() } addToken(SyntaxType.CommentBlock, comment.trim()) diff --git a/src/tests/parser/fixtures/empty-comments.thrift b/src/tests/parser/fixtures/empty-comments.thrift new file mode 100644 index 0000000..d0c4999 --- /dev/null +++ b/src/tests/parser/fixtures/empty-comments.thrift @@ -0,0 +1,14 @@ +// +/* */ +/** */ +/* + * + */ +/** + * + */ +/* + */ +/** + */ +const string test = 'test' diff --git a/src/tests/parser/parser.spec.ts b/src/tests/parser/parser.spec.ts index 956d8fa..e0f390f 100644 --- a/src/tests/parser/parser.spec.ts +++ b/src/tests/parser/parser.spec.ts @@ -21,6 +21,14 @@ function loadSolution(name: string): object { ) } +function saveSolution(fileName: string, obj: any) { + fs.writeFileSync( + path.join(__dirname, `./solutions/${fileName}.solution.json`), + JSON.stringify(obj, null, 4), + 'utf-8', + ) +} + function objectify(thrift: ThriftDocument): object { return JSON.parse(JSON.stringify(thrift)) } @@ -414,6 +422,19 @@ describe('Parser', () => { assert.deepEqual(objectify(thrift), expected) }) + it('should correctly handle empty block comments', () => { + const content: string = loadSource('empty-comments') + const scanner: Scanner = createScanner(content) + const tokens: Array = scanner.scan() + + const parser: Parser = createParser(tokens) + const thrift: ThriftDocument = parser.parse() + + const expected: any = loadSolution('empty-comments') + + assert.deepEqual(objectify(thrift), expected) + }) + it('should correctly parse annotations', () => { const content: string = loadSource('annotations') const scanner: Scanner = createScanner(content) diff --git a/src/tests/parser/solutions/comments-mapping.solution.json b/src/tests/parser/solutions/comments-mapping.solution.json index 96039b4..6ae2d39 100644 --- a/src/tests/parser/solutions/comments-mapping.solution.json +++ b/src/tests/parser/solutions/comments-mapping.solution.json @@ -1,1070 +1,1070 @@ { - "type": "ThriftDocument", - "body": [ - { - "type": "TypedefDefinition", - "name": { - "type": "Identifier", - "value": "MyInteger", - "loc": { - "start": { - "line": 5, - "column": 13, - "index": 52 - }, - "end": { - "line": 5, - "column": 22, - "index": 61 - } - } - }, - "definitionType": { - "type": "I32Keyword", - "loc": { - "start": { - "line": 5, - "column": 9, - "index": 48 - }, - "end": { - "line": 5, - "column": 12, - "index": 51 - } - } - }, - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Multiline typedef", - " comment" - ], - "loc": { - "start": { - "line": 1, - "column": 1, - "index": 0 - }, - "end": { - "line": 4, - "column": 4, - "index": 39 - } - } - } - ], - "loc": { - "start": { - "line": 5, - "column": 1, - "index": 40 - }, - "end": { - "line": 5, - "column": 22, - "index": 61 - } - } - }, - { - "type": "ConstDefinition", - "name": { - "type": "Identifier", - "value": "INT32CONSTANT", - "loc": { - "start": { - "line": 10, - "column": 11, - "index": 99 - }, - "end": { - "line": 10, - "column": 24, - "index": 112 - } - } - }, - "fieldType": { - "type": "I32Keyword", - "loc": { - "start": { - "line": 10, - "column": 7, - "index": 95 - }, - "end": { - "line": 10, - "column": 10, - "index": 98 - } - } - }, - "initializer": { - "type": "IntConstant", - "value": { - "type": "IntegerLiteral", - "value": "9853", - "loc": { - "start": { - "line": 10, - "column": 27, - "index": 115 - }, - "end": { - "line": 10, - "column": 31, - "index": 119 - } - } - }, - "loc": { - "start": { - "line": 10, - "column": 27, - "index": 115 - }, - "end": { - "line": 10, - "column": 31, - "index": 119 - } - } - }, - "comments": [ + "type": "ThriftDocument", + "body": [ { - "type": "CommentBlock", - "value": [ - "Const1 comment" - ], - "loc": { - "start": { - "line": 7, - "column": 1, - "index": 63 + "type": "TypedefDefinition", + "name": { + "type": "Identifier", + "value": "MyInteger", + "loc": { + "start": { + "line": 5, + "column": 13, + "index": 52 + }, + "end": { + "line": 5, + "column": 22, + "index": 61 + } + } }, - "end": { - "line": 9, - "column": 4, - "index": 88 - } - } - } - ], - "loc": { - "start": { - "line": 10, - "column": 1, - "index": 89 - }, - "end": { - "line": 10, - "column": 31, - "index": 119 - } - } - }, - { - "type": "ConstDefinition", - "name": { - "type": "Identifier", - "value": "STRINGCONSTANT", - "loc": { - "start": { - "line": 13, - "column": 14, - "index": 152 - }, - "end": { - "line": 13, - "column": 28, - "index": 166 - } - } - }, - "fieldType": { - "type": "StringKeyword", - "loc": { - "start": { - "line": 13, - "column": 7, - "index": 145 - }, - "end": { - "line": 13, - "column": 13, - "index": 151 - } - } - }, - "initializer": { - "type": "StringLiteral", - "value": "hello world", - "loc": { - "start": { - "line": 13, - "column": 31, - "index": 169 - }, - "end": { - "line": 13, - "column": 44, - "index": 182 - } - } - }, - "comments": [ - { - "type": "CommentLine", - "value": "Const2 comment", - "loc": { - "start": { - "line": 12, - "column": 1, - "index": 121 + "definitionType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 5, + "column": 9, + "index": 48 + }, + "end": { + "line": 5, + "column": 12, + "index": 51 + } + } }, - "end": { - "line": 12, - "column": 18, - "index": 138 - } - } - } - ], - "loc": { - "start": { - "line": 13, - "column": 1, - "index": 139 - }, - "end": { - "line": 13, - "column": 44, - "index": 182 - } - } - }, - { - "type": "EnumDefinition", - "name": { - "type": "Identifier", - "value": "Operation", - "loc": { - "start": { - "line": 18, - "column": 6, - "index": 213 - }, - "end": { - "line": 18, - "column": 15, - "index": 222 - } - } - }, - "members": [ - { - "type": "EnumMember", - "name": { - "type": "Identifier", - "value": "ADD", + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Multiline typedef", + "comment" + ], + "loc": { + "start": { + "line": 1, + "column": 1, + "index": 0 + }, + "end": { + "line": 4, + "column": 4, + "index": 39 + } + } + } + ], "loc": { - "start": { - "line": 19, - "column": 3, - "index": 227 - }, - "end": { - "line": 19, - "column": 6, - "index": 230 - } - } - }, - "initializer": { - "type": "IntConstant", - "value": { - "type": "IntegerLiteral", - "value": "1", - "loc": { "start": { - "line": 19, - "column": 9, - "index": 233 + "line": 5, + "column": 1, + "index": 40 }, "end": { - "line": 19, - "column": 10, - "index": 234 + "line": 5, + "column": 22, + "index": 61 } - } - }, - "loc": { - "start": { - "line": 19, - "column": 9, - "index": 233 - }, - "end": { - "line": 19, - "column": 10, - "index": 234 - } - } - }, - "comments": [], - "loc": { - "start": { - "line": 19, - "column": 3, - "index": 227 - }, - "end": { - "line": 19, - "column": 10, - "index": 234 } - } }, { - "type": "EnumMember", - "name": { - "type": "Identifier", - "value": "MULTIPLY", - "loc": { - "start": { - "line": 21, - "column": 3, - "index": 262 - }, - "end": { - "line": 21, - "column": 11, - "index": 270 - } - } - }, - "initializer": null, - "comments": [ - { - "type": "CommentLine", - "value": "Enum field comment", - "loc": { - "start": { - "line": 20, - "column": 3, - "index": 238 - }, - "end": { - "line": 20, - "column": 24, - "index": 259 + "type": "ConstDefinition", + "name": { + "type": "Identifier", + "value": "INT32CONSTANT", + "loc": { + "start": { + "line": 10, + "column": 11, + "index": 99 + }, + "end": { + "line": 10, + "column": 24, + "index": 112 + } } - } - } - ], - "loc": { - "start": { - "line": 21, - "column": 3, - "index": 262 }, - "end": { - "line": 21, - "column": 11, - "index": 270 - } - } - } - ], - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Enum comment" - ], - "loc": { - "start": { - "line": 15, - "column": 1, - "index": 184 + "fieldType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 10, + "column": 7, + "index": 95 + }, + "end": { + "line": 10, + "column": 10, + "index": 98 + } + } }, - "end": { - "line": 17, - "column": 4, - "index": 207 - } - } - } - ], - "loc": { - "start": { - "line": 18, - "column": 1, - "index": 208 - }, - "end": { - "line": 22, - "column": 2, - "index": 272 - } - } - }, - { - "type": "StructDefinition", - "name": { - "type": "Identifier", - "value": "Work", - "loc": { - "start": { - "line": 27, - "column": 8, - "index": 307 - }, - "end": { - "line": 27, - "column": 12, - "index": 311 - } - } - }, - "fields": [ - { - "type": "FieldDefinition", - "name": { - "type": "Identifier", - "value": "num1", - "loc": { - "start": { - "line": 28, - "column": 10, - "index": 323 - }, - "end": { - "line": 28, - "column": 14, - "index": 327 - } - } - }, - "fieldID": { - "type": "FieldID", - "value": 1, - "loc": { - "start": { - "line": 28, - "column": 3, - "index": 316 - }, - "end": { - "line": 28, - "column": 5, - "index": 318 - } - } - }, - "fieldType": { - "type": "I32Keyword", - "loc": { - "start": { - "line": 28, - "column": 6, - "index": 319 - }, - "end": { - "line": 28, - "column": 9, - "index": 322 - } - } - }, - "requiredness": null, - "defaultValue": { - "type": "IntConstant", - "value": { - "type": "IntegerLiteral", - "value": "0", - "loc": { - "start": { - "line": 28, - "column": 17, - "index": 330 + "initializer": { + "type": "IntConstant", + "value": { + "type": "IntegerLiteral", + "value": "9853", + "loc": { + "start": { + "line": 10, + "column": 27, + "index": 115 + }, + "end": { + "line": 10, + "column": 31, + "index": 119 + } + } }, - "end": { - "line": 28, - "column": 18, - "index": 331 + "loc": { + "start": { + "line": 10, + "column": 27, + "index": 115 + }, + "end": { + "line": 10, + "column": 31, + "index": 119 + } } - } - }, - "loc": { - "start": { - "line": 28, - "column": 17, - "index": 330 - }, - "end": { - "line": 28, - "column": 18, - "index": 331 - } - } - }, - "comments": [], - "loc": { - "start": { - "line": 28, - "column": 3, - "index": 316 }, - "end": { - "line": 28, - "column": 19, - "index": 332 - } - } - }, - { - "type": "FieldDefinition", - "name": { - "type": "Identifier", - "value": "op", - "loc": { - "start": { - "line": 30, - "column": 16, - "index": 374 - }, - "end": { - "line": 30, - "column": 18, - "index": 376 - } - } - }, - "fieldID": { - "type": "FieldID", - "value": 2, - "loc": { - "start": { - "line": 30, - "column": 3, - "index": 361 - }, - "end": { - "line": 30, - "column": 5, - "index": 363 - } - } - }, - "fieldType": { - "type": "Identifier", - "value": "Operation", + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Const1 comment" + ], + "loc": { + "start": { + "line": 7, + "column": 1, + "index": 63 + }, + "end": { + "line": 9, + "column": 4, + "index": 88 + } + } + } + ], "loc": { - "start": { - "line": 30, - "column": 6, - "index": 364 - }, - "end": { - "line": 30, - "column": 15, - "index": 373 - } - } - }, - "requiredness": null, - "defaultValue": null, - "comments": [ - { - "type": "CommentLine", - "value": "Struct field comment", - "loc": { "start": { - "line": 29, - "column": 3, - "index": 335 + "line": 10, + "column": 1, + "index": 89 }, "end": { - "line": 29, - "column": 26, - "index": 358 + "line": 10, + "column": 31, + "index": 119 } - } - } - ], - "loc": { - "start": { - "line": 30, - "column": 3, - "index": 361 - }, - "end": { - "line": 30, - "column": 19, - "index": 377 - } - } - } - ], - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Struct comment" - ], - "loc": { - "start": { - "line": 24, - "column": 1, - "index": 274 - }, - "end": { - "line": 26, - "column": 4, - "index": 299 } - } - } - ], - "loc": { - "start": { - "line": 27, - "column": 1, - "index": 300 }, - "end": { - "line": 31, - "column": 2, - "index": 379 - } - } - }, - { - "type": "ServiceDefinition", - "name": { - "type": "Identifier", - "value": "Calculator", - "loc": { - "start": { - "line": 36, - "column": 9, - "index": 416 - }, - "end": { - "line": 36, - "column": 19, - "index": 426 - } - } - }, - "extends": { - "type": "Identifier", - "value": "shared.SharedService", - "loc": { - "start": { - "line": 36, - "column": 20, - "index": 427 - }, - "end": { - "line": 36, - "column": 48, - "index": 455 - } - } - }, - "functions": [ { - "type": "FunctionDefinition", - "name": { - "type": "Identifier", - "value": "add", - "loc": { - "start": { - "line": 42, - "column": 8, - "index": 503 - }, - "end": { - "line": 42, - "column": 11, - "index": 506 - } - } - }, - "returnType": { - "type": "I32Keyword", - "loc": { - "start": { - "line": 42, - "column": 4, - "index": 499 - }, - "end": { - "line": 42, - "column": 7, - "index": 502 - } - } - }, - "fields": [ - { - "type": "FieldDefinition", - "name": { + "type": "ConstDefinition", + "name": { "type": "Identifier", - "value": "num1", + "value": "STRINGCONSTANT", "loc": { - "start": { - "line": 42, - "column": 24, - "index": 519 - }, - "end": { - "line": 42, - "column": 28, - "index": 523 - } + "start": { + "line": 13, + "column": 14, + "index": 152 + }, + "end": { + "line": 13, + "column": 28, + "index": 166 + } } - }, - "fieldID": { - "type": "FieldID", - "value": 1, + }, + "fieldType": { + "type": "StringKeyword", "loc": { - "start": { - "line": 42, - "column": 12, - "index": 507 - }, - "end": { - "line": 42, - "column": 14, - "index": 509 - } + "start": { + "line": 13, + "column": 7, + "index": 145 + }, + "end": { + "line": 13, + "column": 13, + "index": 151 + } } - }, - "fieldType": { - "type": "Identifier", - "value": "MyInteger", + }, + "initializer": { + "type": "StringLiteral", + "value": "hello world", "loc": { - "start": { - "line": 42, - "column": 14, - "index": 509 - }, - "end": { - "line": 42, - "column": 23, - "index": 518 - } + "start": { + "line": 13, + "column": 31, + "index": 169 + }, + "end": { + "line": 13, + "column": 44, + "index": 182 + } + } + }, + "comments": [ + { + "type": "CommentLine", + "value": "Const2 comment", + "loc": { + "start": { + "line": 12, + "column": 1, + "index": 121 + }, + "end": { + "line": 12, + "column": 18, + "index": 138 + } + } } - }, - "requiredness": null, - "defaultValue": null, - "comments": [], - "loc": { + ], + "loc": { "start": { - "line": 42, - "column": 12, - "index": 507 + "line": 13, + "column": 1, + "index": 139 }, "end": { - "line": 42, - "column": 29, - "index": 524 + "line": 13, + "column": 44, + "index": 182 } - } - }, - { - "type": "FieldDefinition", - "name": { + } + }, + { + "type": "EnumDefinition", + "name": { "type": "Identifier", - "value": "num2", + "value": "Operation", "loc": { - "start": { - "line": 42, - "column": 36, - "index": 531 - }, - "end": { - "line": 42, - "column": 40, - "index": 535 - } + "start": { + "line": 18, + "column": 6, + "index": 213 + }, + "end": { + "line": 18, + "column": 15, + "index": 222 + } } - }, - "fieldID": { - "type": "FieldID", - "value": 2, - "loc": { - "start": { - "line": 42, - "column": 30, - "index": 525 - }, - "end": { - "line": 42, - "column": 32, - "index": 527 - } + }, + "members": [ + { + "type": "EnumMember", + "name": { + "type": "Identifier", + "value": "ADD", + "loc": { + "start": { + "line": 19, + "column": 3, + "index": 227 + }, + "end": { + "line": 19, + "column": 6, + "index": 230 + } + } + }, + "initializer": { + "type": "IntConstant", + "value": { + "type": "IntegerLiteral", + "value": "1", + "loc": { + "start": { + "line": 19, + "column": 9, + "index": 233 + }, + "end": { + "line": 19, + "column": 10, + "index": 234 + } + } + }, + "loc": { + "start": { + "line": 19, + "column": 9, + "index": 233 + }, + "end": { + "line": 19, + "column": 10, + "index": 234 + } + } + }, + "comments": [], + "loc": { + "start": { + "line": 19, + "column": 3, + "index": 227 + }, + "end": { + "line": 19, + "column": 10, + "index": 234 + } + } + }, + { + "type": "EnumMember", + "name": { + "type": "Identifier", + "value": "MULTIPLY", + "loc": { + "start": { + "line": 21, + "column": 3, + "index": 262 + }, + "end": { + "line": 21, + "column": 11, + "index": 270 + } + } + }, + "initializer": null, + "comments": [ + { + "type": "CommentLine", + "value": "Enum field comment", + "loc": { + "start": { + "line": 20, + "column": 3, + "index": 238 + }, + "end": { + "line": 20, + "column": 24, + "index": 259 + } + } + } + ], + "loc": { + "start": { + "line": 21, + "column": 3, + "index": 262 + }, + "end": { + "line": 21, + "column": 11, + "index": 270 + } + } } - }, - "fieldType": { - "type": "I32Keyword", - "loc": { - "start": { - "line": 42, - "column": 32, - "index": 527 - }, - "end": { - "line": 42, - "column": 35, - "index": 530 - } + ], + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Enum comment" + ], + "loc": { + "start": { + "line": 15, + "column": 1, + "index": 184 + }, + "end": { + "line": 17, + "column": 4, + "index": 207 + } + } } - }, - "requiredness": null, - "defaultValue": null, - "comments": [], - "loc": { + ], + "loc": { "start": { - "line": 42, - "column": 30, - "index": 525 + "line": 18, + "column": 1, + "index": 208 }, "end": { - "line": 42, - "column": 40, - "index": 535 + "line": 22, + "column": 2, + "index": 272 } - } } - ], - "throws": [], - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Function 1 comment" - ], - "loc": { + }, + { + "type": "StructDefinition", + "name": { + "type": "Identifier", + "value": "Work", + "loc": { + "start": { + "line": 27, + "column": 8, + "index": 307 + }, + "end": { + "line": 27, + "column": 12, + "index": 311 + } + } + }, + "fields": [ + { + "type": "FieldDefinition", + "name": { + "type": "Identifier", + "value": "num1", + "loc": { + "start": { + "line": 28, + "column": 10, + "index": 323 + }, + "end": { + "line": 28, + "column": 14, + "index": 327 + } + } + }, + "fieldID": { + "type": "FieldID", + "value": 1, + "loc": { + "start": { + "line": 28, + "column": 3, + "index": 316 + }, + "end": { + "line": 28, + "column": 5, + "index": 318 + } + } + }, + "fieldType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 28, + "column": 6, + "index": 319 + }, + "end": { + "line": 28, + "column": 9, + "index": 322 + } + } + }, + "requiredness": null, + "defaultValue": { + "type": "IntConstant", + "value": { + "type": "IntegerLiteral", + "value": "0", + "loc": { + "start": { + "line": 28, + "column": 17, + "index": 330 + }, + "end": { + "line": 28, + "column": 18, + "index": 331 + } + } + }, + "loc": { + "start": { + "line": 28, + "column": 17, + "index": 330 + }, + "end": { + "line": 28, + "column": 18, + "index": 331 + } + } + }, + "comments": [], + "loc": { + "start": { + "line": 28, + "column": 3, + "index": 316 + }, + "end": { + "line": 28, + "column": 19, + "index": 332 + } + } + }, + { + "type": "FieldDefinition", + "name": { + "type": "Identifier", + "value": "op", + "loc": { + "start": { + "line": 30, + "column": 16, + "index": 374 + }, + "end": { + "line": 30, + "column": 18, + "index": 376 + } + } + }, + "fieldID": { + "type": "FieldID", + "value": 2, + "loc": { + "start": { + "line": 30, + "column": 3, + "index": 361 + }, + "end": { + "line": 30, + "column": 5, + "index": 363 + } + } + }, + "fieldType": { + "type": "Identifier", + "value": "Operation", + "loc": { + "start": { + "line": 30, + "column": 6, + "index": 364 + }, + "end": { + "line": 30, + "column": 15, + "index": 373 + } + } + }, + "requiredness": null, + "defaultValue": null, + "comments": [ + { + "type": "CommentLine", + "value": "Struct field comment", + "loc": { + "start": { + "line": 29, + "column": 3, + "index": 335 + }, + "end": { + "line": 29, + "column": 26, + "index": 358 + } + } + } + ], + "loc": { + "start": { + "line": 30, + "column": 3, + "index": 361 + }, + "end": { + "line": 30, + "column": 19, + "index": 377 + } + } + } + ], + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Struct comment" + ], + "loc": { + "start": { + "line": 24, + "column": 1, + "index": 274 + }, + "end": { + "line": 26, + "column": 4, + "index": 299 + } + } + } + ], + "loc": { "start": { - "line": 38, - "column": 3, - "index": 461 + "line": 27, + "column": 1, + "index": 300 }, "end": { - "line": 40, - "column": 6, - "index": 494 + "line": 31, + "column": 2, + "index": 379 } - } - } - ], - "oneway": false, - "modifiers": [], - "loc": { - "start": { - "line": 42, - "column": 4, - "index": 499 - }, - "end": { - "line": 42, - "column": 42, - "index": 537 } - } }, { - "type": "FunctionDefinition", - "name": { - "type": "Identifier", - "value": "zip", - "loc": { - "start": { - "line": 47, - "column": 16, - "index": 591 - }, - "end": { - "line": 47, - "column": 19, - "index": 594 - } - } - }, - "returnType": { - "type": "VoidKeyword", - "loc": { - "start": { - "line": 47, - "column": 11, - "index": 586 - }, - "end": { - "line": 47, - "column": 15, - "index": 590 - } - } - }, - "fields": [ - { - "type": "FieldDefinition", - "name": { + "type": "ServiceDefinition", + "name": { "type": "Identifier", - "value": "vasia", - "loc": { - "start": { - "line": 47, - "column": 26, - "index": 601 - }, - "end": { - "line": 47, - "column": 31, - "index": 606 - } - } - }, - "fieldID": { - "type": "FieldID", - "value": 1, + "value": "Calculator", "loc": { - "start": { - "line": 47, - "column": 20, - "index": 595 - }, - "end": { - "line": 47, - "column": 22, - "index": 597 - } + "start": { + "line": 36, + "column": 9, + "index": 416 + }, + "end": { + "line": 36, + "column": 19, + "index": 426 + } } - }, - "fieldType": { - "type": "I32Keyword", + }, + "extends": { + "type": "Identifier", + "value": "shared.SharedService", "loc": { - "start": { - "line": 47, - "column": 22, - "index": 597 - }, - "end": { - "line": 47, - "column": 25, - "index": 600 - } + "start": { + "line": 36, + "column": 20, + "index": 427 + }, + "end": { + "line": 36, + "column": 48, + "index": 455 + } } - }, - "requiredness": null, - "defaultValue": null, - "comments": [], - "loc": { - "start": { - "line": 47, - "column": 20, - "index": 595 + }, + "functions": [ + { + "type": "FunctionDefinition", + "name": { + "type": "Identifier", + "value": "add", + "loc": { + "start": { + "line": 42, + "column": 8, + "index": 503 + }, + "end": { + "line": 42, + "column": 11, + "index": 506 + } + } + }, + "returnType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 42, + "column": 4, + "index": 499 + }, + "end": { + "line": 42, + "column": 7, + "index": 502 + } + } + }, + "fields": [ + { + "type": "FieldDefinition", + "name": { + "type": "Identifier", + "value": "num1", + "loc": { + "start": { + "line": 42, + "column": 24, + "index": 519 + }, + "end": { + "line": 42, + "column": 28, + "index": 523 + } + } + }, + "fieldID": { + "type": "FieldID", + "value": 1, + "loc": { + "start": { + "line": 42, + "column": 12, + "index": 507 + }, + "end": { + "line": 42, + "column": 14, + "index": 509 + } + } + }, + "fieldType": { + "type": "Identifier", + "value": "MyInteger", + "loc": { + "start": { + "line": 42, + "column": 14, + "index": 509 + }, + "end": { + "line": 42, + "column": 23, + "index": 518 + } + } + }, + "requiredness": null, + "defaultValue": null, + "comments": [], + "loc": { + "start": { + "line": 42, + "column": 12, + "index": 507 + }, + "end": { + "line": 42, + "column": 29, + "index": 524 + } + } + }, + { + "type": "FieldDefinition", + "name": { + "type": "Identifier", + "value": "num2", + "loc": { + "start": { + "line": 42, + "column": 36, + "index": 531 + }, + "end": { + "line": 42, + "column": 40, + "index": 535 + } + } + }, + "fieldID": { + "type": "FieldID", + "value": 2, + "loc": { + "start": { + "line": 42, + "column": 30, + "index": 525 + }, + "end": { + "line": 42, + "column": 32, + "index": 527 + } + } + }, + "fieldType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 42, + "column": 32, + "index": 527 + }, + "end": { + "line": 42, + "column": 35, + "index": 530 + } + } + }, + "requiredness": null, + "defaultValue": null, + "comments": [], + "loc": { + "start": { + "line": 42, + "column": 30, + "index": 525 + }, + "end": { + "line": 42, + "column": 40, + "index": 535 + } + } + } + ], + "throws": [], + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Function 1 comment" + ], + "loc": { + "start": { + "line": 38, + "column": 3, + "index": 461 + }, + "end": { + "line": 40, + "column": 6, + "index": 494 + } + } + } + ], + "oneway": false, + "modifiers": [], + "loc": { + "start": { + "line": 42, + "column": 4, + "index": 499 + }, + "end": { + "line": 42, + "column": 42, + "index": 537 + } + } }, - "end": { - "line": 47, - "column": 31, - "index": 606 + { + "type": "FunctionDefinition", + "name": { + "type": "Identifier", + "value": "zip", + "loc": { + "start": { + "line": 47, + "column": 16, + "index": 591 + }, + "end": { + "line": 47, + "column": 19, + "index": 594 + } + } + }, + "returnType": { + "type": "VoidKeyword", + "loc": { + "start": { + "line": 47, + "column": 11, + "index": 586 + }, + "end": { + "line": 47, + "column": 15, + "index": 590 + } + } + }, + "fields": [ + { + "type": "FieldDefinition", + "name": { + "type": "Identifier", + "value": "vasia", + "loc": { + "start": { + "line": 47, + "column": 26, + "index": 601 + }, + "end": { + "line": 47, + "column": 31, + "index": 606 + } + } + }, + "fieldID": { + "type": "FieldID", + "value": 1, + "loc": { + "start": { + "line": 47, + "column": 20, + "index": 595 + }, + "end": { + "line": 47, + "column": 22, + "index": 597 + } + } + }, + "fieldType": { + "type": "I32Keyword", + "loc": { + "start": { + "line": 47, + "column": 22, + "index": 597 + }, + "end": { + "line": 47, + "column": 25, + "index": 600 + } + } + }, + "requiredness": null, + "defaultValue": null, + "comments": [], + "loc": { + "start": { + "line": 47, + "column": 20, + "index": 595 + }, + "end": { + "line": 47, + "column": 31, + "index": 606 + } + } + } + ], + "throws": [], + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Function 2 comment" + ], + "loc": { + "start": { + "line": 44, + "column": 4, + "index": 542 + }, + "end": { + "line": 46, + "column": 6, + "index": 575 + } + } + } + ], + "oneway": true, + "modifiers": [ + { + "type": "OnewayKeyword", + "text": "oneway", + "loc": { + "start": { + "line": 47, + "column": 4, + "index": 579 + }, + "end": { + "line": 47, + "column": 10, + "index": 585 + } + } + } + ], + "loc": { + "start": { + "line": 47, + "column": 11, + "index": 586 + }, + "end": { + "line": 47, + "column": 32, + "index": 607 + } + } } - } - } - ], - "throws": [], - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Function 2 comment" - ], - "loc": { - "start": { - "line": 44, - "column": 4, - "index": 542 - }, - "end": { - "line": 46, - "column": 6, - "index": 575 + ], + "comments": [ + { + "type": "CommentBlock", + "value": [ + "Service comment" + ], + "loc": { + "start": { + "line": 33, + "column": 1, + "index": 381 + }, + "end": { + "line": 35, + "column": 4, + "index": 407 + } + } } - } - } - ], - "oneway": true, - "modifiers": [ - { - "type": "OnewayKeyword", - "text": "oneway", - "loc": { + ], + "loc": { "start": { - "line": 47, - "column": 4, - "index": 579 + "line": 36, + "column": 1, + "index": 408 }, "end": { - "line": 47, - "column": 10, - "index": 585 + "line": 49, + "column": 2, + "index": 610 } - } - } - ], - "loc": { - "start": { - "line": 47, - "column": 11, - "index": 586 - }, - "end": { - "line": 47, - "column": 32, - "index": 607 } - } - } - ], - "comments": [ - { - "type": "CommentBlock", - "value": [ - "Service comment" - ], - "loc": { - "start": { - "line": 33, - "column": 1, - "index": 381 - }, - "end": { - "line": 35, - "column": 4, - "index": 407 - } - } - } - ], - "loc": { - "start": { - "line": 36, - "column": 1, - "index": 408 - }, - "end": { - "line": 49, - "column": 2, - "index": 610 } - } - } - ] -} + ] +} \ No newline at end of file diff --git a/src/tests/parser/solutions/complex-comments.solution.json b/src/tests/parser/solutions/complex-comments.solution.json index 66b6ebb..1f9e47c 100644 --- a/src/tests/parser/solutions/complex-comments.solution.json +++ b/src/tests/parser/solutions/complex-comments.solution.json @@ -8,12 +8,12 @@ "value": "Test", "loc": { "start": { - "line": 9, + "line": 10, "column": 9, "index": 137 }, "end": { - "line": 9, + "line": 10, "column": 13, "index": 141 } @@ -28,12 +28,12 @@ "value": "ding", "loc": { "start": { - "line": 12, + "line": 13, "column": 9, "index": 192 }, "end": { - "line": 12, + "line": 13, "column": 13, "index": 196 } @@ -43,12 +43,12 @@ "type": "I32Keyword", "loc": { "start": { - "line": 12, + "line": 13, "column": 5, "index": 188 }, "end": { - "line": 12, + "line": 13, "column": 8, "index": 191 } @@ -62,12 +62,12 @@ "value": "bool foo();", "loc": { "start": { - "line": 10, + "line": 11, "column": 5, "index": 148 }, "end": { - "line": 10, + "line": 11, "column": 18, "index": 161 } @@ -78,12 +78,12 @@ "value": "string dang(),", "loc": { "start": { - "line": 11, + "line": 12, "column": 5, "index": 166 }, "end": { - "line": 11, + "line": 12, "column": 22, "index": 183 } @@ -94,12 +94,12 @@ "modifiers": [], "loc": { "start": { - "line": 12, + "line": 13, "column": 5, "index": 188 }, "end": { - "line": 12, + "line": 13, "column": 15, "index": 198 } @@ -127,7 +127,7 @@ "type": "CommentBlock", "value": [ "This is a multi-line", - " comment for testing" + "comment for testing" ], "loc": { "start": { @@ -155,8 +155,8 @@ "index": 83 }, "end": { - "line": 8, - "column": 23, + "line": 9, + "column": 3, "index": 128 } } @@ -164,16 +164,16 @@ ], "loc": { "start": { - "line": 9, + "line": 10, "column": 1, "index": 129 }, "end": { - "line": 13, + "line": 14, "column": 2, "index": 200 } } } ] -} +} \ No newline at end of file diff --git a/src/tests/parser/solutions/empty-comments.solution.json b/src/tests/parser/solutions/empty-comments.solution.json new file mode 100644 index 0000000..19571f6 --- /dev/null +++ b/src/tests/parser/solutions/empty-comments.solution.json @@ -0,0 +1,193 @@ +{ + "type": "ThriftDocument", + "body": [ + { + "type": "ConstDefinition", + "name": { + "type": "Identifier", + "value": "test", + "loc": { + "start": { + "line": 13, + "column": 14, + "index": 65 + }, + "end": { + "line": 13, + "column": 18, + "index": 69 + } + } + }, + "fieldType": { + "type": "StringKeyword", + "loc": { + "start": { + "line": 13, + "column": 7, + "index": 58 + }, + "end": { + "line": 13, + "column": 13, + "index": 64 + } + } + }, + "initializer": { + "type": "StringLiteral", + "value": "test", + "loc": { + "start": { + "line": 13, + "column": 21, + "index": 72 + }, + "end": { + "line": 13, + "column": 27, + "index": 78 + } + } + }, + "comments": [ + { + "type": "CommentLine", + "value": "", + "loc": { + "start": { + "line": 1, + "column": 1, + "index": 0 + }, + "end": { + "line": 1, + "column": 4, + "index": 3 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 1, + "column": 4, + "index": 3 + }, + "end": { + "line": 1, + "column": 9, + "index": 8 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 2, + "column": 1, + "index": 9 + }, + "end": { + "line": 2, + "column": 7, + "index": 15 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 3, + "column": 1, + "index": 16 + }, + "end": { + "line": 5, + "column": 4, + "index": 25 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 6, + "column": 1, + "index": 26 + }, + "end": { + "line": 8, + "column": 4, + "index": 36 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 9, + "column": 1, + "index": 37 + }, + "end": { + "line": 10, + "column": 4, + "index": 43 + } + } + }, + { + "type": "CommentBlock", + "value": [ + "" + ], + "loc": { + "start": { + "line": 11, + "column": 1, + "index": 44 + }, + "end": { + "line": 12, + "column": 4, + "index": 51 + } + } + } + ], + "loc": { + "start": { + "line": 13, + "column": 1, + "index": 52 + }, + "end": { + "line": 13, + "column": 27, + "index": 78 + } + } + } + ] +} \ No newline at end of file diff --git a/src/tests/scanner/fixtures/comment-block-empty.txt b/src/tests/scanner/fixtures/comment-block-empty.txt new file mode 100644 index 0000000..47ec3aa --- /dev/null +++ b/src/tests/scanner/fixtures/comment-block-empty.txt @@ -0,0 +1,18 @@ +// This service does nothing +/* + * This is a multi-line + * comment for testing + */ +/* +Another muliti-line +comment for testing +*/ +/* +*/ +/**/ +/* */ +/** */ +/** + * + */ +const string test = 'test' diff --git a/src/tests/scanner/fixtures/comment-block-line.txt b/src/tests/scanner/fixtures/comment-block-line.txt new file mode 100644 index 0000000..ae516b6 --- /dev/null +++ b/src/tests/scanner/fixtures/comment-block-line.txt @@ -0,0 +1 @@ +/* This is a nice comment */ diff --git a/src/tests/scanner/scanner.spec.ts b/src/tests/scanner/scanner.spec.ts index 02e395c..9be5820 100644 --- a/src/tests/scanner/scanner.spec.ts +++ b/src/tests/scanner/scanner.spec.ts @@ -20,6 +20,14 @@ function loadSolution(name: string): object { ) } +function saveSolution(fileName: string, obj: any) { + fs.writeFileSync( + path.join(__dirname, `./solutions/${fileName}.solution.json`), + JSON.stringify(obj, null, 4), + 'utf-8', + ) +} + describe('Scanner', () => { it('should correctly recognize floats', () => { const content = loadSource('floats') @@ -141,6 +149,26 @@ describe('Scanner', () => { assert.deepEqual(tokens, expected) }) + it(`should correctly handle single-line comments with '/* ... */'`, () => { + const content: string = loadSource('comment-block-line') + const scanner: Scanner = createScanner(content) + const tokens: Array = scanner.scan() + + const expected: any = loadSolution('comment-block-line') + + assert.deepEqual(tokens, expected) + }) + + it(`should correctly handle complex empty block comments`, () => { + const content: string = loadSource('comment-block-empty') + const scanner: Scanner = createScanner(content) + const tokens: Array = scanner.scan() + + const expected: any = loadSolution('comment-block-empty') + + assert.deepEqual(tokens, expected) + }) + it('should correctly return tokens for a simple const', () => { const content = loadSource('const-string') const scanner: Scanner = createScanner(content) diff --git a/src/tests/scanner/solutions/comment-block-empty.solution.json b/src/tests/scanner/solutions/comment-block-empty.solution.json new file mode 100644 index 0000000..43961bc --- /dev/null +++ b/src/tests/scanner/solutions/comment-block-empty.solution.json @@ -0,0 +1,226 @@ +[ + { + "type": "CommentLine", + "text": "This service does nothing", + "loc": { + "start": { + "line": 1, + "column": 1, + "index": 0 + }, + "end": { + "line": 1, + "column": 29, + "index": 28 + } + } + }, + { + "type": "CommentBlock", + "text": "This is a multi-line\ncomment for testing", + "loc": { + "start": { + "line": 2, + "column": 1, + "index": 29 + }, + "end": { + "line": 5, + "column": 4, + "index": 82 + } + } + }, + { + "type": "CommentBlock", + "text": "Another muliti-line\ncomment for testing", + "loc": { + "start": { + "line": 6, + "column": 1, + "index": 83 + }, + "end": { + "line": 9, + "column": 3, + "index": 128 + } + } + }, + { + "type": "CommentBlock", + "text": "", + "loc": { + "start": { + "line": 10, + "column": 1, + "index": 129 + }, + "end": { + "line": 11, + "column": 3, + "index": 134 + } + } + }, + { + "type": "CommentBlock", + "text": "", + "loc": { + "start": { + "line": 12, + "column": 1, + "index": 135 + }, + "end": { + "line": 12, + "column": 5, + "index": 139 + } + } + }, + { + "type": "CommentBlock", + "text": "", + "loc": { + "start": { + "line": 13, + "column": 1, + "index": 140 + }, + "end": { + "line": 13, + "column": 6, + "index": 145 + } + } + }, + { + "type": "CommentBlock", + "text": "", + "loc": { + "start": { + "line": 14, + "column": 1, + "index": 146 + }, + "end": { + "line": 14, + "column": 7, + "index": 152 + } + } + }, + { + "type": "CommentBlock", + "text": "", + "loc": { + "start": { + "line": 15, + "column": 1, + "index": 153 + }, + "end": { + "line": 17, + "column": 4, + "index": 163 + } + } + }, + { + "type": "ConstKeyword", + "text": "const", + "loc": { + "start": { + "line": 18, + "column": 1, + "index": 164 + }, + "end": { + "line": 18, + "column": 6, + "index": 169 + } + } + }, + { + "type": "StringKeyword", + "text": "string", + "loc": { + "start": { + "line": 18, + "column": 7, + "index": 170 + }, + "end": { + "line": 18, + "column": 13, + "index": 176 + } + } + }, + { + "type": "Identifier", + "text": "test", + "loc": { + "start": { + "line": 18, + "column": 14, + "index": 177 + }, + "end": { + "line": 18, + "column": 18, + "index": 181 + } + } + }, + { + "type": "EqualToken", + "text": "", + "loc": { + "start": { + "line": 18, + "column": 19, + "index": 182 + }, + "end": { + "line": 18, + "column": 20, + "index": 183 + } + } + }, + { + "type": "StringLiteral", + "text": "test", + "loc": { + "start": { + "line": 18, + "column": 21, + "index": 184 + }, + "end": { + "line": 18, + "column": 27, + "index": 190 + } + } + }, + { + "type": "EOF", + "text": "", + "loc": { + "start": { + "line": 18, + "column": 27, + "index": 191 + }, + "end": { + "line": 19, + "column": 1, + "index": 191 + } + } + } +] \ No newline at end of file diff --git a/src/tests/scanner/solutions/comment-block-line.solution.json b/src/tests/scanner/solutions/comment-block-line.solution.json new file mode 100644 index 0000000..7882fd1 --- /dev/null +++ b/src/tests/scanner/solutions/comment-block-line.solution.json @@ -0,0 +1,34 @@ +[ + { + "type": "CommentBlock", + "text": "This is a nice comment", + "loc": { + "start": { + "line": 1, + "column": 1, + "index": 0 + }, + "end": { + "line": 1, + "column": 29, + "index": 28 + } + } + }, + { + "type": "EOF", + "text": "", + "loc": { + "start": { + "line": 1, + "column": 29, + "index": 29 + }, + "end": { + "line": 2, + "column": 1, + "index": 29 + } + } + } +] diff --git a/src/tests/scanner/solutions/comment-block.solution.json b/src/tests/scanner/solutions/comment-block.solution.json index 80c44ea..874473a 100644 --- a/src/tests/scanner/solutions/comment-block.solution.json +++ b/src/tests/scanner/solutions/comment-block.solution.json @@ -1,7 +1,7 @@ [ { "type": "CommentBlock", - "text": "This is a struct\n it does things", + "text": "This is a struct\nit does things", "loc": { "start": { "line": 1,