diff --git a/src/main.nt b/src/main.nt index f43acd30..9516953f 100644 --- a/src/main.nt +++ b/src/main.nt @@ -910,7 +910,7 @@ void buildBinary(Options options, string mainFile, string execPath, Package[] pa neat_runtime_system("mkdir -p .obj"); auto pool = new ThreadPool(options.threads); - auto workPool = new WorkPool(pool, options.threads); + auto workPool = new WorkPool(pool); auto builtins = builtinSymbols(options); auto defaultImports = [builtins]; auto modParseConfig = new ModuleParserConfig(packages, defaultImports); diff --git a/src/neat/array.nt b/src/neat/array.nt index df19cb46..cc1715ef 100644 --- a/src/neat/array.nt +++ b/src/neat/array.nt @@ -697,20 +697,20 @@ Expression releaseArray(Context context, Expression value) { (nullable ASTSymbol | fail Error) parseArrayLiteral(Parser parser, LexicalContext lexicalContext) { parser.begin; - if (parser.accept("[")?) { } + if (parser.acceptToken(TokenType.lsquarebracket)) { } else { parser.revert; return null; } mut ASTArrayLiteralElement[] values; while (true) { - if (parser.accept("]")?) break; + if (parser.acceptToken(TokenType.rsquarebracket)) break; auto from = parser.from?; ASTSymbol value = lexicalContext.compiler.parseExpression(parser, lexicalContext)? .notNull; values ~= ASTArrayLiteralElement(value, parser.to(from)); - if (parser.accept("]")?) break; - parser.expect(",")?; + if (parser.acceptToken(TokenType.rsquarebracket)) break; + parser.expectToken(TokenType.comma)?; } parser.commit; return new ASTArrayLiteral(null, values); diff --git a/src/neat/base.nt b/src/neat/base.nt index 7b90c7f1..e9044088 100644 --- a/src/neat/base.nt +++ b/src/neat/base.nt @@ -4,6 +4,47 @@ import backend.base; import helpers; import neat.hash; +enum TokenType { + none, + end, + whitespace, + identifier, + number, + comment, + dot, + comma, + singleQuote, + charLiteral, + doubleQuote, // ["]foo $(bar) baz["] + stringLiteral, // "[foo ]$(bar)[ baz]" + formatQuoteStart, // "foo [$](bar)" + backtick, + colon, + semicolon, + lparen, + rparen, + smaller, + greater, + equal, + exclamationmark, + questionmark, + dollar, + lsquarebracket, + rsquarebracket, + lcurlybracket, + rcurlybracket, + plus, + minus, + asterisk, + slash, + backslash, + circumflex, + percent, + tilde, + ampersand, + bar, +} + // something that can be referenced by a name abstract class Symbol { @@ -656,7 +697,7 @@ abstract class CompilerBase string[] cFlags; // parser - abstract Parser createParser(string filename, string fulltext); + abstract (Parser | fail Error) createParser(string filename, string fulltext); abstract (nullable ASTSymbol | fail Error) parseExpression( Parser parser, LexicalContext lexicalContext); @@ -674,7 +715,13 @@ abstract class CompilerBase Parser parser, LexicalContext lexicalContext); abstract (ASTSymbol | fail Error) parseStringLiteral( - Parser parser, LexicalContext lexicalContext, string endMarker, LocRange from); + Parser parser, LexicalContext lexicalContext); + + (ASTSymbol | fail Error) parseStringLiteral( + Parser parser, LexicalContext lexicalContext, string separator, LocRange from) + { + return parseStringLiteral(parser, lexicalContext); + } abstract (nullable ASTSymbolDeclaration | fail Error) parseTemplateStub( Parser parser, LexicalContext lexicalContext, string name, string comment, @@ -1234,14 +1281,20 @@ abstract class Parser abstract LocRange loc(); abstract (LocRange | fail Error) from(); abstract LocRange to(LocRange from); - abstract (void | fail Error) strip(); - abstract (string | fail Error) parseLastComment(); + abstract void strip(); + abstract string parseLastComment(); abstract string text(); + abstract bool acceptToken(TokenType type); + abstract bool acceptToken2(TokenType first, TokenType second); + abstract bool acceptToken3(TokenType first, TokenType second, TokenType third); + abstract bool acceptToken2Not(TokenType match, TokenType nomatch); + abstract bool acceptToken2Not2(TokenType match, TokenType nomatch1, TokenType nomatch2); + abstract (string | :none) acceptTokenStr(TokenType type); + abstract (bool | fail Error) acceptIdentifier(string identifier); + abstract (void | fail Error) expectToken(TokenType token); + abstract bool peekToken(TokenType type); abstract (bool | fail Error) eof(); abstract bool hard_eof(); - abstract void dropOneCharNonNewline(int length); - abstract void drop(int length); - abstract string peekUniChar(); abstract (void | fail Error) verifyTransactions(string msg, (void | fail Error) delegate() dg); abstract (string | fail Error) parseIdentifier(); @@ -1258,19 +1311,6 @@ abstract class Parser return this.fail("'" ~ match ~ "' expected"); } - (bool | fail Error) acceptIdentifier(string identifier) - with (transaction) - { - string nextIdent = parseIdentifier?; - - if (nextIdent != identifier) - { - return false; - } - commit; - return true; - } - ParserTransaction transaction() { return ParserTransaction(this, committed=false); } @@ -1312,6 +1352,7 @@ struct ParserTransaction if (!committed) parser.revert; } void commit() { + assert(!committed); parser.commit; committed = true; } diff --git a/src/neat/class_.nt b/src/neat/class_.nt index 29542eb6..d5ee995a 100644 --- a/src/neat/class_.nt +++ b/src/neat/class_.nt @@ -1547,7 +1547,7 @@ class ASTIntfDecl : ASTSymbolDeclaration mut bool classAbstract = false; mut bool classFinal = false; parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; while (true) { if (parser.acceptIdentifier("abstract")?) { @@ -1572,19 +1572,19 @@ class ASTIntfDecl : ASTSymbolDeclaration (nullable ASTSymbolDeclaration | fail Error) parseRest() { mut ASTSymbol[] supers; - if (parser.accept(":")?) { + if (parser.acceptToken(TokenType.colon)) { while (true) { auto super_ = lexicalContext.compiler.parseType(parser, lexicalContext)?; parser.assert_(!!super_, "expected super class identifier")?; supers ~= super_.notNull; - if (parser.accept(",")?) continue; + if (parser.acceptToken(TokenType.comma)) continue; break; } } mut ASTClassEntry[] members; - parser.expect("{")?; - while (!parser.accept("}")?) { + parser.expectToken(TokenType.lcurlybracket)?; + while (!parser.acceptToken(TokenType.rcurlybracket)) { (void | fail Error) do_() { members ~= parseClassEntry(parser, lexicalContext, name)?; } @@ -1601,21 +1601,21 @@ class ASTIntfDecl : ASTSymbolDeclaration (ASTClassEntry[] | fail Error) parseClassEntry(Parser parser, LexicalContext lexicalContext, string className) { - if (parser.accept("version")?) { + if (parser.acceptIdentifier("version")?) { mut ASTClassEntry[] then, else_; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; string versionStr = parser.parseIdentifier?; - parser.expect(")")?; - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + parser.expectToken(TokenType.rparen)?; + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { then ~= parser.parseClassEntry(lexicalContext, className)?; } } else { then ~= parser.parseClassEntry(lexicalContext, className)?; } - if (parser.accept("else")?) { - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + if (parser.acceptIdentifier("else")?) { + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { else_ ~= parser.parseClassEntry(lexicalContext, className)?; } } else { @@ -1627,7 +1627,7 @@ class ASTIntfDecl : ASTSymbolDeclaration return [result]; } - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; mut nullable ASTSymbol retType; mut string memberName; @@ -1666,7 +1666,7 @@ class ASTIntfDecl : ASTSymbolDeclaration return [entry]; } - if (parser.accept("this")?) + if (parser.acceptIdentifier("this")?) { parser.assert_(!override_, "cannot override constructor")?; retType = new ASTIdentifier("void", false, __RANGE__); @@ -1681,12 +1681,12 @@ class ASTIntfDecl : ASTSymbolDeclaration parser.assert_(memberName.length > 0, "expected member name")?; } auto locRange = parser.to(from); - if (parser.accept("(")?) // method + if (parser.acceptToken(TokenType.lparen)) // method { auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=true)?; assert(!params.variadic); mut nullable ASTStatement stmt = null; - if (abstract_ && parser.accept(";")?) { + if (abstract_ && parser.acceptToken(TokenType.semicolon)) { ASTSymbol astName = new ASTStringLiteral(className ~ "." ~ memberName, __RANGE__); stmt = lexicalContext.compiler.$stmt { import package(compiler).neat.runtime.stdlib : exit; @@ -1711,9 +1711,9 @@ class ASTIntfDecl : ASTSymbolDeclaration members ~= ASTClassDeclMember(protection, memberName, retType.notNull, locRange); } addMember(locRange)?; - while (!parser.accept(";")?) { - parser.expect(",")?; - if (parser.accept(";")?) break; + while (!parser.acceptToken(TokenType.semicolon)) { + parser.expectToken(TokenType.comma)?; + if (parser.acceptToken(TokenType.semicolon)) break; auto memberFrom = parser.from?; string ident = parser.parseIdentifier?; auto memberLocRange = parser.to(from); @@ -1728,7 +1728,7 @@ class ASTIntfDecl : ASTSymbolDeclaration (nullable ASTSymbolDeclaration | fail Error) parseIntfDecl(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; if (!parser.acceptIdentifier("interface")?) { @@ -1742,18 +1742,18 @@ class ASTIntfDecl : ASTSymbolDeclaration (nullable ASTSymbolDeclaration | fail Error) parseRest() { mut ASTSymbol[] superIntfs; - if (parser.accept(":")?) { + if (parser.acceptToken(TokenType.colon)) { while (true) { auto superIntf = lexicalContext.compiler.parseType(parser, lexicalContext)?; parser.assert_(!!superIntf, "expected super interface")?; superIntfs ~= superIntf.notNull; - if (parser.accept(",")?) continue; + if (parser.acceptToken(TokenType.comma)) continue; break; } } mut (ASTIntfDeclMethod | ASTDeclaration)[] members; - parser.expect("{")?; - while (!parser.accept("}")?) + parser.expectToken(TokenType.lcurlybracket)?; + while (!parser.acceptToken(TokenType.rcurlybracket)) { auto from = parser.from?; @@ -1767,11 +1767,11 @@ class ASTIntfDecl : ASTSymbolDeclaration string memberName = parser.parseIdentifier?; parser.to(from).assert(memberName.length > 0, "expected member name")?; auto locRange = parser.to(from); - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=true)?; assert(!params.variadic); auto params = params.params; - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; // TODO error on specific parameter locRange.assert( [all a.case(ASTParameter: true, ASTThisAssignment: false) for a in params], @@ -1991,14 +1991,14 @@ bool hasAsParent(Class class_, Class parent) { parser.begin; auto from = parser.from?; - if (!(parser.accept(".")? && parser.accept("instanceOf")?)) + if (!(parser.acceptToken(TokenType.dot) && parser.acceptIdentifier("instanceOf")?)) { parser.revert; return null; } - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; nullable ASTSymbol type = lexicalContext.compiler.parseType(parser, lexicalContext)?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; parser.commit; return new ASTInstanceOf(left, type.notNull, parser.to(from)); } diff --git a/src/neat/compiler.nt b/src/neat/compiler.nt index aeb1c0e6..51d99171 100644 --- a/src/neat/compiler.nt +++ b/src/neat/compiler.nt @@ -13,6 +13,7 @@ import neat.either; import neat.expr; import neat.formatstring; import neat.function_; +import neat.lexer; import neat.parser; import neat.pragmas; import neat.statements; @@ -51,8 +52,8 @@ class CompilerImpl : CompilerBase } // parser - override Parser createParser(string filename, string fulltext) { - return new ParserImpl(filename, fulltext); + override (Parser | fail Error) createParser(string filename, string fulltext) { + return new ParserImpl(filename, fulltext, tokenize(fulltext)?); } override (nullable ASTSymbol | fail Error) parseExpression(Parser parser, LexicalContext lexicalContext) @@ -81,9 +82,9 @@ class CompilerImpl : CompilerBase } override (ASTSymbol | fail Error) parseStringLiteral( - Parser parser, LexicalContext lexicalContext, string endMarker, LocRange from) + Parser parser, LexicalContext lexicalContext) { - return .parseStringLiteral(parser.instanceOf(Parser).notNull, lexicalContext, endMarker, from); + return .parseStringLiteral(parser, lexicalContext); } override (nullable ASTSymbolDeclaration | fail Error) parseTemplateStub( @@ -98,7 +99,7 @@ class CompilerImpl : CompilerBase if (parser.accept("=>")?) { auto expr = parseExpression(parser, lexicalContext)?; if (!expr) return parser.fail("function expression expected"); - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return astReturn(expr.notNull, failProp=false, expr.locRange); } return .parseStatement(parser.instanceOf(Parser).notNull, lexicalContext); diff --git a/src/neat/decl.nt b/src/neat/decl.nt index d58af0b9..1de8211a 100644 --- a/src/neat/decl.nt +++ b/src/neat/decl.nt @@ -26,8 +26,8 @@ class ASTAliasDecl : ASTSymbolDeclaration (nullable ASTDeclaration | fail Error) parseAliasDecl(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; - if (!parser.accept("alias")?) { + auto comment = parser.parseLastComment; + if (!parser.acceptIdentifier("alias")?) { parser.revert; return null; } @@ -37,10 +37,10 @@ class ASTAliasDecl : ASTSymbolDeclaration parser.commit; auto locRange = parser.to(from); (nullable ASTSymbolDeclaration | fail Error) parseRest() { - parser.expect("=")?; + parser.expectToken(TokenType.equal)?; mut nullable ASTSymbol target = lexicalContext.compiler.parseType(parser, lexicalContext)?; if (!target) target = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return new ASTAliasDecl(name, target.notNull, locRange); } if (auto ret = lexicalContext.compiler.parseTemplateStub(parser, lexicalContext, name, comment, &parseRest)?) diff --git a/src/neat/either.nt b/src/neat/either.nt index aa999b4f..5ba0256e 100644 --- a/src/neat/either.nt +++ b/src/neat/either.nt @@ -776,7 +776,9 @@ private (Expression | fail Error) compileExprCaseExpr(Context context, Expressio (nullable ASTSymbol | fail Error) parseCaseExpr(Parser parser, LexicalContext lexicalContext, ASTSymbol current) { parser.begin; - if (!(parser.accept(".")? && parser.acceptIdentifier("case")? && parser.accept("(")?)) + if (!(parser.acceptToken(TokenType.dot) + && parser.acceptIdentifier("case")? + && parser.acceptToken(TokenType.lparen))) { parser.revert; return null; @@ -785,10 +787,10 @@ private (Expression | fail Error) compileExprCaseExpr(Context context, Expressio parser.commit; mut ASTEitherCaseExprCase[] cases; - while (!parser.accept(")")?) + while (!parser.acceptToken(TokenType.rparen)) { - if (cases.length) parser.expect(",")?; - if (parser.accept(")")?) break; // .case(a, b,) + if (cases.length) parser.expectToken(TokenType.comma)?; + if (parser.acceptToken(TokenType.rparen)) break; // .case(a, b,) auto caseFrom = parser.from?; mut bool parsedType; (ASTSymbol | fail Error) type() { @@ -803,7 +805,7 @@ private (Expression | fail Error) compileExprCaseExpr(Context context, Expressio auto type = type?; mut string identifier; if (parsedType) identifier = parser.parseIdentifier?; - parser.expect(":")?; + parser.expectToken(TokenType.colon)?; ASTSymbol expr = lexicalContext.compiler.parseExpression(parser, lexicalContext)? .notNull; cases ~= ASTEitherCaseExprCase(parser.to(caseFrom), type, identifier, false, expr); @@ -931,7 +933,7 @@ class ASTEitherCaseStmt : ASTStatement } auto member = expr.instanceOf(ASTMemberBase); auto from = parser.from?; - if (!member || member.member != "case" || !parser.accept("{")?) { + if (!member || member.member != "case" || !parser.acceptToken(TokenType.lcurlybracket)) { parser.revert; return null; } @@ -940,10 +942,10 @@ class ASTEitherCaseStmt : ASTStatement mut ASTEitherCaseStmtCase[] cases; mut nullable ASTStatement default_ = null; - while (!parser.accept("}")?) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { auto caseFrom = parser.from?; if (parser.accept("default")?) { - parser.expect(":")?; + parser.expectToken(TokenType.colon)?; if (default_) { return parser.fail("cannot have more than one default block"); } @@ -956,7 +958,7 @@ class ASTEitherCaseStmt : ASTStatement parser.assert_(!!type, "case type expected")?; // identifier can be empty string identifier = parser.parseIdentifier?; - parser.expect(":")?; + parser.expectToken(TokenType.colon)?; auto caseLocRange = parser.to(caseFrom); auto stmt = parseEitherBlock(parser, lexicalContext)?; cases ~= ASTEitherCaseStmtCase(caseLocRange, type.notNull, identifier, stmt); @@ -970,7 +972,7 @@ class ASTEitherCaseStmt : ASTStatement mut ASTStatement[] stmts; while (true) { parser.begin; - bool end = parser.accept("}")?; + bool end = parser.acceptToken(TokenType.rcurlybracket); parser.revert; if (end) break; // are we seeing a new block? diff --git a/src/neat/enums.nt b/src/neat/enums.nt index 3f245f70..19cfa7a4 100644 --- a/src/neat/enums.nt +++ b/src/neat/enums.nt @@ -98,13 +98,13 @@ class ASTEnumDecl : ASTSymbolDeclaration auto locRange = parser.to(from); string name = parser.parseIdentifier?; mut EnumEntry[] entries; - parser.expect("{")?; - while (!parser.accept("}")?) + parser.expectToken(TokenType.lcurlybracket)?; + while (!parser.acceptToken(TokenType.rcurlybracket)) { if (entries.length) - parser.expect(",")?; + parser.expectToken(TokenType.comma)?; // is there a better way to write 'there may be a trailing comma'? - if (parser.accept("}")?) + if (parser.acceptToken(TokenType.rcurlybracket)) break; string entryName = parser.parseIdentifier?; entries ~= EnumEntry(entryName, cast(int) entries.length); @@ -124,7 +124,7 @@ class ASTEnumDecl : ASTSymbolDeclaration ASTStatement body_() { mut ASTStatement ifTree = compiler.$stmt { - import neat.runtime : assert, print; + import package(compiler).neat.runtime : assert, print; print("error: enum value outside domain"); assert(false); }; diff --git a/src/neat/function_.nt b/src/neat/function_.nt index 04b01a8a..f0b6c7fe 100644 --- a/src/neat/function_.nt +++ b/src/neat/function_.nt @@ -680,7 +680,7 @@ class ASTFunction : ASTSymbolDeclaration (nullable ASTDeclaration | fail Error) parseFunction(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; mut uninitialized (ASTSymbol | :auto_) ret; if (parser.acceptIdentifier("auto")?) { @@ -701,7 +701,7 @@ class ASTFunction : ASTSymbolDeclaration return null; } (nullable ASTSymbolDeclaration | fail Error) parseRest() { - if (!parser.accept("(")?) + if (!parser.acceptToken(TokenType.lparen)) return null; auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=false)?; assert(!params.variadic); @@ -735,7 +735,7 @@ class ASTFunction : ASTSymbolDeclaration mut bool hadDefaultValue = false; (nullable ASTSymbol | fail Error) parseDefaultValue() { mut nullable ASTSymbol defaultValue; - if (parser.accept("=")?) { + if (parser.acceptToken(TokenType.equal)) { auto defaultValue = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; if (!defaultValue) return parser.fail("default parameter value expected"); hadDefaultValue = true; @@ -744,29 +744,27 @@ class ASTFunction : ASTSymbolDeclaration return parser.fail("only default parameters allowed after default parameter"); } } - while (!parser.accept(")")?) - { - if (params.length > 0) - { - if (!parser.accept(",")?) - { + while (!parser.acceptToken(TokenType.rparen)) { + if (params.length > 0) { + if (!parser.acceptToken(TokenType.comma)) { return parser.fail("',' or ')' expected"); } } // foo(int a, ) - if (parser.accept(")")?) break; + if (parser.acceptToken(TokenType.rparen)) break; auto from = parser.from?; if (variadic) { parser.begin; - if (parser.accept("...")? && parser.accept(")")?) { + if (parser.acceptToken3(TokenType.dot, TokenType.dot, TokenType.dot) + && parser.acceptToken(TokenType.rparen)) { parser.commit; haveVariadic = true; break; } parser.revert; } - if (thisAssignment && parser.accept("this.")?) + if (thisAssignment && parser.acceptIdentifier("this")? && parser.acceptToken(TokenType.dot)) { string paramName = parser.parseIdentifier?; auto defaultValue = parseDefaultValue?; diff --git a/src/neat/lexer.nt b/src/neat/lexer.nt new file mode 100644 index 00000000..8526b445 --- /dev/null +++ b/src/neat/lexer.nt @@ -0,0 +1,238 @@ +module neat.lexer; + +macro import std.macro.assert; + +import neat.base; +import std.file; +import std.stdio; + +// FIXME => +// FIXME 'c' + +struct Token { + TokenType type; + int pos; + int row, col; + version (firstpass) + string toString() return "<$(row+1):$(col+1): $(cast(int) type)>"; + else + string toString() return "<$(row+1):$(col+1): $type>"; +} + +// FIXME with (TokenType) +(Token[] | fail Error) tokenize(string text, string filename="", mut int row=0, mut int col=0) { + mut Token[] result = new Token[](4); + mut int resultIdx = 0; + mut int pos = 0; + mut int parenLevel = 0; + mut int[] stringParenLevels; + void stepRow(int dist, int cols) { + pos += dist; + col += cols; + } + void stepRow1() { + stepRow(1, 1); + } + void stepNewline() { + pos++; + row++; + col = 0; + } + void stepUtf8() { + // FIXME return stepRow + if (text[pos] < 0b1000_0000) { stepRow(1, 1); return; } + if (text[pos] < 0b1110_0000) { stepRow(2, 1); return; } + if (text[pos] < 0b1111_0000) { stepRow(3, 1); return; } + stepRow(4, 1); + return; + } + void addTokenInplace(TokenType type) { + if (resultIdx == result.length) { + auto old = result; + result = new Token[](result.length * 2); + for (i, token in old) result[i] = token; + } + result[resultIdx++] = Token(type, pos, row, col); + } + void addToken(TokenType type, bool newline=false) { + addTokenInplace(type); + if (newline) stepNewline; + else stepRow1; + } + auto isIdentStarter = ch => ch.isAlpha || ch == "_"[0] || ch.isUtf8MbStart; + auto isIdentMiddle = ch => ch.isAlnum || ch == "_"[0] || ch.isUtf8MbStart; + // if (!filename.empty) print("=== tokenize $filename ==="); + void stepStringLiteral() { + while (pos < text.length && text[pos] != "\""[0]) { + if (pos < text.length - 1 && text[pos .. pos + 2] == "\$(") { + stringParenLevels ~= parenLevel++; + addToken(TokenType.formatQuoteStart); + addToken(TokenType.lparen); + return; + } + if (text[pos] == "\$"[0]) { + addToken(TokenType.formatQuoteStart); + if (pos < text.length && text[pos].isIdentStarter) { + addToken(TokenType.identifier); + while (pos < text.length && text[pos].isIdentMiddle) + pos++; + } + if (pos < text.length && text[pos] != "\""[0]) + addTokenInplace(TokenType.stringLiteral); + continue; + } + if (pos < text.length - 1 && text[pos] == "\\"[0]) { + // skip escaped character + stepRow1; + } + if (text[pos] == "\n"[0]) stepNewline; + else stepUtf8; + } + if (pos < text.length) + addToken(TokenType.doubleQuote); + } + while (pos < text.length) { + auto ch = text[pos]; + if (ch.isIdentStarter) { + addTokenInplace(TokenType.identifier); + stepUtf8; + while (pos < text.length && text[pos].isIdentMiddle) + stepUtf8; + } + else if (ch.isWhitespace) { + addToken(TokenType.whitespace, newline=text[pos] == "\n"[0]); + while (pos < text.length && text[pos].isWhitespace) { + if (text[pos] == "\n"[0]) stepNewline; + else stepRow1; + } + } + else if (ch.isNumeric) { + addToken(TokenType.number); + while (pos < text.length && (text[pos].isNumeric || text[pos] == "_"[0])) + stepRow1; + } + else if (ch == "/"[0] && pos + 1 < text.length && text[pos + 1] == "/"[0]) { + addToken(TokenType.comment); + stepRow(1, 1); + while (pos < text.length && text[pos] != "\n"[0]) + stepRow1; + } + else if (ch == "/"[0] && pos + 1 < text.length && text[pos + 1] == "*"[0]) { + addToken(TokenType.comment); + stepRow(1, 1); + mut int depth = 1; + while (pos + 1 < text.length) { + if (text[pos .. pos + 2] == "/*") { + depth++; + stepRow(2, 2); + } else if (text[pos .. pos + 2] == "*/") { + depth--; + stepRow(2, 2); + if (depth == 0) break; + } else if (text[pos] == "\n"[0]) stepNewline; + else stepUtf8; + } + } + else if (ch == "'"[0]) { + addToken(TokenType.singleQuote); + if (pos < text.length && text[pos] != "'"[0]) + addTokenInplace(TokenType.charLiteral); + while (pos < text.length && text[pos] != "'"[0]) { + if (pos < text.length - 1 && text[pos] == "\\"[0]) { + // skip escaped character + stepRow1; + } + stepUtf8; + } + addToken(TokenType.singleQuote); + } + else if (ch == "\""[0]) { + addToken(TokenType.doubleQuote); + if (pos < text.length && text[pos] != "\""[0]) + addTokenInplace(TokenType.stringLiteral); + stepStringLiteral; + } + else if (ch == "("[0]) { + addToken(TokenType.lparen); + parenLevel++; + } + else if (ch == ")"[0]) { + addToken(TokenType.rparen); + parenLevel--; + if (!stringParenLevels.empty && stringParenLevels[$ - 1] == parenLevel) { + addTokenInplace(TokenType.stringLiteral); + stringParenLevels = stringParenLevels[0 .. $ - 1]; + stepStringLiteral; + } + } + else if (ch == "."[0]) addToken(TokenType.dot); + else if (ch == ","[0]) addToken(TokenType.comma); + else if (ch == "`"[0]) addToken(TokenType.backtick); + else if (ch == ":"[0]) addToken(TokenType.colon); + else if (ch == ";"[0]) addToken(TokenType.semicolon); + else if (ch == "<"[0]) addToken(TokenType.smaller); + else if (ch == ">"[0]) addToken(TokenType.greater); + else if (ch == "="[0]) addToken(TokenType.equal); + else if (ch == "!"[0]) addToken(TokenType.exclamationmark); + else if (ch == "?"[0]) addToken(TokenType.questionmark); + else if (ch == "\$"[0]) addToken(TokenType.dollar); + else if (ch == "["[0]) addToken(TokenType.lsquarebracket); + else if (ch == "]"[0]) addToken(TokenType.rsquarebracket); + else if (ch == "{"[0]) addToken(TokenType.lcurlybracket); + else if (ch == "}"[0]) addToken(TokenType.rcurlybracket); + else if (ch == "+"[0]) addToken(TokenType.plus); + else if (ch == "-"[0]) addToken(TokenType.minus); + else if (ch == "*"[0]) addToken(TokenType.asterisk); + else if (ch == "/"[0]) addToken(TokenType.slash); + else if (ch == "\\"[0]) addToken(TokenType.backslash); + else if (ch == "^"[0]) addToken(TokenType.circumflex); + else if (ch == "%"[0]) addToken(TokenType.percent); + else if (ch == "~"[0]) addToken(TokenType.tilde); + else if (ch == "&"[0]) addToken(TokenType.ampersand); + else if (ch == "|"[0]) addToken(TokenType.bar); + else { + print("TODO $(cast(int) ch)"); + assert(false); + } + } + addTokenInplace(TokenType.end); + return result[0 .. resultIdx]; +} + +bool isAlpha(char ch) return ch >= "a"[0] && ch <= "z"[0] || ch >= "A"[0] && ch <= "Z"[0]; +bool isNumeric(char ch) return ch >= "0"[0] && ch <= "9"[0]; +bool isAlnum(char ch) return ch.isAlpha || ch.isNumeric; +bool isWhitespace(char ch) return ch == " "[0] || ch == "\t"[0] || ch == "\r"[0] || ch == "\n"[0]; +bool isUtf8MbStart(char ch) return ch >= 0x80; + +private (string | fail Error) annotate(string text) { + import std.string : split; + + mut auto tokens = text.tokenize("")?; + mut string annot; + mut int pos = 0; + + while (pos < text.length) { + while (!tokens.empty && tokens[0].pos < pos) + tokens = tokens[1 .. $]; + if (!tokens.empty && tokens[0].pos == pos) + version (firstpass) + annot ~= "$(cast(int) tokens[0].type)"; + else + annot ~= tokens[0].type.toString.front + "A"[0] - "a"[0]; + else + annot ~= " "; + if (text[pos] < 0b1000_0000) pos += 1; + else if (text[pos] < 0b1110_0000) pos += 2; + else if (text[pos] < 0b1111_0000) pos += 3; + else pos += 4; + } + return annot; +} + +unittest { + assert( + "assert(sin(π) == 0);".annotate == + "I LI LIRWEEWNRS"); +} + diff --git a/src/neat/parser.nt b/src/neat/parser.nt index 2d75e20c..7903f9a5 100644 --- a/src/neat/parser.nt +++ b/src/neat/parser.nt @@ -3,54 +3,13 @@ module neat.parser; import backend.base; import helpers; import neat.base; - -/** - * Advance row/column by byte distance `distance`. - */ -StackEntry advance(mut StackEntry entry, string fulltext, int distance) { - mut auto skiptext = fulltext[entry.offset .. entry.offset + distance]; - entry.offset += distance; - - mut auto nl = skiptext.find("\n"); - if (nl == -1) { - // TODO unicode count - entry.column += cast(int) distance; - entry.stripped = false; - return entry; - } - while (nl != -1) { - skiptext = skiptext[nl + 1 .. $]; - entry.row += 1; - nl = skiptext.find("\n"); - } - // no more newlines found, remainder are columns - // TODO unicode count - entry.column = cast(int) skiptext.length; - entry.stripped = false; - return entry; -} - -struct StackEntry -{ - int offset; - int row, column; - bool stripped; - - /** - * Advance row/column by byte distance `distance`, known to represent one character that is not a \n. - */ - void advanceOneCharNonNewline(int distance) { - offset += distance; - column += 1; - stripped = false; - } -} +import neat.lexer; final class ParserImpl : Parser { - StackEntry[] stack; + int[] stack; - StackEntry current; + int current; int level; @@ -58,25 +17,24 @@ final class ParserImpl : Parser string fulltext; - (int from, int to) lastIdentifier; + Token[] tokens; - this(this.filename, this.fulltext) + this(this.filename, this.fulltext, this.tokens) { - this.stack = new StackEntry[](0); + this.stack = new int[](0); this.level = 0; - this.current = StackEntry(offset=0, row=0, column=0, stripped=false); - lastIdentifier = (-1, -1); + this.current = 0; verify; } override LocRange loc() { - auto tuple = this.current.((row, column)); + auto tuple = tokens[current].((row, col)); return LocRange(this.filename, tuple, tuple); } override (LocRange | fail Error) from() { - this.strip?; - auto tuple = this.current.((row, column)); + strip; + auto tuple = tokens[current].((row, col)); return LocRange(this.filename, tuple, tuple); } @@ -84,7 +42,7 @@ final class ParserImpl : Parser return LocRange( this.filename, from.from, - this.current.((row, column)), + tokens[current].((row, col)), ); } @@ -96,45 +54,24 @@ final class ParserImpl : Parser } override (string | fail Error) parseIdentifier() - with (transaction) { - strip?; - if (current.offset == lastIdentifier.from) { - auto res = fulltext[lastIdentifier.from .. lastIdentifier.to]; - drop(lastIdentifier.to - lastIdentifier.from); - commit; - return res; + strip; + // foo + if (tokens[current].type == TokenType.identifier) { + current++; + return fulltext[tokens[current - 1].pos .. tokens[current].pos]; } - auto from = current.offset; - mut string identifier = ""; - string startText = text; - { - if (hard_eof) { - return ""; - } - string unichar = peekUniChar; - int codepoint = unichar.utf8Decode; - if (!isAlpha(codepoint) - && codepoint != cast(int) "_"[0] && codepoint != cast(int) "\$"[0]) - { - return ""; - } - identifier = startText[0 .. unichar.length]; - dropOneCharNonNewline(cast(int) unichar.length); + // $foo + if (tokens[current].type == TokenType.dollar && tokens[current + 1].type == TokenType.identifier) { + current += 2; + return fulltext[tokens[current - 2].pos .. tokens[current].pos]; } - - while (!hard_eof) { - string unichar = peekUniChar; - int codepoint = unichar.utf8Decode; - if (!isAlnum(codepoint) - && codepoint != cast(int) "_"[0]) - break; - dropOneCharNonNewline(cast(int) unichar.length); - identifier = startText[0 .. identifier.length + unichar.length]; + // $ + if (tokens[current].type == TokenType.dollar) { + current ++; + return fulltext[tokens[current - 1].pos .. tokens[current].pos]; } - lastIdentifier = (from, current.offset); - commit; - return identifier; + return ""; } void verify() @@ -151,15 +88,95 @@ final class ParserImpl : Parser override string text() { - return this.fulltext[this.current.offset .. $]; + return this.fulltext[tokens[this.current].pos .. $]; + } + + override bool acceptToken(TokenType type) { + strip; + if (tokens[current].type == type) { + current++; + return true; + } + return false; + } + + override bool acceptToken2(TokenType first, TokenType second) { + strip; + if (tokens[current].type == first && tokens[current + 1].type == second) { + current += 2; + return true; + } + return false; + } + + override bool acceptToken3(TokenType first, TokenType second, TokenType third) { + strip; + if (tokens[current].type == first + && tokens[current + 1].type == second + && tokens[current + 2].type == third) + { + current += 3; + return true; + } + return false; + } + + override bool acceptToken2Not(TokenType match, TokenType nomatch) { + strip; + if (tokens[current].type == match && tokens[current + 1].type != nomatch) { + current ++; + return true; + } + return false; + } + + override bool acceptToken2Not2(TokenType match, TokenType nomatch1, TokenType nomatch2) { + strip; + if (tokens[current].type == match && tokens[current + 1].type.(that != nomatch1 && that != nomatch2)) { + current ++; + return true; + } + return false; + } + + override (string | :none) acceptTokenStr(TokenType type) { + strip; + if (tokens[current].type == type) { + current++; + return fulltext[tokens[current - 1].pos .. tokens[current].pos]; + } + return :none; + } + + override (bool | fail Error) acceptIdentifier(string identifier) { + strip; + if (tokens[current].type == TokenType.identifier + && fulltext[tokens[current].pos .. tokens[current + 1].pos] == identifier) + { + current++; + return true; + } + return false; + } + + override (void | Error) expectToken(TokenType type) { + strip; + if (tokens[current].type != type) { + version (firstpass) + return fail("$(cast(int) type) expected"); + else + return fail("$type expected"); + } + current++; } override void begin() { this.verify; + // TODO why this weird manual doubling? if (this.level == this.stack.length) { - StackEntry[] newStack = new StackEntry[](this.stack.length * 2 + 1); + int[] newStack = new int[](this.stack.length * 2 + 1); for (i, stack in this.stack) newStack[i] = stack; this.stack = newStack; } @@ -180,48 +197,39 @@ final class ParserImpl : Parser this.current = this.stack[this.level]; } - override void drop(int length) - { - this.current = this.current.advance(this.fulltext, length); - } - - override void dropOneCharNonNewline(int length) - { - this.current.advanceOneCharNonNewline(length); - } - - override string peekUniChar() + override bool peekToken(TokenType tokenType) { - auto text = this.text; - auto len = text.utf8NextLength; - return text[0 .. len]; + return tokens[current].type == tokenType; } override (bool | fail Error) accept(string match) { - this.begin; - this.strip?; - auto text = this.text; - if (text.length < match.length) - { - this.revert; - return false; - } - if (text[0 .. match.length] == match) - { - this.drop(cast(int) match.length); - this.commit; - return true; + // print("accept '$match' $(text[0 .. 20])"); + mut int pos = current; + while (tokens[pos].type == TokenType.whitespace || tokens[pos].type == TokenType.comment) + pos++; + auto matchTokens = match.tokenize?; + // ignore end token + for (i, token in matchTokens[0 .. $ - 1]) { + if (pos + i >= tokens.length + || token.type != tokens[pos + i].type + || match[token.pos .. matchTokens[i + 1].pos] != + fulltext[tokens[pos + i].pos .. tokens[pos + i + 1].pos]) + { + // print(" $i fail $(cast(int) token.type) $(cast(int) tokens[pos + i].type)"); + return false; + } } - this.revert; - return false; + current = pos + cast(int) matchTokens[0 .. $ - 1].length; + // print(" succ $(text[0 .. 20])"); + return true; } // checks that the text is at the hard end of the file, // ie. no more text at all. override bool hard_eof() { - return this.current.offset == this.fulltext.length; + return tokens[current].type == TokenType.end; } // checks that the parser is at the "soft" end of the file, @@ -229,7 +237,7 @@ final class ParserImpl : Parser override (bool | fail Error) eof() { this.begin; - this.strip?; + this.strip; if (hard_eof) { this.commit; @@ -239,79 +247,34 @@ final class ParserImpl : Parser return false; } - override (void | fail Error) strip() + override void strip() { - if (this.current.stripped) return; - parseLastComment?; - this.current.stripped = true; + while (current < tokens.length && tokens[current].type.( + that == TokenType.whitespace || that == TokenType.comment) + ) + current++; } // Accept many comments; return the last one. - override (string | fail Error) parseLastComment() + override string parseLastComment() { - mut string lastComment; + mut (int | :none) lastComment = :none; while (true) { - /* this.text = this.text.strip; */ - mut string text = this.text; - while (text.length > 0 && isWhitespace(text[0])) - text = text[1 .. $]; - drop(cast(int) (this.text.length - text.length)); - auto comment = parseComment?; - if (!comment.ptr) { - return lastComment; - } - lastComment = comment; - } - } - - // Because it's called from strip(), this among all accept functions does not strip on its own. - (string | fail Error) parseComment() - { - if (startsWith(this.text, "//")) - { - int newline = find(this.text, "\n"); - assert(newline != -1); - auto ret = this.text[0 .. newline]; - drop(newline + 1); - return ret; - } - else if (startsWith(this.text, "/*")) - { - mut auto text = this.text; - text = text["/*".length .. $]; - mut int commentLevel = 1; - while (commentLevel > 0) - { - int more = find(text, "/*"); - int less = find(text, "*/"); - - if (more == -1 && less == -1) { - drop(cast(int) (this.text.length - text.length)); - return this.fail("comment spans end of file"); - } - if (less != -1 && (more == -1 || less < more)) - { - text = text[less + "*/".length .. $]; - commentLevel = commentLevel - 1; - } - if (more != -1 && (less == -1 || more < less)) - { - text = text[more + "/*".length .. $]; - commentLevel = commentLevel + 1; - } + while (current < tokens.length && tokens[current].type == TokenType.whitespace) + current++; + if (tokens[current].type == TokenType.comment) { + lastComment = current++; } - auto offset = this.text.length - text.length; - auto ret = this.text[0 .. offset]; - drop(cast(int) offset); - return ret; - } else { - return null; + else break; } + return lastComment.case( + :none: null, + int t: fulltext[tokens[t].pos .. tokens[t + 1].pos]); } override Error fail(string message) { - this.strip?; + this.strip; return this.loc.fail(message); } @@ -462,71 +425,61 @@ struct Loc { } } -bool isAlpha(int ch) { - // TODO support full unicode - if ((ch >= cast(int) "a"[0] && ch <= cast(int) "z"[0]) - || (ch >= cast(int) "A"[0] && ch <= cast(int) "Z"[0])) - return true; - if (ch < 0x80) - return false; - // greek letters - if (ch >= 0x0391 && ch <= 0x03c9) - return true; - return false; -} - bool isDigit(int ch) { return ch >= cast(int) "0"[0] && ch <= cast(int) "9"[0]; } (:failure | :success, long, bool withBasePrefix | fail Error) parseNumber(Parser parser) { + import std.string : replace; + parser.begin; - mut bool negative = parser.accept("-")?; - if (parser.accept("-")?) - negative = true; - parser.strip?; - if (parser.accept("0x")?) - { - return parseHexNumber(parser, negative).case( - :failure: :failure, - (:success, long value): (:success, value, withBasePrefix=true)); - } - if (parser.accept("0b")?) - { - return parseBinaryNumber(parser, negative).case( - :failure: :failure, - (:success, long value): (:success, value, withBasePrefix=true)); - } - if (parser.hard_eof || !isDigit(parser.text[0])) - { + bool negative = parser.acceptToken(TokenType.minus); + parser.strip; + auto number = parser.acceptTokenStr(TokenType.number) + .case(:none: ({ parser.revert; return :failure; })); + // we definitely have at least a number. + parser.commit; + if (number == "0") { + // check for 0x..., 0b... + parser.begin; + auto ident = parser.acceptTokenStr(TokenType.identifier).case(:none: ""); + if (ident.startsWith("x")) { + parseHexNumber(ident[1 .. $], negative).case { + :failure: + parser.revert; + return (:failure); + (:success, long value): + parser.commit; + return (:success, value, withBasePrefix=true); + } + } + if (ident.startsWith("b")) { + parseBinaryNumber(ident[1 .. $], negative).case { + :failure: + parser.revert; + return (:failure); + (:success, long value): + parser.commit; + return (:success, value, withBasePrefix=true); + } + } parser.revert; - return :failure; } - mut string number; - while (!parser.hard_eof && (parser.text[0].isDigit || parser.text[0] == "_"[0])) - { - if (parser.text[0] != "_"[0]) - number ~= parser.text[0]; - parser.dropOneCharNonNewline(1); - } - parser.commit; + auto number = number.replace("_", ""); mut long l = atol(number); if (negative) l = -l; return (:success, l, withBasePrefix=false); } -(:failure | :success, long value) parseHexNumber(Parser parser, bool negative) +(:failure | :success, long value) parseHexNumber(mut string number, bool negative) { - if (parser.hard_eof || !isHexDigit(parser.text[0 .. 1])) - { - parser.revert; + if (number.empty || !isHexDigit(number.front)) return :failure; - } mut long result; - while (!parser.hard_eof && (parser.text[0 .. 1].isHexDigit || parser.text[0] == "_"[0])) + while (!number.empty && (number.front.isHexDigit || number.front == "_"[0])) { - auto ch = parser.text[0 .. 1]; + auto ch = number[0 .. 1]; mut int digit; if (isDigit(ch[0])) digit = atoi(ch); else if (ch == "a" || ch == "A") digit = 10; @@ -536,55 +489,53 @@ bool isDigit(int ch) { else if (ch == "e" || ch == "E") digit = 14; else if (ch == "f" || ch == "F") digit = 15; else if (ch == "_") { - parser.dropOneCharNonNewline(1); + number = number[1 .. $]; continue; } else assert(false); result = result * 16 + digit; - parser.dropOneCharNonNewline(1); + number = number[1 .. $]; } - parser.commit; + if (!number.empty) return :failure; if (negative) result = -result; return (:success, result); } -bool isHexDigit(string digit) +bool isHexDigit(char digit) { - if (isDigit(digit[0])) return true; - if (digit == "a" || digit == "A") return true; - if (digit == "b" || digit == "B") return true; - if (digit == "c" || digit == "C") return true; - if (digit == "d" || digit == "D") return true; - if (digit == "e" || digit == "E") return true; - if (digit == "f" || digit == "F") return true; + if (isDigit(digit)) return true; + if (digit == "a"[0] || digit == "A"[0]) return true; + if (digit == "b"[0] || digit == "B"[0]) return true; + if (digit == "c"[0] || digit == "C"[0]) return true; + if (digit == "d"[0] || digit == "D"[0]) return true; + if (digit == "e"[0] || digit == "E"[0]) return true; + if (digit == "f"[0] || digit == "F"[0]) return true; return false; } -(:failure | :success, long value) parseBinaryNumber(Parser parser, bool negative) +(:failure | :success, long value) parseBinaryNumber(mut string number, bool negative) { - bool isBinaryDigit(string d) { - return d == "0" || d == "1"; + bool isBinaryDigit(char ch) { + return ch == "0"[0] || ch == "1"[0]; } - if (parser.hard_eof || !parser.text[0 .. 1].isBinaryDigit) - { - parser.revert; + if (number.empty || !isBinaryDigit(number.front)) { return :failure; } mut long result; - while (!parser.hard_eof && (parser.text[0 .. 1].isBinaryDigit || parser.text[0] == "_"[0])) + while (!number.empty && (number.front.isBinaryDigit || number.front == "_"[0])) { - auto ch = parser.text[0 .. 1]; + auto ch = number[0 .. 1]; // mut int digit = if (ch == "0") 0; else 1; mut int digit; if (ch == "0") digit = 0; else if (ch == "1") digit = 1; else if (ch == "_") { - parser.dropOneCharNonNewline(1); + number = number[1 .. $]; continue; } else assert(false); result = result * 2 + digit; - parser.dropOneCharNonNewline(1); + number = number[1 .. $]; } - parser.commit; + if (!number.empty) return :failure; if (negative) result = -result; return (:success, result); } @@ -592,35 +543,22 @@ bool isHexDigit(string digit) (:failure | :success, float | :success, double | fail Error) parseFloat(Parser parser) with (parser.transaction) { - bool negative = parser.accept("-")?; - parser.strip?; - mut string number; - while (!parser.hard_eof && isDigit(parser.text[0])) - { - number ~= parser.text[0]; - parser.dropOneCharNonNewline(1); - } - if (parser.accept(".")?) {} - else return :failure; + bool negative = parser.acceptToken(TokenType.minus); + parser.strip; + mut string number = parser.acceptTokenStr(TokenType.number) + .case(:none: return :failure); + if (!parser.acceptToken(TokenType.dot)) return :failure; number ~= "."; // 2.foo - if (parser.hard_eof || !isDigit(parser.text[0])) return :failure; - while (!parser.hard_eof && isDigit(parser.text[0])) - { - number ~= parser.text[0]; - parser.dropOneCharNonNewline(1); - } + auto cont = parser.acceptTokenStr(TokenType.number) + .case(:none: return :failure); + if (parser.hard_eof) return :failure; + number ~= cont; commit; mut double d = atof(number); if (negative) d = -d; - if (!parser.hard_eof && parser.text[0] == "f"[0]) { - parser.dropOneCharNonNewline(1); + if (parser.acceptIdentifier("f")?) { return (:success, cast(float) d); } return (:success, d); } - -bool isAlnum(int ch) -{ - return isAlpha(ch) || isDigit(ch); -} diff --git a/src/neat/pragmas.nt b/src/neat/pragmas.nt index 0b58471d..e4c9972b 100644 --- a/src/neat/pragmas.nt +++ b/src/neat/pragmas.nt @@ -11,18 +11,18 @@ import neat.types; { if (!parser.accept("pragma")?) return null; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto from = parser.from?; auto name = parser.parseIdentifier?; auto locRange = parser.to(from); mut ASTSymbol[] args; - while (parser.accept(",")?) { + while (parser.acceptToken(TokenType.comma)) { if (auto arg = lexicalContext.compiler.parseExpression(parser, lexicalContext)?) args ~= arg; else return parser.fail("Expected expression argument"); } - parser.expect(")")?; - parser.expect(";")?; + parser.expectToken(TokenType.rparen)?; + parser.expectToken(TokenType.semicolon)?; return new ASTPragma(name, args, locRange); } diff --git a/src/neat/simple_parser.nt b/src/neat/simple_parser.nt new file mode 100644 index 00000000..e1bd9a3b --- /dev/null +++ b/src/neat/simple_parser.nt @@ -0,0 +1,699 @@ +/* + * The old non-lexer parser impl. + */ +module neat.simple_parser; + +import backend.base; +import helpers; +import neat.base; + +/** + * Advance row/column by byte distance `distance`. + */ +StackEntry advance(mut StackEntry entry, string fulltext, int distance) { + mut auto skiptext = fulltext[entry.offset .. entry.offset + distance]; + entry.offset += distance; + + mut auto nl = skiptext.find("\n"); + if (nl == -1) { + // TODO unicode count + entry.column += cast(int) distance; + entry.stripped = false; + return entry; + } + while (nl != -1) { + skiptext = skiptext[nl + 1 .. $]; + entry.row += 1; + nl = skiptext.find("\n"); + } + // no more newlines found, remainder are columns + // TODO unicode count + entry.column = cast(int) skiptext.length; + entry.stripped = false; + return entry; +} + +struct StackEntry +{ + int offset; + int row, column; + bool stripped; + + /** + * Advance row/column by byte distance `distance`, known to represent one character that is not a \n. + */ + void advanceOneCharNonNewline(int distance) { + offset += distance; + column += 1; + stripped = false; + } +} + +final class SimpleParser +{ + StackEntry[] stack; + + StackEntry current; + + int level; + + string filename; + + string fulltext; + + (int from, int to) lastIdentifier; + + this(this.filename, this.fulltext) + { + this.stack = new StackEntry[](0); + this.level = 0; + this.current = StackEntry(offset=0, row=0, column=0, stripped=false); + lastIdentifier = (-1, -1); + verify; + } + + LocRange loc() { + auto tuple = this.current.((row, column)); + return LocRange(this.filename, tuple, tuple); + } + + (LocRange | fail Error) from() { + this.strip?; + auto tuple = this.current.((row, column)); + return LocRange(this.filename, tuple, tuple); + } + + LocRange to(LocRange from) { + return LocRange( + this.filename, + from.from, + this.current.((row, column)), + ); + } + + (void | fail Error) verifyTransactions(string msg, (void | fail Error) delegate() dg) { + auto locEntry = loc; + auto startLevel = level; + dg()?; + locEntry.hardCheck(level == startLevel, msg); + } + + (string | fail Error) parseIdentifier() + with (transaction) + { + strip?; + if (current.offset == lastIdentifier.from) { + auto res = fulltext[lastIdentifier.from .. lastIdentifier.to]; + drop(lastIdentifier.to - lastIdentifier.from); + commit; + return res; + } + auto from = current.offset; + mut string identifier = ""; + string startText = text; + { + if (hard_eof) { + return ""; + } + string unichar = peekUniChar; + int codepoint = unichar.utf8Decode; + if (!isAlpha(codepoint) + && codepoint != cast(int) "_"[0] && codepoint != cast(int) "\$"[0]) + { + return ""; + } + identifier = startText[0 .. unichar.length]; + dropOneCharNonNewline(cast(int) unichar.length); + } + + while (!hard_eof) { + string unichar = peekUniChar; + int codepoint = unichar.utf8Decode; + if (!isAlnum(codepoint) + && codepoint != cast(int) "_"[0]) + break; + dropOneCharNonNewline(cast(int) unichar.length); + identifier = startText[0 .. identifier.length + unichar.length]; + } + lastIdentifier = (from, current.offset); + commit; + return identifier; + } + + void verify() + { + if (this.level < 0 || this.level > this.stack.length) { + print("parse stack violation: $(this.level), $(this.stack.length)"); + exit(1); + } + if (this.stack.length > 1024) { + print("parse stack overflow"); + exit(1); + } + } + + string text() + { + return this.fulltext[this.current.offset .. $]; + } + + void begin() + { + this.verify; + if (this.level == this.stack.length) + { + StackEntry[] newStack = new StackEntry[](this.stack.length * 2 + 1); + for (i, stack in this.stack) newStack[i] = stack; + this.stack = newStack; + } + this.stack[this.level] = this.current; + this.level += 1; + } + + void commit() + { + assert(this.level > 0); + this.level -= 1; + } + + void revert() + { + this.verify; + this.level -= 1; + this.current = this.stack[this.level]; + } + + void drop(int length) + { + this.current = this.current.advance(this.fulltext, length); + } + + void dropOneCharNonNewline(int length) + { + this.current.advanceOneCharNonNewline(length); + } + + string peekUniChar() + { + auto text = this.text; + auto len = text.utf8NextLength; + return text[0 .. len]; + } + + (bool | fail Error) accept(string match) + { + this.begin; + this.strip?; + auto text = this.text; + if (text.length < match.length) + { + this.revert; + return false; + } + if (text[0 .. match.length] == match) + { + this.drop(cast(int) match.length); + this.commit; + return true; + } + this.revert; + return false; + } + + // checks that the text is at the hard end of the file, + // ie. no more text at all. + bool hard_eof() + { + return this.current.offset == this.fulltext.length; + } + + // checks that the parser is at the "soft" end of the file, + // ie. no more meaningful content; only comments and whitespace. + (bool | fail Error) eof() + { + this.begin; + this.strip?; + if (hard_eof) + { + this.commit; + return true; + } + this.revert; + return false; + } + + (void | fail Error) strip() + { + if (this.current.stripped) return; + parseLastComment?; + this.current.stripped = true; + } + + // Accept many comments; return the last one. + (string | fail Error) parseLastComment() + { + mut string lastComment; + while (true) { + /* this.text = this.text.strip; */ + mut string text = this.text; + while (text.length > 0 && isWhitespace(text[0])) + text = text[1 .. $]; + drop(cast(int) (this.text.length - text.length)); + auto comment = parseComment?; + if (!comment.ptr) { + return lastComment; + } + lastComment = comment; + } + } + + // Because it's called from strip(), this among all accept functions does not strip on its own. + (string | fail Error) parseComment() + { + if (startsWith(this.text, "//")) + { + int newline = find(this.text, "\n"); + assert(newline != -1); + auto ret = this.text[0 .. newline]; + drop(newline + 1); + return ret; + } + else if (startsWith(this.text, "/*")) + { + mut auto text = this.text; + text = text["/*".length .. $]; + mut int commentLevel = 1; + while (commentLevel > 0) + { + int more = find(text, "/*"); + int less = find(text, "*/"); + + if (more == -1 && less == -1) { + drop(cast(int) (this.text.length - text.length)); + return this.fail("comment spans end of file"); + } + if (less != -1 && (more == -1 || less < more)) + { + text = text[less + "*/".length .. $]; + commentLevel = commentLevel - 1; + } + if (more != -1 && (less == -1 || more < less)) + { + text = text[more + "/*".length .. $]; + commentLevel = commentLevel + 1; + } + } + auto offset = this.text.length - text.length; + auto ret = this.text[0 .. offset]; + drop(cast(int) offset); + return ret; + } else { + return null; + } + } + + Error fail(string message) + { + this.strip?; + return this.loc.fail(message); + } + + (void | fail Error) assert2s2(int test, string a, string b) { + if (!test) return this.fail(a ~ b); + return; + } + (void | fail Error) assert2s3(int test, string a, string b, string c) { + if (!test) return this.fail(a ~ b ~ c); + return; + } + (void | fail Error) assert2s4(int test, string a, string b, string c, string d) { + if (!test) return this.fail(a ~ b ~ c ~ d); + return; + } + (void | fail Error) assert2s5(int test, string a, string b, string c, string d, string e) { + if (!test) return this.fail(a ~ b ~ c ~ d ~ e); + return; + } + + (void | fail Error) assert_(bool flag, string message) + { + if (!flag) return this.fail(message); + return; + } + + (void | fail Error) expect(string match) + { + if (this.accept(match)?) + return; + return this.fail("'" ~ match ~ "' expected"); + } + + (bool | fail Error) acceptIdentifier(string identifier) + with (transaction) + { + string nextIdent = parseIdentifier?; + + if (nextIdent != identifier) + { + return false; + } + commit; + return true; + } + + ParserTransaction transaction() { + return ParserTransaction(this, committed=false); + } + + (bool | fail Error) acceptButNot(string match, string nomatch) { + // TODO peek? + begin; + bool fail = accept(nomatch)?; + revert; + return !fail && accept(match)?; + } + + (bool | fail Error) acceptButNot2(string match, string nomatch1, string nomatch2) { + // TODO peek? + begin; + bool fail2 = accept(nomatch1)?; + revert; + begin; + bool fail1 = accept(nomatch2)?; + revert; + return !fail1 && !fail2 && accept(match)?; + } + + bool isKeyword(string identifier) { + // :sigh: TODO Give macros the ability to define custom reserved keywords. + if (identifier == "the") return true; + return identifier == "mut" || identifier == "auto" || identifier == "new" || identifier == "return"; + } +} + +struct ParserTransaction +{ + private SimpleParser parser; + private bool committed; + void onEnter() { + parser.begin; + } + void onExit() { + if (!committed) parser.revert; + } + void commit() { + assert(!committed); + parser.commit; + committed = true; + } +} + +bool isWhitespace(char c) +{ + return c == " "[0] || c == "\t"[0] || c == "\r"[0] || c == "\n"[0]; +} + +int utf8Decode(string ch) +{ + assert(ch.length > 0); + if (ch.length == 1) return ch[0]; + if (ch.length == 2) return cast(int)(ch[0]) & 0x1f << 6 | cast(int)(ch[1]) & 0x3f; + if (ch.length == 3) return cast(int)(ch[0]) & 0x0f << 12 | cast(int)(ch[1]) & 0x3f << 6 | cast(int)(ch[2]) & 0x3f; + if (ch.length == 4) + return cast(int)(ch[0]) & 0x07 << 18 | cast(int)(ch[1]) & 0x3f << 12 + | cast(int)(ch[2]) & 0x3f << 6 | cast(int)(ch[3]) & 0x3f; + if (ch.length == 5) + return cast(int)(ch[0]) & 0x03 << 24 | cast(int)(ch[1]) & 0x3f << 18 + | cast(int)(ch[2]) & 0x3f << 12 | cast(int)(ch[3]) & 0x3f << 6 | cast(int)(ch[4]) & 0x3f; + return cast(int)(ch[0]) & 0x01 << 30 | cast(int)(ch[1]) & 0x3f << 24 | cast(int)(ch[2]) & 0x3f << 18 + | cast(int)(ch[3]) & 0x3f << 12 | cast(int)(ch[4]) & 0x3f << 6 | cast(int)(ch[5]) & 0x3f; +} + +int utf8NextLength(string text) +{ + // see https://en.wikipedia.org/wiki/UTF-8#FSS-UTF + if (text.length < 1) return 0; + int ch0 = text[0]; + if (ch0 < 128) return 1; + assert(ch0 >= 192); + assert(text.length >= 2); + if (ch0 < 224) return 2; + assert(text.length >= 3); + if (ch0 < 240) return 3; + assert(text.length >= 4); + if (ch0 < 248) return 4; + assert(text.length >= 5); + if (ch0 < 252) return 5; + assert(text.length >= 6); + if (ch0 < 254) return 6; + assert(false); +} + +bool startsWith(string haystack, string needle) +{ + if (needle.length == 1) { + return haystack.length >= 1 && haystack[0] == needle[0]; + } else if (needle.length == 2) { + return haystack.length >= 2 && haystack[0] == needle[0] && haystack[1] == needle[1]; + } else { + return haystack.length >= needle.length && haystack[0 .. needle.length] == needle; + } +} + +int find(string haystack, string needle) +{ + if (haystack.length < needle.length) return -1; + if (needle.length == 1) { + for (i in 0 .. haystack.length) { + if (haystack[i] == needle[0]) return cast(int) i; + } + } else if (needle.length == 2) { + for (i in 0 .. haystack.length - 1) { + if (haystack[i] == needle[0] && haystack[i + 1] == needle[1]) return cast(int) i; + } + } else { + for (i in 0 .. haystack.length - needle.length + 1) { + if (haystack[i .. i + needle.length] == needle) return cast(int) i; + } + } + return -1; +} + +extern(C) void exit(int); + +struct Loc { + string filename; + int row, column; + + BackendLocation toBackendLoc() { + mut BackendLocation ret; + ret.file = filename; + ret.line = row; + ret.column = column; + return ret; + } + + string location() { + return filename ~ ":" ~ itoa(row + 1) ~ ":" ~ itoa(column + 1); + } + + Error fail(string message) { + return new Error([locRange], message); + } + + LocRange locRange() { + return LocRange(filename, (row, column), (row, column)); + } + + (void | fail Error) assert(bool flag, string message) { + if (!flag) { + return this.fail(message); + } + } + + // This should have been caught earlier. + void hardCheck(bool flag, string message) { + if (flag) return; + print("$(location): Internal compiler error!"); + print(message); + exit(1); + } + + (void | fail Error) assert2s2(int test, string a, string b) { + if (!test) return this.fail(a ~ b); + return; + } + (void | fail Error) assert2s3(int test, string a, string b, string c) { + if (!test) return this.fail(a ~ b ~ c); + return; + } + (void | fail Error) assert2s4(int test, string a, string b, string c, string d) { + if (!test) return this.fail(a ~ b ~ c ~ d); + return; + } + (void | fail Error) assert2s5(int test, string a, string b, string c, string d, string e) { + if (!test) return this.fail(a ~ b ~ c ~ d ~ e); + return; + } +} + +bool isAlpha(int ch) { + // TODO support full unicode + if ((ch >= cast(int) "a"[0] && ch <= cast(int) "z"[0]) + || (ch >= cast(int) "A"[0] && ch <= cast(int) "Z"[0])) + return true; + if (ch < 0x80) + return false; + // greek letters + if (ch >= 0x0391 && ch <= 0x03c9) + return true; + return false; +} + +bool isDigit(int ch) { + return ch >= cast(int) "0"[0] && ch <= cast(int) "9"[0]; +} + +(:failure | :success, long, bool withBasePrefix | fail Error) parseNumber(SimpleParser parser) +{ + parser.begin; + bool negative = parser.accept("-")?; + parser.strip?; + if (parser.accept("0x")?) + { + return parseHexNumber(parser, negative).case( + :failure: :failure, + (:success, long value): (:success, value, withBasePrefix=true)); + } + if (parser.accept("0b")?) + { + return parseBinaryNumber(parser, negative).case( + :failure: :failure, + (:success, long value): (:success, value, withBasePrefix=true)); + } + if (parser.hard_eof || !isDigit(parser.text[0])) + { + parser.revert; + return :failure; + } + mut string number; + while (!parser.hard_eof && (parser.text[0].isDigit || parser.text[0] == "_"[0])) + { + if (parser.text[0] != "_"[0]) + number ~= parser.text[0]; + parser.dropOneCharNonNewline(1); + } + parser.commit; + mut long l = atol(number); + if (negative) l = -l; + return (:success, l, withBasePrefix=false); +} + +(:failure | :success, long value) parseHexNumber(SimpleParser parser, bool negative) +{ + if (parser.hard_eof || !isHexDigit(parser.text[0 .. 1])) + { + parser.revert; + return :failure; + } + mut long result; + while (!parser.hard_eof && (parser.text[0 .. 1].isHexDigit || parser.text[0] == "_"[0])) + { + auto ch = parser.text[0 .. 1]; + mut int digit; + if (isDigit(ch[0])) digit = atoi(ch); + else if (ch == "a" || ch == "A") digit = 10; + else if (ch == "b" || ch == "B") digit = 11; + else if (ch == "c" || ch == "C") digit = 12; + else if (ch == "d" || ch == "D") digit = 13; + else if (ch == "e" || ch == "E") digit = 14; + else if (ch == "f" || ch == "F") digit = 15; + else if (ch == "_") { + parser.dropOneCharNonNewline(1); + continue; + } else assert(false); + result = result * 16 + digit; + parser.dropOneCharNonNewline(1); + } + parser.commit; + if (negative) result = -result; + return (:success, result); +} + +bool isHexDigit(string digit) +{ + if (isDigit(digit[0])) return true; + if (digit == "a" || digit == "A") return true; + if (digit == "b" || digit == "B") return true; + if (digit == "c" || digit == "C") return true; + if (digit == "d" || digit == "D") return true; + if (digit == "e" || digit == "E") return true; + if (digit == "f" || digit == "F") return true; + return false; +} + +(:failure | :success, long value) parseBinaryNumber(SimpleParser parser, bool negative) +{ + bool isBinaryDigit(string d) { + return d == "0" || d == "1"; + } + if (parser.hard_eof || !parser.text[0 .. 1].isBinaryDigit) + { + parser.revert; + return :failure; + } + mut long result; + while (!parser.hard_eof && (parser.text[0 .. 1].isBinaryDigit || parser.text[0] == "_"[0])) + { + auto ch = parser.text[0 .. 1]; + // mut int digit = if (ch == "0") 0; else 1; + mut int digit; + if (ch == "0") digit = 0; + else if (ch == "1") digit = 1; + else if (ch == "_") { + parser.dropOneCharNonNewline(1); + continue; + } else assert(false); + result = result * 2 + digit; + parser.dropOneCharNonNewline(1); + } + parser.commit; + if (negative) result = -result; + return (:success, result); +} + +(:failure | :success, float | :success, double | fail Error) parseFloat(SimpleParser parser) +with (parser.transaction) +{ + bool negative = parser.accept("-")?; + parser.strip?; + mut string number; + while (!parser.hard_eof && isDigit(parser.text[0])) + { + number ~= parser.text[0]; + parser.dropOneCharNonNewline(1); + } + if (parser.accept(".")?) {} + else return :failure; + number ~= "."; + // 2.foo + if (parser.hard_eof || !isDigit(parser.text[0])) return :failure; + while (!parser.hard_eof && isDigit(parser.text[0])) + { + number ~= parser.text[0]; + parser.dropOneCharNonNewline(1); + } + commit; + mut double d = atof(number); + if (negative) d = -d; + if (!parser.hard_eof && parser.text[0] == "f"[0]) { + parser.dropOneCharNonNewline(1); + return (:success, cast(float) d); + } + return (:success, d); +} + +bool isAlnum(int ch) +{ + return isAlpha(ch) || isDigit(ch); +} diff --git a/src/neat/struct_.nt b/src/neat/struct_.nt index f06f0c94..11e270be 100644 --- a/src/neat/struct_.nt +++ b/src/neat/struct_.nt @@ -779,7 +779,7 @@ class StructLiteral : Expression (nullable ASTDeclaration | fail Error) parseStructDecl(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; if (parser.parseIdentifier? != "struct") { parser.revert; @@ -790,8 +790,8 @@ class StructLiteral : Expression assert(!!name.length); auto locRange = parser.to(from); mut ASTStructEntry[] entries; - parser.expect("{")?; - while (!parser.accept("}")?) { + parser.expectToken(TokenType.lcurlybracket)?; + while (!parser.acceptToken(TokenType.rcurlybracket)) { auto entry = parseStructEntry(parser, lexicalContext)?; entries ~= entry; } @@ -810,8 +810,8 @@ class StructLiteral : Expression (ASTStructEntry[] | fail Error) parseStructBlock(Parser parser, LexicalContext lexicalContext) { mut ASTStructEntry[] entries; - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { auto entry = parser.parseStructEntry(lexicalContext)?; entries ~= entry; } @@ -825,13 +825,13 @@ class StructLiteral : Expression (ASTStructEntry[] | fail Error) parseStructEntry(Parser parser, LexicalContext lexicalContext) { // FIXME merge version into static if via __traits(versionSet) - if (parser.accept("version")?) { - parser.expect("(")?; + if (parser.acceptIdentifier("version")?) { + parser.expectToken(TokenType.lparen)?; string versionStr = parser.parseIdentifier?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; ASTStructEntry[] then = parser.parseStructBlock(lexicalContext)?; mut ASTStructEntry[] else_; - if (parser.accept("else")?) { + if (parser.acceptIdentifier("else")?) { else_ = parser.parseStructBlock(lexicalContext)?; } // FIXME explicitly type hinted literal @@ -839,18 +839,18 @@ class StructLiteral : Expression return [result]; } parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; if (parser.acceptIdentifier("static")? && parser.acceptIdentifier("if")?) { parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto cond = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; if (!cond) { return parser.fail("static if condition expected"); } - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; ASTStructEntry[] then = parser.parseStructBlock(lexicalContext)?; mut ASTStructEntry[] else_; - if (parser.accept("else")?) { + if (parser.acceptIdentifier("else")?) { else_ = parser.parseStructBlock(lexicalContext)?; } // FIXME explicitly type hinted literal @@ -865,7 +865,7 @@ class StructLiteral : Expression mut Protection protection = Protection.public_; auto from = parser.from?; while (true) { - if (parser.accept("static")?) { + if (parser.acceptIdentifier("static")?) { static_ = true; continue; } @@ -888,7 +888,7 @@ class StructLiteral : Expression entries ~= aliasDecl.notNull; return entries; } - if (parser.accept("this")?) + if (parser.acceptIdentifier("this")?) { memberType = new ASTIdentifier("void", false, __RANGE__); memberName = "__ctor"; @@ -907,7 +907,7 @@ class StructLiteral : Expression if (!memberName.length) return parser.fail("expected member name"); } auto locRange = parser.to(from); - if (parser.accept("(")?) // method + if (parser.acceptToken(TokenType.lparen)) // method { auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=true)?; assert(!params.variadic); @@ -926,9 +926,9 @@ class StructLiteral : Expression entries ~= (memberName, protection, memberType.notNull); } addMember?; - while (!parser.accept(";")?) { - parser.expect(",")?; - if (parser.accept(";")?) break; + while (!parser.acceptToken(TokenType.semicolon)) { + parser.expectToken(TokenType.comma)?; + if (parser.acceptToken(TokenType.semicolon)) break; string ident = parser.parseIdentifier?; memberName = ident; if (!memberName.length) return parser.fail("expected member name"); diff --git a/src/neat/stuff.nt b/src/neat/stuff.nt index f42a0acb..773e682d 100644 --- a/src/neat/stuff.nt +++ b/src/neat/stuff.nt @@ -22,6 +22,7 @@ import neat.formatstring; import neat.function_; import neat.hash; import neat.hashmap; +import neat.lexer; import neat.parser; import neat.pragmas; import neat.statements; @@ -88,7 +89,7 @@ class DefineClassIntf : FinishedSymbol while (true) { auto from = parser.from?; - if (parser.accept("*")?) + if (parser.acceptToken(TokenType.asterisk)) { current = new ASTPointer(current, parser.to(from)); continue; @@ -96,11 +97,11 @@ class DefineClassIntf : FinishedSymbol with (parser.transaction) { string identifier = parser.parseIdentifier?; if (identifier == "function" || identifier == "delegate") { - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; mut ASTSymbol[] args; - while (!parser.accept(")")?) { + while (!parser.acceptToken(TokenType.rparen)) { if (args.length) { - if (!parser.accept(",")?) + if (!parser.acceptToken(TokenType.comma)) return parser.fail("',' or ')' expected"); } auto argType = parser.parseType(lexicalContext)?; @@ -116,8 +117,8 @@ class DefineClassIntf : FinishedSymbol } } with (parser.transaction) { - if (parser.accept("[")?) { - if (parser.accept("]")?) { + if (parser.acceptToken(TokenType.lsquarebracket)) { + if (parser.acceptToken(TokenType.rsquarebracket)) { commit; current = new ASTArray(current, parser.to(from)); @@ -125,7 +126,7 @@ class DefineClassIntf : FinishedSymbol } // hashmap? might be assignment intro auto keyType = parser.parseType(lexicalContext)?; - if (!keyType || !parser.accept("]")?) { + if (!keyType || !parser.acceptToken(TokenType.rsquarebracket)) { return null; } commit; @@ -141,7 +142,8 @@ class DefineClassIntf : FinishedSymbol with (parser.transaction) { auto instanceFrom = parser.from?; // TODO better way to exclude all the negated operators - if (parser.accept("!")? && !parser.accept("=")? && !parser.acceptIdentifier("is")?) + if (parser.acceptToken(TokenType.exclamationmark) + && !parser.acceptToken(TokenType.equal) && !parser.acceptIdentifier("is")?) { mut ASTSymbol[] templateArgs; (void | fail Error) getTemplateArg(bool short_) { @@ -158,12 +160,12 @@ class DefineClassIntf : FinishedSymbol } } - if (parser.accept("(")?) { + if (parser.acceptToken(TokenType.lparen)) { // ( [a [, a]*]? ) - if (!parser.accept(")")?) { + if (!parser.acceptToken(TokenType.rparen)) { getTemplateArg(false)?; - while (!parser.accept(")")?) { - parser.expect(",")?; + while (!parser.acceptToken(TokenType.rparen)) { + parser.expectToken(TokenType.comma)?; getTemplateArg(false)?; } } @@ -190,7 +192,7 @@ with (parser.transaction) { parser.begin; auto from = parser.from?; - if (!parser.accept("(")?) { + if (!parser.acceptToken(TokenType.lparen)) { parser.revert; return null; } @@ -198,7 +200,7 @@ with (parser.transaction) mut ASTEitherDeclEntry[] declEntries; mut (string name, LocRange locRange, ASTSymbol sym)[] members; mut bool fail = false; - if (parser.accept("fail")?) fail = true; + if (parser.acceptIdentifier("fail")?) fail = true; void flush() { assert(members.length > 0); @@ -213,12 +215,11 @@ with (parser.transaction) members = []; fail = false; } - while (!parser.accept(")")?) - { + while (!parser.acceptToken(TokenType.rparen)) { if (members.length) { mut bool tuplesep, eithersep; - if (parser.accept(",")?) tuplesep = true; - else if (parser.accept("|")?) eithersep = true; + if (parser.acceptToken(TokenType.comma)) tuplesep = true; + else if (parser.acceptToken(TokenType.bar)) eithersep = true; else { // be a bit lenient for (bla bla) if (members.length == 1) { @@ -228,7 +229,7 @@ with (parser.transaction) } if (eithersep) { flush; - if (parser.accept("fail")?) fail = true; + if (parser.acceptIdentifier("fail")?) fail = true; } } auto from = parser.from?; @@ -275,13 +276,13 @@ with (parser.transaction) if (auto symbol = parser.parseTupleType(lexicalContext)?) return symbol; with (parser.transaction) { - if (parser.acceptIdentifier("Vector")? && parser.accept("(")?) { + if (parser.acceptIdentifier("Vector")? && parser.acceptToken(TokenType.lparen)) { auto element = lexicalContext.compiler.parseType(parser, lexicalContext)?; parser.assert_(!!element, "element type expected")?; - parser.expect(",")?; + parser.expectToken(TokenType.comma)?; auto count = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; parser.assert_(count && count.instanceOf(ASTNumberLiteral), "number expected")?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; commit; return new ASTVectorType( element.notNull, @@ -289,17 +290,17 @@ with (parser.transaction) parser.to(from)); } - if (parser.accept("typeof")?) + if (parser.acceptIdentifier("typeof")?) { - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto expr = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; if (!expr) return parser.fail("expression expected"); - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; commit; return new ASTTypeof(expr.notNull, captureImplicitReturn=false, parser.to(from)); } - if (parser.accept(":")?) { + if (parser.acceptToken(TokenType.colon)) { string symbolIdentifier = parser.parseIdentifier?; if (symbolIdentifier != "") { commit; @@ -910,18 +911,18 @@ class ASTCall : ASTSymbol (ASTArgument[] | fail Error) parseArgumentList(Parser parser, LexicalContext lexicalContext) { mut ASTArgument[] args; - while (!parser.accept(")")?) + while (!parser.acceptToken(TokenType.rparen)) { if (args.length > 0) - parser.expect(",")?; + parser.expectToken(TokenType.comma)?; // foo(a, b, c, ) - if (parser.accept(")")?) break; + if (parser.acceptToken(TokenType.rparen)) break; auto from = parser.from?; mut string name; with (parser.transaction) { string n = parser.parseIdentifier?; - if (n.length > 0 && parser.acceptButNot2("=", "==", "=>")?) { + if (n.length > 0 && parser.acceptToken2Not2(TokenType.equal, TokenType.equal, TokenType.greater)) { commit; name = n; } @@ -937,7 +938,7 @@ class ASTCall : ASTSymbol with (parser.transaction) { auto from = parser.from?; - if (!parser.accept("(")?) + if (!parser.acceptToken(TokenType.lparen)) return null; mut ASTArgument[] args = parser.parseArgumentList(lexicalContext)?; commit; @@ -1076,12 +1077,12 @@ class ASTArraySlice : ASTSymbol with (parser.transaction) { auto from = parser.from?; - if (!parser.accept("[")?) + if (!parser.acceptToken(TokenType.lsquarebracket)) return null; nullable ASTSymbol index = parseExpression(parser, lexicalContext)?; parser.to(from).assert(!!index, "index expected")?; auto index = index.notNull; - parser.expect("]")?; + parser.expectToken(TokenType.rsquarebracket)?; commit; if (auto iota = index.instanceOf(ASTIota)) { return new ASTArraySlice(base, iota.from, iota.to, parser.to(from)); @@ -1204,7 +1205,9 @@ class ASTMember : ASTMemberBase { parser.begin; // don't accept '..' or '.(': conflicting syntax - if (parser.accept("..")? || parser.accept(".(")? || !parser.accept(".")?) + if (parser.acceptToken2(TokenType.dot, TokenType.dot) + || parser.acceptToken2(TokenType.dot, TokenType.lparen) + || !parser.acceptToken(TokenType.dot)) { parser.revert; return null; @@ -1295,7 +1298,10 @@ class ASTParenPropertyExpression : ASTSymbol ) { parser.begin; auto from = parser.from?; - if (parser.accept("..")? || !parser.accept(".")? || !parser.accept("(")?) // don't accept '..' + // don't accept '..' + if (parser.acceptToken2(TokenType.dot, TokenType.dot) + || !parser.acceptToken(TokenType.dot) + || !parser.acceptToken(TokenType.lparen)) { parser.revert; return null; @@ -1350,7 +1356,7 @@ class ASTPropagateFailureExpr : ASTSymbol { auto from = parser.from?; parser.begin; - if (!parser.accept("?")?) { + if (!parser.acceptToken(TokenType.questionmark)) { parser.revert; return null; } @@ -1362,8 +1368,8 @@ class ASTPropagateFailureExpr : ASTSymbol { mut (:inc | :dec | :none) preincdec = :none, postincdec = :none; auto fromPre = parser.from?; - if (parser.accept("--")?) preincdec = :dec; - else if (parser.accept("++")?) preincdec = :inc; + if (parser.acceptToken2(TokenType.minus, TokenType.minus)) preincdec = :dec; + else if (parser.acceptToken2(TokenType.plus, TokenType.plus)) preincdec = :inc; auto toPre = parser.to(fromPre); /** @@ -1423,8 +1429,8 @@ class ASTPropagateFailureExpr : ASTSymbol break; } auto fromPost = parser.from?; - if (parser.accept("--")?) postincdec = :dec; - else if (parser.accept("++")?) postincdec = :inc; + if (parser.acceptToken2(TokenType.minus, TokenType.minus)) postincdec = :dec; + else if (parser.acceptToken2(TokenType.plus, TokenType.plus)) postincdec = :inc; auto toPost = parser.to(fromPost); if (preincdec != :none && postincdec != :none) return toPost.fail("cannot have both pre- and post-increment operator"); @@ -1619,9 +1625,9 @@ class ASTLambdaExpr : ASTSymbol parser.begin; auto from = parser.from?; mut string[] args; - if (parser.accept("(")?) { - while (!parser.accept(")")?) { - if (!args.empty && !parser.accept(",")?) { + if (parser.acceptToken(TokenType.lparen)) { + while (!parser.acceptToken(TokenType.rparen)) { + if (!args.empty && !parser.acceptToken(TokenType.comma)) { parser.revert; return null; } @@ -1640,7 +1646,7 @@ class ASTLambdaExpr : ASTSymbol } args ~= arg; } - if (!parser.accept("=>")?) { + if (!parser.acceptToken2(TokenType.equal, TokenType.greater)) { parser.revert; return null; } @@ -1657,14 +1663,14 @@ class ASTLambdaExpr : ASTSymbol // Leave it in anyway - otherwise, you can cause stack overflow by // just repeatedly spamming * or &. parser.begin; - if (parser.accept("*")?) { + if (parser.acceptToken(TokenType.asterisk)) { nullable ASTSymbol next = parseExpressionLeaf(parser, lexicalContext)?; parser.to(from).assert(!!next, "expression expected")?; parser.commit; return new ASTDereference(next.notNull, parser.to(from)); } - if (parser.accept("&")?) { + if (parser.acceptToken(TokenType.ampersand)) { nullable ASTSymbol next = parseExpressionLeaf(parser, lexicalContext)?; parser.to(from).assert(!!next, "expression expected")?; parser.commit; @@ -1677,7 +1683,7 @@ class ASTLambdaExpr : ASTSymbol parser.commit; mut ASTArgument[] args; - if (parser.accept("(")?) + if (parser.acceptToken(TokenType.lparen)) { args = parseArgumentList(parser, lexicalContext)?; } @@ -1686,25 +1692,25 @@ class ASTLambdaExpr : ASTSymbol } if (parser.acceptIdentifier("sizeof")?) { parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; parser.begin; mut nullable ASTSymbol sym = lexicalContext.compiler.parseType(parser, lexicalContext)?; - if (!sym || !parser.accept(")")?) { + if (!sym || !parser.acceptToken(TokenType.rparen)) { parser.revert; parser.begin; sym = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; parser.to(from).assert(!!sym, "expression expected")?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; } parser.commit; return new ASTSizeOf(sym.notNull); } if (parser.acceptIdentifier("cast")?) { - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; nullable ASTSymbol target = parser.parseType(lexicalContext)?; parser.to(from).assert(!!target, "expression expected")?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; nullable ASTSymbol value = parseExpressionLeaf(parser, lexicalContext)?; parser.to(from).assert(!!value, "expression expected")?; parser.commit; @@ -1733,14 +1739,14 @@ class ASTLambdaExpr : ASTSymbol return ret; } parser.begin; - if (parser.accept("!")?) { + if (parser.acceptToken(TokenType.exclamationmark)) { nullable ASTSymbol next = parser.parseExpressionLeaf(lexicalContext)?; parser.to(from).assert(!!next, "expression expected")?; parser.commit; return new ASTNegation(next.notNull, parser.to(from)); } - if (parser.acceptButNot("-", "--")?) { + if (parser.acceptToken2Not(TokenType.minus, TokenType.minus)) { nullable ASTSymbol next = parser.parseExpressionLeaf(lexicalContext)?; parser.to(from).assert(!!next, "expression expected")?; parser.commit; @@ -1753,106 +1759,114 @@ class ASTLambdaExpr : ASTSymbol return null; } -(char | :dollar | fail Error) parseChar(Parser parser) { - string ch = parser.text[0 .. 1]; - parser.drop(1); - if (ch != "\\") { - if (ch == "\$") return :dollar; - return ch[0]; +(string text, int size | fail :error, string) processChar(string text) { + assert(!text.empty); + if (text.front != "\\"[0]) { + return (text[0 .. 1], 1); } - if (parser.text.empty) - return parser.fail("expected control character, got end of file"); - string ctl = parser.text[0 .. 1]; - parser.drop(1); + if (text.length == 1) + return (:error, "expected control character"); + string ctl = text[1 .. 2]; // TODO use char literals if (ctl == "r") { - return "\r"[0]; + return ("\r", 2); } else if (ctl == "\$") { - return "\$"[0]; + return ("\$", 2); } else if (ctl == "n") { - return "\n"[0]; + return ("\n", 2); } else if (ctl == "t") { - return "\t"[0]; + return ("\t", 2); } else if (ctl == "\"") { - return "\""[0]; + return ("\"", 2); } else if (ctl == "'") { - return "'"[0]; + return ("'", 2); } else if (ctl == "\\") { - return "\\"[0]; + return ("\\", 2); } else if (ctl == "0") { - return "\x00"[0]; // TODO \0 + return ("\x00", 2); // TODO \0 } else if (ctl == "x") { - if (parser.text.length < 2) return parser.fail("expected hex code, got end of file"); - char ch1 = parser.text[0], ch2 = parser.text[1]; - int decodeHex(char ch) { + if (text.length < 4) return (:error, "expected two-digit hex code"); + char ch1 = text[2], ch2 = text[3]; + (int | fail :error, string) decodeHex(char ch) { if (ch >= "0"[0] && ch <= "9"[0]) return ch - "0"[0]; if (ch >= "a"[0] && ch <= "f"[0]) return ch + 10 - "a"[0]; if (ch >= "A"[0] && ch <= "F"[0]) return ch + 10 - "A"[0]; - print("Unknown hex digit '" ~ ch ~ "'"); - assert(false); + return (:error, "unknown hex digit '" ~ ch ~ "'"); } - char ord = cast(char) (decodeHex(ch1) * 16 + decodeHex(ch2)); - parser.drop(2); - return ord; + return ("" ~ cast(char) (decodeHex(ch1)? * 16 + decodeHex(ch2)?), 4); } else { - return parser.fail("unknown control sequence \\$ctl"); + return (:error, "unknown control sequence \\$ctl"); } } +(string | (:error, string msg)) cleanup(mut string s) { + mut string ret; + while (!s.empty) { + processChar(s).case { + (:error, string msg): + return (:error, msg); + (string text, int size): + ret ~= text; + s = s[size .. $]; + } + } + return ret; +} + (ASTSymbol | fail Error) parseCharLiteral( - Parser parser, LexicalContext lexicalContext, LocRange from) + Parser parser, LexicalContext lexicalContext) { - if (parser.text.empty) + if (parser.hard_eof) return parser.fail("expected character, got end of file"); - auto ch = parser.parseChar?.case(:dollar: "\$"[0]); + + auto from = parser.from?; + auto res = parser.acceptTokenStr(TokenType.charLiteral) + .case(:none: return parser.fail("character expected")) + .processChar + .case((:error, string msg): return parser.fail(msg)); + if (!parser.acceptToken(TokenType.singleQuote)) + return parser.fail("closing quote expected"); auto locRange = parser.to(from); - parser.expect("'")?; - return new ASTCharLiteral(ch, locRange); + return new ASTCharLiteral(res.text.front, locRange); } (ASTSymbol | fail Error) parseStringLiteral( - Parser parser, LexicalContext lexicalContext, string endMarker, LocRange from) + Parser parser, LexicalContext lexicalContext) { - mut string current; + auto from = parser.from?; + if (parser.acceptToken(TokenType.doubleQuote)) + return new ASTStringLiteral("", parser.to(from)); + mut string str = parser.acceptTokenStr(TokenType.stringLiteral) + .case(:none: return parser.fail("lexer error: string literal should be here")); mut ASTSymbol[] parts; - void flush() { - if (current != "") parts ~= new ASTStringLiteral(current, parser.to(from)); - current = ""; - } - while (parser.text.length < endMarker.length || parser.text[0 .. endMarker.length] != endMarker) - { - if (parser.text.length == 0) { - return parser.fail("expected end of string, got end of file"); - } - auto ch = parser.parseChar?; - if (ch == :dollar) { - flush; - if (parser.accept("(")?) { - auto litFrom = parser.from?; - nullable ASTSymbol expr = parser.parseExpression(lexicalContext)?; - parser.to(litFrom).assert(!!expr, "Expression expected.")?; - parser.expect(")")?; - parts ~= expr.notNull; - } else { - auto varFrom = parser.from?; - auto var = parser.parseIdentifier?; - parser.to(varFrom).assert(var != "", "Identifier expected.")?; - parts ~= new ASTIdentifier(var, moduleLevel=false, parser.to(varFrom)); - } + parts ~= new ASTStringLiteral( + str.cleanup.case((:error, string msg): return parser.fail(msg)), parser.to(from)); + while (parser.acceptToken(TokenType.formatQuoteStart)) { + if (parser.acceptToken(TokenType.lparen)) { + auto litFrom = parser.from?; + nullable ASTSymbol expr = parser.parseExpression(lexicalContext)?; + parser.to(litFrom).assert(!!expr, "Expression expected.")?; + if (!parser.acceptToken(TokenType.rparen)) + return parser.fail("closing ')' expected"); + parts ~= expr.notNull; } else { - bottom die() { assert(false); } - current ~= ch.case(:dollar: die); + auto varFrom = parser.from?; + auto var = parser.parseIdentifier?; + parser.to(varFrom).assert(var != "", "Identifier expected.")?; + parts ~= new ASTIdentifier(var, moduleLevel=false, parser.to(varFrom)); } + auto from = parser.from?; + mut string str = parser.acceptTokenStr(TokenType.stringLiteral) + .case(:none: continue); + parts ~= new ASTStringLiteral( + str.cleanup.case((:error, string msg): return parser.fail(msg)), parser.to(from)); } - if (!parser.accept(endMarker)?) - { - return parser.fail("this should never happen"); - } - flush; - if (parts.length == 0) return new ASTStringLiteral("", parser.to(from)); + if (!parser.acceptToken(TokenType.doubleQuote)) + return parser.fail("closing quote expected"); + if (parts.length == 1) { if (auto literal = parts[0].instanceOf(ASTStringLiteral)) return literal; @@ -1866,16 +1880,16 @@ with (parser.transaction) { auto from = parser.from?; mut string op; - if (parser.accept("__borrow")?) op = "borrow"; - else if (parser.accept("__take")?) op = "take"; - else if (parser.accept("__leak")?) op = "leak"; + if (parser.acceptIdentifier("__borrow")?) op = "borrow"; + else if (parser.acceptIdentifier("__take")?) op = "take"; + else if (parser.acceptIdentifier("__leak")?) op = "leak"; else return null; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto fromSource = parser.from?; auto source = parser.parseExpression(lexicalContext)?; parser.to(fromSource).assert(!!source, "expression expected")?; auto source = source.notNull; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; commit; if (op == "borrow") return new ASTOverrideLifetime(source, Lifetime.lexical, parser.to(from)); @@ -1957,10 +1971,10 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) } if (auto destroy_ = parseDestroy(parser, lexicalContext)?) return destroy_; parser.begin; - parser.strip?; + parser.strip; auto loc = parser.loc; auto from = parser.from?; - if (parser.accept(".")?) { + if (parser.acceptToken(TokenType.dot)) { // .name is module level string name = parser.parseIdentifier?; if (name.length) { @@ -1979,7 +1993,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) } { parser.begin; - if (parser.accept(":")?) { + if (parser.acceptToken(TokenType.colon)) { string symbolIdentifier = parser.parseIdentifier?; if (symbolIdentifier != "") { parser.commit; @@ -2007,18 +2021,18 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) } return new ASTNumberLiteral(l, parser.to(from)); } - if (parser.accept("'")?) { - return parseCharLiteral(parser, lexicalContext, from); + if (parser.acceptToken(TokenType.singleQuote)) { + return parseCharLiteral(parser, lexicalContext); } - if (parser.accept("\"")?) { - return parseStringLiteral(parser, lexicalContext, "\"", from); + if (parser.acceptToken(TokenType.doubleQuote)) { + return parseStringLiteral(parser, lexicalContext); } if (auto statementExpr = parseStatementExpr(parser, lexicalContext)?) return statementExpr; - if (parser.accept("(")?) { + if (parser.acceptToken(TokenType.lparen)) { nullable ASTSymbol result = parseExpression(parser, lexicalContext)?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; return result.notNull; } if (auto result = parseArrayLiteral(parser, lexicalContext)?) @@ -2033,7 +2047,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) { parser.begin; auto from = parser.from?; - if (parser.acceptButNot("&", "&&")?) + if (parser.acceptToken2Not(TokenType.ampersand, TokenType.ampersand)) { parser.commit; ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2054,7 +2068,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) { parser.begin; auto from = parser.from?; - if (parser.accept("^")?) + if (parser.acceptToken(TokenType.circumflex)) { parser.commit; ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2075,7 +2089,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) { parser.begin; auto from = parser.from?; - if (parser.acceptButNot("|", "||")?) + if (parser.acceptToken2Not(TokenType.bar, TokenType.bar)) { parser.commit; ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2095,19 +2109,19 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) while (true) { auto from = parser.from?; - if (parser.accept("<<")?) + if (parser.acceptToken2(TokenType.smaller, TokenType.smaller)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp("<<", left, right, parser.to(from)); } - else if (parser.accept(">>>")?) + else if (parser.acceptToken3(TokenType.greater, TokenType.greater, TokenType.greater)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp(">>>", left, right, parser.to(from)); } - else if (parser.accept(">>")?) + else if (parser.acceptToken2(TokenType.greater, TokenType.greater)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2122,19 +2136,19 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) while (true) { auto from = parser.from?; - if (parser.acceptButNot("+", "++")?) + if (parser.acceptToken2Not(TokenType.plus, TokenType.plus)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp("+", left, right, parser.to(from)); } - else if (parser.acceptButNot("-", "--")?) + else if (parser.acceptToken2Not(TokenType.minus, TokenType.minus)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp("-", left, right, parser.to(from)); } - else if (parser.accept("~")?) + else if (parser.acceptToken(TokenType.tilde)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2149,19 +2163,19 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) while (true) { auto from = parser.from?; - if (parser.accept("*")?) + if (parser.acceptToken(TokenType.asterisk)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp("*", left, right, parser.to(from)); } - else if (parser.accept("/")?) + else if (parser.acceptToken(TokenType.slash)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; left = new ASTBinaryOp("/", left, right, parser.to(from)); } - else if (parser.accept("%")?) + else if (parser.acceptToken(TokenType.percent)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2176,7 +2190,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) while (true) { auto from = parser.from?; - if (parser.accept("&&")?) + if (parser.acceptToken2(TokenType.ampersand, TokenType.ampersand)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2190,7 +2204,7 @@ ASTSymbol locRangeExpr(LocRange locRange, LexicalContext lexicalContext) while (true) { auto from = parser.from?; - if (parser.accept("||")?) + if (parser.acceptToken2(TokenType.bar, TokenType.bar)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2241,7 +2255,7 @@ class ASTIota : ASTSymbol (ASTSymbol | fail Error) parseIota(Parser parser, LexicalContext lexicalContext, ASTSymbol left, int myLevel) { auto from = parser.from?; - if (parser.accept("..")?) + if (parser.acceptToken2(TokenType.dot, TokenType.dot)) { ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2253,56 +2267,59 @@ class ASTIota : ASTSymbol (ASTSymbol | fail Error) parseComparison(Parser parser, LexicalContext lexicalContext, ASTSymbol left, int myLevel) { auto from = parser.from?; - if (parser.accept("==")?) + if (parser.acceptToken2(TokenType.equal, TokenType.equal)) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTBinaryOp("==", left, right, locRange); } - if (parser.accept("!=")?) // same as !(a == b) + if (parser.acceptToken2(TokenType.exclamationmark, TokenType.equal)) // same as !(a == b) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTNegation(new ASTBinaryOp("==", left, right, locRange), locRange); } - if (parser.accept("is")?) + if (parser.acceptIdentifier("is")?) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTBinaryOp("is", left, right, locRange); } - if (parser.accept("!is")?) - { - auto locRange = parser.to(from); - ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; + with (parser.transaction) { + if (parser.acceptToken(TokenType.exclamationmark) && parser.acceptIdentifier("is")?) + { + commit; + auto locRange = parser.to(from); + ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; - return new ASTNegation(new ASTBinaryOp("is", left, right, locRange), locRange); + return new ASTNegation(new ASTBinaryOp("is", left, right, locRange), locRange); + } } - if (parser.accept(">=")?) + if (parser.acceptToken2(TokenType.greater, TokenType.equal)) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTBinaryOp(">=", left, right, locRange); } - if (parser.acceptButNot(">", ">>")?) + if (parser.acceptToken2Not(TokenType.greater, TokenType.greater)) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTBinaryOp(">", left, right, locRange); } - if (parser.accept("<=")?) + if (parser.acceptToken2(TokenType.smaller, TokenType.equal)) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; return new ASTBinaryOp("<=", left, right, locRange); } - if (parser.acceptButNot("<", "<<")?) + if (parser.acceptToken2Not(TokenType.smaller, TokenType.smaller)) { auto locRange = parser.to(from); ASTSymbol right = expectArithmetic(parser, lexicalContext, myLevel + 1)?; @@ -2342,10 +2359,11 @@ class ASTTernaryIf : ASTSymbol (ASTSymbol | fail Error) parseTernaryOp(Parser parser, LexicalContext lexicalContext, mut ASTSymbol then, int myLevel) { with (parser.transaction) { auto from = parser.from?; - if (!parser.accept("if")?) return then; + if (!parser.acceptIdentifier("if")?) return then; auto cond = parseExpression(parser, lexicalContext)?; if (!cond) return then; - parser.expect("else")?; + if (!parser.acceptIdentifier("else")?) + return parser.fail("'else' expected"); auto else_ = expectArithmetic(parser, lexicalContext, 0)?; commit; return new ASTTernaryIf(cond.notNull, then, else_, parser.to(from)); @@ -2379,8 +2397,9 @@ class ASTTernaryIf : ASTSymbol (nullable ASTSymbol | fail Error) parseExpression(Parser parser, LexicalContext lexicalContext) { - if (auto leaf = parseExpressionLeaf(parser, lexicalContext)?) + if (auto leaf = parseExpressionLeaf(parser, lexicalContext)?) { return parseArithmetic(parser, lexicalContext, leaf, 0); + } return null; } @@ -2452,7 +2471,11 @@ class ASTVoidExpression : ASTSymbol // peek parser.begin; // TODO acceptExprTerminator - if (parser.accept(";")? || parser.accept(",")? || parser.accept(")")?) { // return; + if (parser.acceptToken(TokenType.semicolon) + || parser.acceptToken(TokenType.comma) + || parser.acceptToken(TokenType.rparen)) + { + // return; parser.revert; expr = new ASTVoidExpression(parser.to(from)); } else { @@ -2695,7 +2718,7 @@ class ASTIfStatement : ASTStatement } mut uninitialized (ASTSymbol | ASTVarDeclStatement | ASTVarExtractDeclStatement) test; auto locRange = parser.to(from); - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto vardecl = parseVarDecl(parser, lexicalContext, requireInitializer=true)?; if (auto vardecl = vardecl) test = vardecl; @@ -2706,10 +2729,10 @@ class ASTIfStatement : ASTStatement test = parseExpression(parser, lexicalContext)?.notNull; } } - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; ASTStatement thenStmt = parseStatement(parser, lexicalContext)?; mut nullable ASTStatement elseStatement; - if (parser.accept("else")?) + if (parser.acceptIdentifier("else")?) { elseStatement = parseStatement(parser, lexicalContext)?; } @@ -2802,17 +2825,17 @@ class ASTSequenceStatement : ASTStatement // This is not necessary, but ensures we get an error instead of segfault if we just spam brackets. parser.begin; auto from = parser.from?; - if (!parser.accept("{")?) { + if (!parser.acceptToken(TokenType.lcurlybracket)) { parser.revert; return null; } mut ASTStatement[] statements; - while (!parser.accept("}")?) + while (!parser.acceptToken(TokenType.rcurlybracket)) { (void | fail Error) appendStmt() { statements ~= parseStatement(parser, lexicalContext)?; } - parser.verifyTransactions("statement parse rule imbalance", &appendStmt)?; + parser.verifyTransactions("scope statement parse rule imbalance", &appendStmt)?; } parser.commit; return new ASTScopeStatement(statements, parser.to(from)); @@ -2829,17 +2852,27 @@ class ASTSequenceStatement : ASTStatement } auto from = parser.from?; mut string operator; - if (parser.accept("=")?) { } - else if (parser.accept("+=")?) operator = "+"; - else if (parser.accept("-=")?) operator = "-"; - else if (parser.accept("*=")?) operator = "*"; - else if (parser.accept("/=")?) operator = "/"; - else if (parser.accept("~=")?) operator = "~"; - else if (parser.accept("&=")?) operator = "&"; - else if (parser.accept("|=")?) operator = "|"; - else if (parser.accept("^=")?) operator = "^"; - else if (parser.accept("&&=")?) operator = "&&"; - else if (parser.accept("||=")?) operator = "||"; + if (parser.acceptToken(TokenType.equal)) { } + else if (parser.acceptToken2(TokenType.plus, TokenType.equal)) + operator = "+"; + else if (parser.acceptToken2(TokenType.minus, TokenType.equal)) + operator = "-"; + else if (parser.acceptToken2(TokenType.asterisk, TokenType.equal)) + operator = "*"; + else if (parser.acceptToken2(TokenType.slash, TokenType.equal)) + operator = "/"; + else if (parser.acceptToken2(TokenType.tilde, TokenType.equal)) + operator = "~"; + else if (parser.acceptToken2(TokenType.ampersand, TokenType.equal)) + operator = "&"; + else if (parser.acceptToken2(TokenType.bar, TokenType.equal)) + operator = "|"; + else if (parser.acceptToken2(TokenType.circumflex, TokenType.equal)) + operator = "^"; + else if (parser.acceptToken3(TokenType.ampersand, TokenType.ampersand, TokenType.equal)) + operator = "&&"; + else if (parser.acceptToken3(TokenType.bar, TokenType.bar, TokenType.equal)) + operator = "||"; else { parser.revert; return null; @@ -3131,8 +3164,13 @@ class ASTVarDeclStatement : ASTStatement mut nullable ASTSymbol initial; auto from = parser.from?; mut string name; + bool accept() { + if (operator == "=") return parser.acceptToken(TokenType.equal); + if (operator == "<-") return parser.acceptToken2(TokenType.smaller, TokenType.minus); + assert(false); + } // <- is not as ambiguous as = - if ((infer || operator == "<-") && parser.accept(operator)?) + if ((infer || operator == "<-") && accept()) { initial = parseExpression(parser, lexicalContext)?; if (!initial) return parser.fail("variable initializer expected"); @@ -3146,7 +3184,7 @@ class ASTVarDeclStatement : ASTStatement return null; } if (infer) { - if (!parser.accept(operator)?) + if (!accept()) { parser.revert; return null; @@ -3154,12 +3192,12 @@ class ASTVarDeclStatement : ASTStatement initial = parseExpression(parser, lexicalContext)?; if (!initial) return parser.fail("variable initializer expected"); } - else if (parser.accept(operator)?) { + else if (accept()) { initial = parseExpression(parser, lexicalContext)?; if (!initial) return parser.fail("variable initializer expected"); } else { // FIXME this is very hacky - if (operator == "=" && parser.accept("<-")?) { + if (operator == "=" && parser.acceptToken2(TokenType.smaller, TokenType.minus)) { // recognize that we're failing to parse a <- instead of == // FIXME check for both here instead of passing a parameter parser.revert; @@ -3232,7 +3270,7 @@ class ASTVarDeclStatement : ASTStatement } mut ASTStatement[] decls; while (true) { - if (decls.length) parser.expect(",")?; + if (decls.length) parser.expectToken(TokenType.comma)?; auto decl = parseVarInitialization(parser, lexicalContext, infer, mutable, uninitialized, type, "=", false)?; if (auto decl = decl) { decls ~= decl; @@ -3248,7 +3286,7 @@ class ASTVarDeclStatement : ASTStatement return null; } } - if (parser.accept(";")?) break; + if (parser.acceptToken(TokenType.semicolon)) break; } parser.commit; return new ASTSequenceStatement(decls, parser.to(from)); @@ -3488,7 +3526,7 @@ class ASTNestedFunctionDeclExpr : ASTSymbol Parser parser, LexicalContext lexicalContext) { with (parser.transaction) { - if (!parser.accept("(")?) { + if (!parser.acceptToken(TokenType.lparen)) { return null; } auto from = parser.from?; @@ -3496,7 +3534,9 @@ class ASTNestedFunctionDeclExpr : ASTSymbol .case(Error err: return null); // Extremely dirty hack. - if (parser.accept(",")? || parser.accept(")")? || parser.accept(";")?) { + if (parser.acceptToken(TokenType.comma) || parser.acceptToken(TokenType.rparen) + || parser.acceptToken(TokenType.semicolon)) + { return null; } commit; @@ -3551,9 +3591,9 @@ class ASTWhile : ASTStatement parser.revert; return null; } - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; nullable ASTSymbol cond = parseExpression(parser, lexicalContext)?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; parser.commit; ASTStatement body_ = parseStatement(parser, lexicalContext)?; @@ -3622,10 +3662,13 @@ class ASTStatementExpression : ASTSymbol { parser.begin; auto from = parser.from?; - if (!parser.accept("({")?) { parser.revert; return null; } + if (!parser.acceptToken2(TokenType.lparen, TokenType.lcurlybracket)) { + parser.revert; + return null; + } parser.commit; mut ASTStatement[] statements; - while (!parser.accept("})")?) { + while (!parser.acceptToken2(TokenType.rcurlybracket, TokenType.rparen)) { (void | fail Error) appendStatement() { statements ~= parseStatement(parser, lexicalContext)?; } @@ -3686,16 +3729,16 @@ class ASTForLoop : ASTStatement return null; } parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; nullable ASTVarDeclStatement varDecl = parseVarDecl(parser, lexicalContext, requireInitializer=false)?; if (!varDecl) { return parser.fail("loop declaration expected"); } - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; nullable ASTSymbol condition = parseExpression(parser, lexicalContext)?; - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; nullable ASTStatement step = parseForStep(parser, lexicalContext)?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; ASTStatement body_ = parseStatement(parser, lexicalContext)?; return new ASTForLoop(varDecl.notNull, condition.notNull, step.notNull, body_, parser.to(from)); @@ -3730,7 +3773,7 @@ class ASTStatementStatement : ASTStatement parser.commit; auto variable = parser.parseIdentifier?; if (!variable.length) return parser.fail("ASTStatement variable expected"); - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return new ASTStatementStatement(variable, parser.to(from)); } @@ -3752,7 +3795,10 @@ class ASTStatementStatement : ASTStatement auto ident = parser.parseIdentifier?; parser.begin; // peek ahead - if (parser.accept(",")? || parser.acceptIdentifier("in")? || parser.accept("<-")?) { + if (parser.acceptToken(TokenType.comma) + || parser.acceptIdentifier("in")? + || parser.acceptToken2(TokenType.smaller, TokenType.minus)) + { // (i in 0..10) parser.revert; // revert peek parser.commit; // commit identifier @@ -3884,7 +3930,7 @@ class ASTExtForLoop : ASTStatement { parser.begin; auto from = parser.from?; - if (!parser.acceptIdentifier("for")? || !parser.accept("(")?) { + if (!parser.acceptIdentifier("for")? || !parser.acceptToken(TokenType.lparen)) { parser.revert; return null; } @@ -3904,7 +3950,7 @@ class ASTExtForLoop : ASTStatement } mut bool keyValueLoop = false; mut uninitialized ExtForVarDecl match2decl; - if (parser.accept(",")?) { + if (parser.acceptToken(TokenType.comma)) { parseExtForVarDecl(parser, lexicalContext).case { (:fail): { parser.revert; @@ -3917,7 +3963,7 @@ class ASTExtForLoop : ASTStatement } keyValueLoop = true; } - if (parser.accept("<-")?) { + if (parser.acceptToken2(TokenType.smaller, TokenType.minus)) { print("warn: $(parser.to(from).repr): use 'in'"); } else if (!parser.acceptIdentifier("in")?) { parser.revert; @@ -3927,7 +3973,7 @@ class ASTExtForLoop : ASTStatement if (!source) return parser.fail("expression expected"); auto source = source.notNull; // array loop, for (i, k in array) - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; auto body_ = parseStatement(parser, lexicalContext)?; parser.commit; @@ -3964,10 +4010,10 @@ class ASTDestroy : ASTSymbol parser.revert; return null; } - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; nullable ASTSymbol target = parseExpression(parser, lexicalContext)?; if (!target) return parser.fail("expression expected"); - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; parser.commit; return new ASTDestroy(target.notNull, parser.to(from)); @@ -4009,18 +4055,18 @@ class ASTStaticIfStatement : ASTStatement mut nullable ASTSymbol cond; if (parser.acceptIdentifier("static")? && parser.acceptIdentifier("if")?) { parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto e = parser.parseExpression(lexicalContext)?; if (!e) { return parser.fail("static if condition expected"); } cond = e; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; } else if (parser.acceptIdentifier("version")?) { parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; string versionStr = parser.parseIdentifier?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; auto versionSym = lexicalContext.compiler.astStringLiteral(versionStr, __RANGE__); cond = new ASTTrait("versionSet", [versionSym], __RANGE__); } else { @@ -4029,16 +4075,16 @@ class ASTStaticIfStatement : ASTStatement } mut ASTStatement[] then, else_; - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { then ~= parser.parseStatement(lexicalContext)?; } } else { then ~= parser.parseStatement(lexicalContext)?; } - if (parser.accept("else")?) { - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + if (parser.acceptIdentifier("else")?) { + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { else_ ~= parser.parseStatement(lexicalContext)?; } } else { @@ -4094,8 +4140,9 @@ class ASTStaticIfStatement : ASTStatement { if (auto stmt = parseAssignment(parser, lexicalContext)?) return stmt; - if (auto expr = parseExpression(parser, lexicalContext)?) + if (auto expr = parseExpression(parser, lexicalContext)?) { return new ASTExprStatement(expr, expr.locRange); + } return null; } @@ -4131,7 +4178,7 @@ class ASTStaticIfStatement : ASTStatement if (ASTStatement ret = args.statement?) return ret; } if (ASTStatement stmt = parseForStep(parser, lexicalContext)?) { - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return stmt; } return parser.fail("statement expected"); @@ -4648,21 +4695,22 @@ class ASTExternFunction : ASTExternFunctionBase { parser.begin; auto from = parser.from?; - if (!parser.accept("extern")?) + if (!parser.acceptIdentifier("extern")?) { parser.revert; return :none; } parser.commit; - parser.expect("(")?; - parser.expect("C")?; - parser.expect(")")?; + parser.expectToken(TokenType.lparen)?; + if (parser.parseIdentifier? != "C") + return parser.fail("calling convention expected"); + parser.expectToken(TokenType.rparen)?; auto retFrom = parser.from?; auto type = parser.parseType(lexicalContext)?.notNull; auto retLocRange = parser.to(retFrom); string name = parser.parseIdentifier?; if (!name.length) return parser.fail("identifier expected"); - if (parser.accept("(")?) { + if (parser.acceptToken(TokenType.lparen)) { auto paramList = parseParameterList(parser, lexicalContext, variadic=true, thisAssignment=false)?; ASTParameter pick((ASTParameter | ASTThisAssignment) value) { value.case { @@ -4671,10 +4719,10 @@ class ASTExternFunction : ASTExternFunctionBase } } auto params = [pick(entry) for entry in paramList.params]; - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return new ASTExternFunction(name, type, params=params, variadic=paramList.variadic, retLocRange); } else { - parser.accept(";")?; + parser.expectToken(TokenType.semicolon)?; return new ASTExternVariable(type, name, parser.to(from)); } } @@ -4904,10 +4952,10 @@ class ImportTask : ITask mut string pak; if (parser.acceptIdentifier("package")?) { - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; pak = parser.parseIdentifier?; - parser.expect(")")?; - parser.expect(".")?; + parser.expectToken(TokenType.rparen)?; + parser.expectToken(TokenType.dot)?; } string modname = parser.parseModuleName?; @@ -4915,7 +4963,7 @@ class ImportTask : ITask } mut string[] symbols; - if (!isMacroImport && parser.accept(":")?) { + if (!isMacroImport && parser.acceptToken(TokenType.colon)) { (string | fail Error) expectIdentifier(Parser parser) { auto ident = parser.parseIdentifier?; parser.assert_(ident.length > 0, "identifier expected")?; @@ -4929,12 +4977,12 @@ class ImportTask : ITask } // : [symbol] [, symbol]* ,? ; symbols ~= parser.expectIdentifier?; - while (parser.accept(",")?) { + while (parser.acceptToken(TokenType.comma)) { if (parser.peek(";")?) { break; } symbols ~= parser.expectIdentifier?; } } - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; return new ASTImportStatement(target, isMacroImport, symbols, parser.to(from)); } @@ -4942,7 +4990,7 @@ class ImportTask : ITask mut string ret; while (true) { ret ~= parser.parseIdentifier?; - if (!parser.accept(".")?) break; + if (!parser.acceptToken(TokenType.dot)) break; ret ~= "."; } return ret; @@ -4962,15 +5010,15 @@ class ModuleParserConfig { parser.begin; auto from = parser.from?; - if (!parser.acceptIdentifier("macro")? || !parser.accept("(")?) + if (!parser.acceptIdentifier("macro")? || !parser.acceptToken(TokenType.lparen)) { parser.revert; return null; } parser.commit; string identifier = parser.parseIdentifier?; - parser.expect(")")?; - parser.expect(";")?; + parser.expectToken(TokenType.rparen)?; + parser.expectToken(TokenType.semicolon)?; auto newMacroState = astModule.macroState.dup; @@ -5006,21 +5054,21 @@ class LexicalContextImpl : LexicalContext (nullable ASTVersion | fail Error) parseModuleVersion(Parser parser, LexicalContext lexicalContext) { - if (parser.accept("version")?) { + if (parser.acceptIdentifier("version")?) { mut ASTModuleEntry[] then, else_; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; string versionStr = parser.parseIdentifier?; - parser.expect(")")?; - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + parser.expectToken(TokenType.rparen)?; + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { then ~= parser.parseModuleEntry(lexicalContext, false)?; } } else { then ~= parser.parseModuleEntry(lexicalContext, false)?; } - if (parser.accept("else")?) { - if (parser.accept("{")?) { - while (!parser.accept("}")?) { + if (parser.acceptIdentifier("else")?) { + if (parser.acceptToken(TokenType.lcurlybracket)) { + while (!parser.acceptToken(TokenType.rcurlybracket)) { else_ ~= parser.parseModuleEntry(lexicalContext, false)?; } } else { @@ -5037,9 +5085,9 @@ class LexicalContextImpl : LexicalContext { parser.begin; mut Protection protection = Protection.private_; - if (parser.accept("public")?) { + if (parser.acceptIdentifier("public")?) { protection = Protection.public_; - } else if (parser.accept("private")?) { + } else if (parser.acceptIdentifier("private")?) { protection = Protection.private_; } @@ -5077,9 +5125,9 @@ class LexicalContextImpl : LexicalContext } mut (Protection | :none) protection = :none; - if (parser.accept("public")?) { + if (parser.acceptIdentifier("public")?) { protection = Protection.public_; - } else if (parser.accept("private")?) { + } else if (parser.acceptIdentifier("private")?) { protection = Protection.private_; } Protection getProtection() { @@ -5361,12 +5409,12 @@ class ParseAstModuleTask : ITask // do not count references on the file source - it will end // up with lots of small inc/decs to fragments that we never free anyway (cast(size_t*) &code)[2] = 0; - Parser parser = new ParserImpl(path, code); + Parser parser = new ParserImpl(path, code, tokenize(code, path)?); auto fromAtModule = parser.from?; parser.expect("module")?; string modname = parser.parseModuleName?; - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; import std.string : endsWith; diff --git a/src/neat/templ.nt b/src/neat/templ.nt index efb86ab5..e19dfea9 100644 --- a/src/neat/templ.nt +++ b/src/neat/templ.nt @@ -16,7 +16,10 @@ import std.thread; parser.begin; auto instanceFrom = parser.from?; // TODO better way to exclude !operator cases - if (!parser.accept("!")? || parser.accept("=")? || parser.acceptIdentifier("is")?) { + if (!parser.acceptToken(TokenType.exclamationmark) + || parser.acceptToken(TokenType.equal) + || parser.acceptIdentifier("is")?) + { parser.revert; return null; } @@ -36,12 +39,12 @@ import std.thread; } } - if (parser.accept("(")?) { + if (parser.acceptToken(TokenType.lparen)) { // ( [a [, a]*]? ) - if (!parser.accept(")")?) { + if (!parser.acceptToken(TokenType.rparen)) { getTemplateArg(false)?; - while (!parser.accept(")")?) { - parser.expect(",")?; + while (!parser.acceptToken(TokenType.rparen)) { + parser.expectToken(TokenType.comma)?; getTemplateArg(false)?; } } @@ -192,6 +195,7 @@ class TemplateDecl : LatentSymbol } with (this.mutex.locked) { + // TODO account for false "loops" from threads somehow for (entry in this.instantiations) { if (entry.types.areSame(parameters)) { entry.value.case { @@ -290,20 +294,20 @@ class ASTTemplateDecl : ASTSymbolDeclaration (nullable ASTTemplateDecl | fail Error) parseTemplateDecl(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; if (!parser.acceptIdentifier("template")?) { parser.revert; return null; } parser.commit; auto name = parser.parseIdentifier?; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; string[] typeParameters = parser.parseTemplateParameters(lexicalContext, swallowErrors=false)? .case(:nothing: ({ assert(false); null; })); - parser.expect("{")?; + parser.expectToken(TokenType.lcurlybracket)?; // TODO generalize (parseSymbol)? mut uninitialized ASTSymbol member; - parser.strip?; + parser.strip; auto contentFrom = parser.from?; nullable ASTSymbolDeclaration member = parseTemplateContents(parser, lexicalContext)? .(that.instanceOf(ASTSymbolDeclaration) if that else null); @@ -311,7 +315,7 @@ class ASTTemplateDecl : ASTSymbolDeclaration return parser.to(contentFrom).fail("template member expected"); } - parser.expect("}")?; + parser.expectToken(TokenType.rcurlybracket)?; return new ASTTemplateDecl(name, typeParameters, member.notNull, comment); } @@ -320,7 +324,7 @@ class ASTTemplateDecl : ASTSymbolDeclaration (nullable ASTSymbolDeclaration | fail Error) delegate() parseInner) { parser.begin; - if (!parser.accept("(")?) { + if (!parser.acceptToken(TokenType.lparen)) { parser.revert; return null; } @@ -346,16 +350,16 @@ class ASTTemplateDecl : ASTSymbolDeclaration // ( [identifier [, identifier]*]? ) auto typeParameter = parser.parseIdentifier?; if (typeParameter.length == 0) { - if (!parser.accept(")")?) { + if (!parser.acceptToken(TokenType.rparen)) { if (swallowErrors) return :nothing; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; } } else { typeParameters ~= typeParameter; - while (!parser.accept(")")?) { - if (!parser.accept(",")?) { + while (!parser.acceptToken(TokenType.rparen)) { + if (!parser.acceptToken(TokenType.comma)) { if (swallowErrors) return :nothing; - parser.expect(",")?; + parser.expectToken(TokenType.comma)?; } auto typeParameter = parser.parseIdentifier?; if (typeParameter.length == 0) { diff --git a/src/neat/traits.nt b/src/neat/traits.nt index 6587aa96..11028a1e 100644 --- a/src/neat/traits.nt +++ b/src/neat/traits.nt @@ -11,19 +11,19 @@ import neat.util; (nullable ASTTrait | fail Error) parseTrait(Parser parser, LexicalContext lexicalContext) { - if (!parser.accept("__traits")?) + if (!parser.acceptIdentifier("__traits")?) return null; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto from = parser.from?; auto trait = parser.parseIdentifier?; auto locRange = parser.to(from); mut ASTSymbol[] args; - while (parser.accept(",")?) { + while (parser.acceptToken(TokenType.comma)) { if (auto arg = lexicalContext.compiler.parseExpression(parser, lexicalContext)?) args ~= arg; else return parser.fail("expected expression argument"); } - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; return new ASTTrait(trait, args, locRange); } diff --git a/src/neat/tuples.nt b/src/neat/tuples.nt index 34cbed6f..e57e3842 100644 --- a/src/neat/tuples.nt +++ b/src/neat/tuples.nt @@ -52,7 +52,7 @@ class ASTTupleExpr : ASTSymbol { parser.begin; auto from = parser.from?; - if (!parser.accept("(")?) { + if (!parser.acceptToken(TokenType.lparen)) { parser.revert; return null; } @@ -69,10 +69,10 @@ class ASTTupleExpr : ASTSymbol { parser.begin; mut (string name, ASTSymbol symbol)[] members; - while (!parser.accept(")")?) + while (!parser.acceptToken(TokenType.rparen)) { auto memberFrom = parser.from?; - if (members.length && !parser.accept(",")?) { + if (members.length && !parser.acceptToken(TokenType.comma)) { // be a bit lenient for (bla bla) if (members.length == 1 && !allowOneSized) { parser.revert; @@ -83,7 +83,7 @@ class ASTTupleExpr : ASTSymbol if (mut auto member = lexicalContext.compiler.parseExpression(parser, lexicalContext)?) { mut string name = ""; if (auto memberIdent = member.instanceOf(ASTIdentifier)) { - if (!memberIdent.moduleLevel && parser.accept("=")?) { + if (!memberIdent.moduleLevel && parser.acceptToken(TokenType.equal)) { name = memberIdent.name; auto member_ = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; if (!member_) diff --git a/src/neat/union_.nt b/src/neat/union_.nt index 5a050710..ffed4c7d 100644 --- a/src/neat/union_.nt +++ b/src/neat/union_.nt @@ -160,7 +160,7 @@ class ASTUnionDecl : ASTDeclaration (nullable ASTUnionDecl | fail Error) parseUnionDecl(Parser parser, LexicalContext lexicalContext) { parser.begin; - auto comment = parser.parseLastComment?; + auto comment = parser.parseLastComment; auto from = parser.from?; if (parser.parseIdentifier? != "union") { parser.revert; @@ -170,8 +170,8 @@ class ASTUnionDecl : ASTDeclaration assert(!!name.length); auto locRange = parser.to(from); mut ASTUnionMember[] members; - parser.expect("{")?; - while (!parser.accept("}")?) { + parser.expectToken(TokenType.lcurlybracket)?; + while (!parser.acceptToken(TokenType.rcurlybracket)) { members ~= parseUnionMember(parser, lexicalContext)?; } parser.commit; @@ -185,7 +185,7 @@ class ASTUnionDecl : ASTDeclaration if (!memberType) return parser.fail("expected member type"); auto memberName = parser.parseIdentifier?; if (!memberName.length) return parser.fail("expected member name"); - parser.expect(";")?; + parser.expectToken(TokenType.semicolon)?; auto locRange = parser.to(from); return (memberName, memberType.notNull); } diff --git a/src/neat/unittest_.nt b/src/neat/unittest_.nt index d3d6b0d4..ecd31e57 100644 --- a/src/neat/unittest_.nt +++ b/src/neat/unittest_.nt @@ -10,7 +10,7 @@ import std.string : startsWith, endsWith; (nullable ASTUnitTest | fail Error) parseUnitTest(Parser parser, LexicalContext lexicalContext) { parser.begin; - parser.strip?; + parser.strip; auto from = parser.from?; if (!parser.accept("unittest")?) { parser.revert; @@ -18,7 +18,7 @@ import std.string : startsWith, endsWith; } parser.commit; auto locRange = parser.to(from); - parser.strip?; + parser.strip; auto text = parser.text; ASTStatement body_ = lexicalContext.compiler.parseStatement(parser, lexicalContext)?; mut auto bodyText = text[0 .. text.length - parser.text.length]; diff --git a/src/neat/with_.nt b/src/neat/with_.nt index aafc4782..3fcfcc89 100644 --- a/src/neat/with_.nt +++ b/src/neat/with_.nt @@ -146,10 +146,10 @@ class ASTWithStatement : ASTStatement return null; } parser.commit; - parser.expect("(")?; + parser.expectToken(TokenType.lparen)?; auto subExpr = lexicalContext.compiler.parseExpression(parser, lexicalContext)?; parser.assert_(!!subExpr, "with expression expected")?; - parser.expect(")")?; + parser.expectToken(TokenType.rparen)?; auto locRange = parser.to(from); auto body_ = lexicalContext.compiler.parseStatement(parser, lexicalContext)?; diff --git a/src/neat/workpool.nt b/src/neat/workpool.nt index 7b782906..9f30e01f 100644 --- a/src/neat/workpool.nt +++ b/src/neat/workpool.nt @@ -19,13 +19,11 @@ class WorkPool : WorkPoolBase ThreadPool pool; - int threads; - Stopwatch total; ThreadLocal!string currentTaskId; - this(this.pool, this.threads) { + this(this.pool) { this.mutex = new Mutex; this.currentTaskId = new ThreadLocal!string; // Represents the "main" thread. diff --git a/src/std/json.nt b/src/std/json.nt index 8de18276..b3e88c03 100644 --- a/src/std/json.nt +++ b/src/std/json.nt @@ -4,6 +4,8 @@ macro import std.macro.assert; macro import std.macro.listcomprehension; import package(compiler).neat.base; +version (firstpass) {} +else import package(compiler).neat.lexer; import package(compiler).neat.parser; import std.string; @@ -19,8 +21,11 @@ struct JSONValue * Parse a JSON value from a string. */ static JSONValue parse(string str) { - auto parser = new ParserImpl("", str); (JSONValue | fail Error) result() { + version (firstpass) + auto parser = new ParserImpl("", str); + else + auto parser = new ParserImpl("", str, tokenize(str, filename="")?); auto ret = jsonParseImpl(parser)?; if (!parser.eof?) return parser.fail("text after json"); return ret; @@ -38,8 +43,12 @@ struct JSONValue import std.file : readText; auto str = readText(file); - auto parser = new ParserImpl(file, str); (JSONValue | fail Error) result() { + version (firstpass) { + auto parser = new ParserImpl(file, str); + } else { + auto parser = new ParserImpl(file, str, tokenize(str, filename=file)?); + } auto ret = jsonParseImpl(parser)?; if (!parser.eof?) return parser.fail("text after json"); return ret; @@ -107,66 +116,153 @@ unittest auto value = JSONValue([("Hello", JSONValue("World"))]); assert(value.str == "{\"Hello\": \"World\"}"); assert(JSONValue.parse("{\"Hello\": \"World\"}").str == "{\"Hello\": \"World\"}"); - // TODO - // auto value = JSONValue({ "Hello": "World" }); } -private (JSONValue | fail Error) jsonParseImpl(ParserImpl parser) { - if (parser.accept("\"")?) { - return JSONValue(parseStringLiteral(parser)?); - } - parser.parseJsonNumber?.case { - (:failure): {} - int value: return JSONValue(value); - } - if (parser.accept("[")?) { - mut JSONValue[] entries; - if (!parser.accept("]")?) while (true) { - entries ~= jsonParseImpl(parser)?; - if (parser.accept("]")?) break; - parser.expect(",")?; +version (firstpass) { + private (JSONValue | fail Error) jsonParseImpl(Parser parser) { + if (parser.accept("\"")?) { + return JSONValue(parseStringLiteral(parser)?); + } + parser.parseJsonNumber?.case { + (:failure): {} + int value: return JSONValue(value); } - return JSONValue(entries); + if (parser.accept("[")?) { + mut JSONValue[] entries; + if (!parser.accept("]")?) while (true) { + entries ~= jsonParseImpl(parser)?; + if (parser.accept("]")?) break; + parser.expect(",")?; + } + return JSONValue(entries); + } + if (parser.accept("{")?) { + mut (string key, JSONValue value)[] entries; + if (!parser.accept("}")?) while (true) { + parser.expect("\"")?; + auto key = parseStringLiteral(parser)?; + parser.expect(":")?; + auto value = jsonParseImpl(parser)?; + entries ~= (key, value); + if (parser.accept("}")?) break; + parser.expect(",")?; + } + return JSONValue(entries); + } + return parser.fail("unexpected input: " ~ parser.text); } - if (parser.accept("{")?) { - mut (string key, JSONValue value)[] entries; - if (!parser.accept("}")?) while (true) { - parser.expect("\"")?; - auto key = parseStringLiteral(parser)?; - parser.expect(":")?; - auto value = jsonParseImpl(parser)?; - entries ~= (key, value); - if (parser.accept("}")?) break; - parser.expect(",")?; + + // reused by the macro + (string | fail Error) parseStringLiteral(Parser parser) + { + mut int matchLen; + auto loc = parser.loc; + string start = parser.text; + while (parser.text[0 .. 1] != "\"") { + if (parser.text.length == 0) { + return parser.fail("expected end of string, got end of file"); + } + if (parser.text[0 .. 1] == "\\") { + matchLen = matchLen + 1; + parser.drop(1); + } + matchLen = matchLen + 1; + parser.drop(1); } - return JSONValue(entries); + string str = start[0 .. matchLen]; + if (!parser.accept("\"")?) { + return parser.fail("this should never happen"); + } + + return replaceEscapes(str); } - return parser.fail("unexpected input: " ~ parser.text); -} -// reused by the macro -(string | fail Error) parseStringLiteral(Parser parser) -{ - mut int matchLen; - auto loc = parser.loc; - string start = parser.text; - while (parser.text[0 .. 1] != "\"") { - if (parser.text.length == 0) { - return parser.fail("expected end of string, got end of file"); + // reused by the macro + (:failure | int | fail Error) parseJsonNumber(Parser parser) + { + parser.begin; + mut bool negative = parser.accept("-")?; + if (parser.accept("-")?) + negative = true; + parser.strip?; + if (parser.hard_eof || !isDigit(parser.text[0])) + { + parser.revert; + return :failure; } - if (parser.text[0 .. 1] == "\\") { - matchLen = matchLen + 1; + mut string number; + while (!parser.hard_eof && isDigit(parser.text[0])) + { + number ~= parser.text[0]; parser.drop(1); } - matchLen = matchLen + 1; - parser.drop(1); + parser.commit; + mut int i = atoi(number); + if (negative) i = -i; + return i; + } +} else { + private (JSONValue | fail Error) jsonParseImpl(Parser parser) { + if (parser.acceptToken(TokenType.doubleQuote)) { + return JSONValue(parseStringLiteral(parser)?); + } + parser.parseJsonNumber?.case { + (:failure): {} + int value: return JSONValue(value); + } + if (parser.acceptToken(TokenType.lsquarebracket)) { + mut JSONValue[] entries; + if (!parser.acceptToken(TokenType.rsquarebracket)) while (true) { + entries ~= jsonParseImpl(parser)?; + if (parser.acceptToken(TokenType.rsquarebracket)) break; + parser.expect(",")?; + } + return JSONValue(entries); + } + if (parser.accept("{")?) { + mut (string key, JSONValue value)[] entries; + if (!parser.accept("}")?) while (true) { + parser.expect("\"")?; + auto key = parseStringLiteral(parser)?; + parser.expect(":")?; + auto value = jsonParseImpl(parser)?; + entries ~= (key, value); + if (parser.accept("}")?) break; + parser.expect(",")?; + } + return JSONValue(entries); + } + return parser.fail("unexpected input: " ~ parser.text); } - string str = start[0 .. matchLen]; - if (!parser.accept("\"")?) { - return parser.fail("this should never happen"); + + // reused by the macro + (string | fail Error) parseStringLiteral(Parser parser) + { + auto str = parser.acceptTokenStr(TokenType.stringLiteral) + .case(:none: return parser.fail("lexer error: string expected")); + if (!parser.acceptToken(TokenType.doubleQuote)) + return parser.fail("'\"' expected"); + return replaceEscapes(str); } - return replaceEscapes(str); + // reused by the macro + (:failure | int | fail Error) parseJsonNumber(Parser parser) + { + parser.begin; + mut bool negative = parser.acceptToken(TokenType.lparen); + if (parser.acceptToken(TokenType.minus)) + negative = true; + parser.acceptTokenStr(TokenType.number).case { + :none: + parser.revert; + return (:failure); + string number: + parser.commit; + mut int i = atoi(number); + if (negative) i = -i; + return i; + } + } } // Helper used by the macro to convert anything into JSONValue - including a JSONValue. @@ -183,31 +279,6 @@ public JSONValue __jv( ); } -// reused by the macro -(:failure | int | fail Error) parseJsonNumber(Parser parser) -{ - parser.begin; - mut bool negative = parser.accept("-")?; - if (parser.accept("-")?) - negative = true; - parser.strip?; - if (parser.hard_eof || !isDigit(parser.text[0])) - { - parser.revert; - return :failure; - } - mut string number; - while (!parser.hard_eof && isDigit(parser.text[0])) - { - number ~= parser.text[0]; - parser.drop(1); - } - parser.commit; - mut int i = atoi(number); - if (negative) i = -i; - return i; -} - /** * Converts a JSON value to a string. */ diff --git a/src/std/json/macro.nt b/src/std/json/macro.nt index 81832d5b..b7a068b8 100644 --- a/src/std/json/macro.nt +++ b/src/std/json/macro.nt @@ -43,7 +43,7 @@ private (ASTSymbol | fail Error) parseJSONValueMacro(Parser parser, LexicalConte } parser.parseJsonNumber?.case { (:failure): {} - long value: return lexicalContext.compiler.astNumberLiteral(value, __RANGE__) + int value: return lexicalContext.compiler.astNumberLiteral(value, __RANGE__) .astJSONValue(lexicalContext.compiler); } if (parser.accept("true")?) { diff --git a/src/std/macro/assert.nt b/src/std/macro/assert.nt index 7ff8a531..8a3ae0da 100644 --- a/src/std/macro/assert.nt +++ b/src/std/macro/assert.nt @@ -276,7 +276,8 @@ class ParseAssert : Macro (nullable ASTStatement | fail Error) parse(Parser parser, LexicalContext context) { parser.begin; - parser.strip?; + version (firstpassmacro) parser.strip?; + else parser.strip; if (!parser.accept("assert")?) { parser.revert; diff --git a/src/std/macro/cimport.nt b/src/std/macro/cimport.nt index 5e5988ba..7f5cb404 100644 --- a/src/std/macro/cimport.nt +++ b/src/std/macro/cimport.nt @@ -7,13 +7,16 @@ module std.macro.cimport; import package(compiler).neat.base; import package(compiler).neat.decl; import package(compiler).neat.expr; -import package(compiler).neat.parser; +version (firstpassmacro) import package(compiler).neat.parser; +else import package(compiler).neat.simple_parser; import package(compiler).neat.struct_; import package(compiler).neat.types; import package(compiler).neat.util; import package(compiler).helpers; import package(compiler).neat.union_; +version (firstpassmacro) alias SimpleParser = ParserImpl; + version (firstpassmacro) class ASTNumberLiteral : ASTSymbol { long value; @@ -47,13 +50,21 @@ class CImportMacro : Macro // TODO move into neat.base auto from = parser.loc; parser.expect("\"")?; - auto includeSym = compiler.parseStringLiteral(parser, lexicalContext, "\"", from)?.instanceOf(ASTStringLiteral); + version (firstpassmacro) { + auto includeSym = compiler.parseStringLiteral(parser, lexicalContext, "\"", from)?.instanceOf(ASTStringLiteral); + } else { + auto includeSym = compiler.parseStringLiteral(parser, lexicalContext)?.instanceOf(ASTStringLiteral); + } parser.assert_(!!includeSym, "string expected")?; mut string flags; if (parser.accept(",")?) { auto from = parser.loc; parser.expect("\"")?; - auto flagsSym = compiler.parseStringLiteral(parser, lexicalContext, "\"", from)?.instanceOf(ASTStringLiteral); + version (firstpassmacro) { + auto flagsSym = compiler.parseStringLiteral(parser, lexicalContext, "\"", from)?.instanceOf(ASTStringLiteral); + } else { + auto flagsSym = compiler.parseStringLiteral(parser, lexicalContext)?.instanceOf(ASTStringLiteral); + } parser.assert_(!!flagsSym, "flags string expected")?; flags = flagsSym.text ~ " "; } @@ -88,8 +99,8 @@ class CImportMacro : Macro // do not count references on the file source - it will end // up with lots of small inc/decs to fragments that we never free anyway (cast(size_t*) &csource)[2] = 0; - auto cparser = lexicalContext.compiler.createParser(headerName ~ ".out", csource); + auto cparser = new SimpleParser(headerName ~ ".out", csource); auto cparserHelper = new CParserHelper(cparser); auto pak = new Package("c_header", "", []); @@ -250,7 +261,7 @@ class CImportMacro : Macro } } -void eatline(Parser parser) { +void eatline(SimpleParser parser) { auto pos = parser.text.find("\n"); if (pos == -1) parser.drop(cast(int) parser.text.length); else parser.drop(pos + 1); @@ -304,7 +315,7 @@ class CStaticArray : ASTSymbol } class CParserHelper { - Parser parser; + SimpleParser parser; ASTModuleBase mod, sysmod; @@ -329,7 +340,9 @@ class CParserHelper { } if (ident.length > 0) { if (auto definedSym = this.mod.getSymbolAlias(ident)) return definedSym; - if (auto definedSym = this.sysmod.getSymbolAlias(ident)) return definedSym; + // FIXME + if (this.sysmod) + if (auto definedSym = this.sysmod.getSymbolAlias(ident)) return definedSym; // print("No such ident " ~ ident); } return null; diff --git a/src/std/macro/listcomprehension.nt b/src/std/macro/listcomprehension.nt index 99377e15..f2d454cf 100644 --- a/src/std/macro/listcomprehension.nt +++ b/src/std/macro/listcomprehension.nt @@ -368,9 +368,16 @@ class ListComprehension : Macro parser.begin(); auto from = parser.from?; - if (!parser.accept("[")?) { - parser.revert(); - return null; + version (firstpassmacro) { + if (!parser.accept("[")?) { + parser.revert(); + return null; + } + } else { + if (!parser.acceptToken(TokenType.lsquarebracket)) { + parser.revert(); + return null; + } } mut string iterationMode; mut nullable ASTSymbol default_; diff --git a/src/std/macro/quasiquoting.nt b/src/std/macro/quasiquoting.nt index fc415d96..8de2ebf6 100644 --- a/src/std/macro/quasiquoting.nt +++ b/src/std/macro/quasiquoting.nt @@ -40,42 +40,76 @@ class QuasiQuoting : Macro } } - (nullable ASTSymbol | fail Error) parse(Parser parser, LexicalContext lexicalContext, ASTSymbol compilerExpr) - { - auto compiler = lexicalContext.compiler; + version (firstpassmacro) { + (nullable ASTSymbol | fail Error) parse(Parser parser, LexicalContext lexicalContext, ASTSymbol compilerExpr) { + auto compiler = lexicalContext.compiler; - auto quoter = new QuasiQuoterImpl(compiler, compilerExpr); - { - parser.begin(); - if (parser.accept(".")? && parser.accept("\$stmt")?) { - parser.commit(); + auto quoter = new QuasiQuoterImpl(compiler, compilerExpr); + { + parser.begin(); + if (parser.accept(".")? && parser.accept("\$stmt")?) { + parser.commit(); + auto stmt = compiler.parseStatement(parser, lexicalContext)?; + parser.assert_(!!stmt, "statement expected")?; + return stmt.quote(quoter); + } + parser.revert(); + } + { + parser.begin(); + if (parser.accept(".")? && parser.accept("\$expr")?) { + parser.commit(); + auto expr = compiler.parseExpression(parser, lexicalContext)?; + parser.assert_(!!expr, "expression expected")?; + return expr.quote(quoter); + } + parser.revert(); + } + { + parser.begin(); + if (parser.accept(".")? && parser.accept("\$type")?) { + parser.commit(); + auto type = compiler.parseType(parser, lexicalContext)?; + parser.assert_(!!type, "type expected")?; + return type.quote(quoter); + } + parser.revert(); + } + return null; + } + } else { + (nullable ASTSymbol | fail Error) parse(Parser parser, LexicalContext lexicalContext, ASTSymbol compilerExpr) { + auto compiler = lexicalContext.compiler; + + auto quoter = new QuasiQuoterImpl(compiler, compilerExpr); + + parser.begin; + if (!parser.acceptToken(TokenType.dot)) { + parser.revert; + return null; + } + auto ident = parser.parseIdentifier?; + if (ident == "\$stmt") { + parser.commit; auto stmt = compiler.parseStatement(parser, lexicalContext)?; parser.assert_(!!stmt, "statement expected")?; return stmt.quote(quoter); } - parser.revert(); - } - { - parser.begin(); - if (parser.accept(".")? && parser.accept("\$expr")?) { - parser.commit(); + if (ident == "\$expr") { + parser.commit; auto expr = compiler.parseExpression(parser, lexicalContext)?; parser.assert_(!!expr, "expression expected")?; return expr.quote(quoter); } - parser.revert(); - } - { - parser.begin(); - if (parser.accept(".")? && parser.accept("\$type")?) { - parser.commit(); + if (ident == "\$type") { + parser.commit; auto type = compiler.parseType(parser, lexicalContext)?; parser.assert_(!!type, "type expected")?; return type.quote(quoter); } parser.revert(); + return null; } - return null; } } diff --git a/test/runnable/mathtest.nt b/test/runnable/mathtest.nt index 57c7ed49..f6700002 100644 --- a/test/runnable/mathtest.nt +++ b/test/runnable/mathtest.nt @@ -1,8 +1,10 @@ module mathtest; -import std.algorithm; macro import std.macro.assert; +import std.algorithm; +import std.math; + void main() { mut int i = 1; assert(i++ == 1); @@ -19,4 +21,5 @@ void main() { assert(5 | 3 == 7); long l = 1_000_000_000_000; assert(l == cast(long) 1_000_000 * 1_000_000); + assert(abs(sin(π)) <= 0.0001); }