Run a lexer on the input source before parsing.

src/main.nt

@@ -910,7 +910,7 @@ void buildBinary(Options options, string mainFile, string execPath, Package[] pa
  neat_runtime_system("mkdir -p .obj");
 
  auto pool = new ThreadPool(options.threads);
- auto workPool = new WorkPool(pool, options.threads);
+ auto workPool = new WorkPool(pool);
  auto builtins = builtinSymbols(options);
  auto defaultImports = [builtins];
  auto modParseConfig = new ModuleParserConfig(packages, defaultImports);

src/neat/array.nt

@@ -697,20 +697,20 @@ Expression releaseArray(Context context, Expression value) {
 (nullable ASTSymbol | fail Error) parseArrayLiteral(Parser parser, LexicalContext lexicalContext)
 {
  parser.begin;
- if (parser.accept("[")?) { }
+ if (parser.acceptToken(TokenType.lsquarebracket)) { }
  else {
  parser.revert;
  return null;
  }
  mut ASTArrayLiteralElement[] values;
  while (true) {
- if (parser.accept("]")?) break;
+ if (parser.acceptToken(TokenType.rsquarebracket)) break;
  auto from = parser.from?;
  ASTSymbol value = lexicalContext.compiler.parseExpression(parser, lexicalContext)?
  .notNull;
  values ~= ASTArrayLiteralElement(value, parser.to(from));
- if (parser.accept("]")?) break;
- parser.expect(",")?;
+ if (parser.acceptToken(TokenType.rsquarebracket)) break;
+ parser.expectToken(TokenType.comma)?;
  }
  parser.commit;
  return new ASTArrayLiteral(null, values);

src/neat/base.nt

@@ -4,6 +4,47 @@ import backend.base;
 import helpers;
 import neat.hash;
 
+enum TokenType {
+ none,
+ end,
+ whitespace,
+ identifier,
+ number,
+ comment,
+ dot,
+ comma,
+ singleQuote,
+ charLiteral,
+ doubleQuote, // ["]foo $(bar) baz["]
+ stringLiteral, // "[foo ]$(bar)[ baz]"
+ formatQuoteStart, // "foo [$](bar)"
+ backtick,
+ colon,
+ semicolon,
+ lparen,
+ rparen,
+ smaller,
+ greater,
+ equal,
+ exclamationmark,
+ questionmark,
+ dollar,
+ lsquarebracket,
+ rsquarebracket,
+ lcurlybracket,
+ rcurlybracket,
+ plus,
+ minus,
+ asterisk,
+ slash,
+ backslash,
+ circumflex,
+ percent,
+ tilde,
+ ampersand,
+ bar,
+}
+
 // something that can be referenced by a name
 abstract class Symbol
 {
@@ -656,7 +697,7 @@ abstract class CompilerBase
  string[] cFlags;
 
  // parser
- abstract Parser createParser(string filename, string fulltext);
+ abstract (Parser | fail Error) createParser(string filename, string fulltext);
 
  abstract (nullable ASTSymbol | fail Error) parseExpression(
  Parser parser, LexicalContext lexicalContext);
@@ -674,7 +715,13 @@ abstract class CompilerBase
  Parser parser, LexicalContext lexicalContext);
 
  abstract (ASTSymbol | fail Error) parseStringLiteral(
- Parser parser, LexicalContext lexicalContext, string endMarker, LocRange from);
+ Parser parser, LexicalContext lexicalContext);
+
+ (ASTSymbol | fail Error) parseStringLiteral(
+ Parser parser, LexicalContext lexicalContext, string separator, LocRange from)
+ {
+ return parseStringLiteral(parser, lexicalContext);
+ }
 
  abstract (nullable ASTSymbolDeclaration | fail Error) parseTemplateStub(
  Parser parser, LexicalContext lexicalContext, string name, string comment,
@@ -1234,14 +1281,20 @@ abstract class Parser
  abstract LocRange loc();
  abstract (LocRange | fail Error) from();
  abstract LocRange to(LocRange from);
- abstract (void | fail Error) strip();
- abstract (string | fail Error) parseLastComment();
+ abstract void strip();
+ abstract string parseLastComment();
  abstract string text();
+ abstract bool acceptToken(TokenType type);
+ abstract bool acceptToken2(TokenType first, TokenType second);
+ abstract bool acceptToken3(TokenType first, TokenType second, TokenType third);
+ abstract bool acceptToken2Not(TokenType match, TokenType nomatch);
+ abstract bool acceptToken2Not2(TokenType match, TokenType nomatch1, TokenType nomatch2);
+ abstract (string | :none) acceptTokenStr(TokenType type);
+ abstract (bool | fail Error) acceptIdentifier(string identifier);
+ abstract (void | fail Error) expectToken(TokenType token);
+ abstract bool peekToken(TokenType type);
  abstract (bool | fail Error) eof();
  abstract bool hard_eof();
- abstract void dropOneCharNonNewline(int length);
- abstract void drop(int length);
- abstract string peekUniChar();
  abstract (void | fail Error) verifyTransactions(string msg, (void | fail Error) delegate() dg);
  abstract (string | fail Error) parseIdentifier();
 
@@ -1258,19 +1311,6 @@ abstract class Parser
  return this.fail("'" ~ match ~ "' expected");
  }
 
- (bool | fail Error) acceptIdentifier(string identifier)
- with (transaction)
- {
- string nextIdent = parseIdentifier?;
-
- if (nextIdent != identifier)
- {
- return false;
- }
- commit;
- return true;
- }
-
  ParserTransaction transaction() {
  return ParserTransaction(this, committed=false);
  }
@@ -1312,6 +1352,7 @@ struct ParserTransaction
  if (!committed) parser.revert;
  }
  void commit() {
+ assert(!committed);
  parser.commit;
  committed = true;
  }

src/neat/class_.nt

@@ -1547,7 +1547,7 @@ class ASTIntfDecl : ASTSymbolDeclaration
  mut bool classAbstract = false;
  mut bool classFinal = false;
  parser.begin;
- auto comment = parser.parseLastComment?;
+ auto comment = parser.parseLastComment;
  auto from = parser.from?;
  while (true) {
  if (parser.acceptIdentifier("abstract")?) {
@@ -1572,19 +1572,19 @@ class ASTIntfDecl : ASTSymbolDeclaration
 
  (nullable ASTSymbolDeclaration | fail Error) parseRest() {
  mut ASTSymbol[] supers;
- if (parser.accept(":")?) {
+ if (parser.acceptToken(TokenType.colon)) {
  while (true) {
  auto super_ = lexicalContext.compiler.parseType(parser, lexicalContext)?;
  parser.assert_(!!super_, "expected super class identifier")?;
  supers ~= super_.notNull;
- if (parser.accept(",")?) continue;
+ if (parser.acceptToken(TokenType.comma)) continue;
  break;
  }
  }
 
  mut ASTClassEntry[] members;
- parser.expect("{")?;
- while (!parser.accept("}")?) {
+ parser.expectToken(TokenType.lcurlybracket)?;
+ while (!parser.acceptToken(TokenType.rcurlybracket)) {
  (void | fail Error) do_() {
  members ~= parseClassEntry(parser, lexicalContext, name)?;
  }
@@ -1601,21 +1601,21 @@ class ASTIntfDecl : ASTSymbolDeclaration
 
 (ASTClassEntry[] | fail Error) parseClassEntry(Parser parser, LexicalContext lexicalContext, string className)
 {
- if (parser.accept("version")?) {
+ if (parser.acceptIdentifier("version")?) {
  mut ASTClassEntry[] then, else_;
- parser.expect("(")?;
+ parser.expectToken(TokenType.lparen)?;
  string versionStr = parser.parseIdentifier?;
- parser.expect(")")?;
- if (parser.accept("{")?) {
- while (!parser.accept("}")?) {
+ parser.expectToken(TokenType.rparen)?;
+ if (parser.acceptToken(TokenType.lcurlybracket)) {
+ while (!parser.acceptToken(TokenType.rcurlybracket)) {
  then ~= parser.parseClassEntry(lexicalContext, className)?;
  }
  } else {
  then ~= parser.parseClassEntry(lexicalContext, className)?;
  }
- if (parser.accept("else")?) {
- if (parser.accept("{")?) {
- while (!parser.accept("}")?) {
+ if (parser.acceptIdentifier("else")?) {
+ if (parser.acceptToken(TokenType.lcurlybracket)) {
+ while (!parser.acceptToken(TokenType.rcurlybracket)) {
  else_ ~= parser.parseClassEntry(lexicalContext, className)?;
  }
  } else {
@@ -1627,7 +1627,7 @@ class ASTIntfDecl : ASTSymbolDeclaration
  return [result];
  }
 
- auto comment = parser.parseLastComment?;
+ auto comment = parser.parseLastComment;
  auto from = parser.from?;
  mut nullable ASTSymbol retType;
  mut string memberName;
@@ -1666,7 +1666,7 @@ class ASTIntfDecl : ASTSymbolDeclaration
  return [entry];
  }
 
- if (parser.accept("this")?)
+ if (parser.acceptIdentifier("this")?)
  {
  parser.assert_(!override_, "cannot override constructor")?;
  retType = new ASTIdentifier("void", false, __RANGE__);
@@ -1681,12 +1681,12 @@ class ASTIntfDecl : ASTSymbolDeclaration
  parser.assert_(memberName.length > 0, "expected member name")?;
  }
  auto locRange = parser.to(from);
- if (parser.accept("(")?) // method
+ if (parser.acceptToken(TokenType.lparen)) // method
  {
  auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=true)?;
  assert(!params.variadic);
  mut nullable ASTStatement stmt = null;
- if (abstract_ && parser.accept(";")?) {
+ if (abstract_ && parser.acceptToken(TokenType.semicolon)) {
  ASTSymbol astName = new ASTStringLiteral(className ~ "." ~ memberName, __RANGE__);
  stmt = lexicalContext.compiler.$stmt {
  import package(compiler).neat.runtime.stdlib : exit;
@@ -1711,9 +1711,9 @@ class ASTIntfDecl : ASTSymbolDeclaration
  members ~= ASTClassDeclMember(protection, memberName, retType.notNull, locRange);
  }
  addMember(locRange)?;
- while (!parser.accept(";")?) {
- parser.expect(",")?;
- if (parser.accept(";")?) break;
+ while (!parser.acceptToken(TokenType.semicolon)) {
+ parser.expectToken(TokenType.comma)?;
+ if (parser.acceptToken(TokenType.semicolon)) break;
  auto memberFrom = parser.from?;
  string ident = parser.parseIdentifier?;
  auto memberLocRange = parser.to(from);
@@ -1728,7 +1728,7 @@ class ASTIntfDecl : ASTSymbolDeclaration
 (nullable ASTSymbolDeclaration | fail Error) parseIntfDecl(Parser parser, LexicalContext lexicalContext)
 {
  parser.begin;
- auto comment = parser.parseLastComment?;
+ auto comment = parser.parseLastComment;
  auto from = parser.from?;
  if (!parser.acceptIdentifier("interface")?)
  {
@@ -1742,18 +1742,18 @@ class ASTIntfDecl : ASTSymbolDeclaration
 
  (nullable ASTSymbolDeclaration | fail Error) parseRest() {
  mut ASTSymbol[] superIntfs;
- if (parser.accept(":")?) {
+ if (parser.acceptToken(TokenType.colon)) {
  while (true) {
  auto superIntf = lexicalContext.compiler.parseType(parser, lexicalContext)?;
  parser.assert_(!!superIntf, "expected super interface")?;
  superIntfs ~= superIntf.notNull;
- if (parser.accept(",")?) continue;
+ if (parser.acceptToken(TokenType.comma)) continue;
  break;
  }
  }
  mut (ASTIntfDeclMethod | ASTDeclaration)[] members;
- parser.expect("{")?;
- while (!parser.accept("}")?)
+ parser.expectToken(TokenType.lcurlybracket)?;
+ while (!parser.acceptToken(TokenType.rcurlybracket))
  {
  auto from = parser.from?;
 
@@ -1767,11 +1767,11 @@ class ASTIntfDecl : ASTSymbolDeclaration
  string memberName = parser.parseIdentifier?;
  parser.to(from).assert(memberName.length > 0, "expected member name")?;
  auto locRange = parser.to(from);
- parser.expect("(")?;
+ parser.expectToken(TokenType.lparen)?;
  auto params = parseParameterList(parser, lexicalContext, variadic=false, thisAssignment=true)?;
  assert(!params.variadic);
  auto params = params.params;
- parser.expect(";")?;
+ parser.expectToken(TokenType.semicolon)?;
  // TODO error on specific parameter
  locRange.assert(
  [all a.case(ASTParameter: true, ASTThisAssignment: false) for a in params],
@@ -1991,14 +1991,14 @@ bool hasAsParent(Class class_, Class parent)
 {
  parser.begin;
  auto from = parser.from?;
- if (!(parser.accept(".")? && parser.accept("instanceOf")?))
+ if (!(parser.acceptToken(TokenType.dot) && parser.acceptIdentifier("instanceOf")?))
  {
  parser.revert;
  return null;
  }
- parser.expect("(")?;
+ parser.expectToken(TokenType.lparen)?;
  nullable ASTSymbol type = lexicalContext.compiler.parseType(parser, lexicalContext)?;
- parser.expect(")")?;
+ parser.expectToken(TokenType.rparen)?;
  parser.commit;
  return new ASTInstanceOf(left, type.notNull, parser.to(from));
 }

src/neat/compiler.nt

@@ -13,6 +13,7 @@ import neat.either;
 import neat.expr;
 import neat.formatstring;
 import neat.function_;
+import neat.lexer;
 import neat.parser;
 import neat.pragmas;
 import neat.statements;
@@ -51,8 +52,8 @@ class CompilerImpl : CompilerBase
  }
 
  // parser
- override Parser createParser(string filename, string fulltext) {
- return new ParserImpl(filename, fulltext);
+ override (Parser | fail Error) createParser(string filename, string fulltext) {
+ return new ParserImpl(filename, fulltext, tokenize(fulltext)?);
  }
 
  override (nullable ASTSymbol | fail Error) parseExpression(Parser parser, LexicalContext lexicalContext)
@@ -81,9 +82,9 @@ class CompilerImpl : CompilerBase
  }
 
  override (ASTSymbol | fail Error) parseStringLiteral(
- Parser parser, LexicalContext lexicalContext, string endMarker, LocRange from)
+ Parser parser, LexicalContext lexicalContext)
  {
- return .parseStringLiteral(parser.instanceOf(Parser).notNull, lexicalContext, endMarker, from);
+ return .parseStringLiteral(parser, lexicalContext);
  }
 
  override (nullable ASTSymbolDeclaration | fail Error) parseTemplateStub(
@@ -98,7 +99,7 @@ class CompilerImpl : CompilerBase
  if (parser.accept("=>")?) {
  auto expr = parseExpression(parser, lexicalContext)?;
  if (!expr) return parser.fail("function expression expected");
- parser.expect(";")?;
+ parser.expectToken(TokenType.semicolon)?;
  return astReturn(expr.notNull, failProp=false, expr.locRange);
  }
  return .parseStatement(parser.instanceOf(Parser).notNull, lexicalContext);

src/neat/decl.nt

@@ -26,8 +26,8 @@ class ASTAliasDecl : ASTSymbolDeclaration
 
 (nullable ASTDeclaration | fail Error) parseAliasDecl(Parser parser, LexicalContext lexicalContext) {
  parser.begin;
- auto comment = parser.parseLastComment?;
- if (!parser.accept("alias")?) {
+ auto comment = parser.parseLastComment;
+ if (!parser.acceptIdentifier("alias")?) {
  parser.revert;
  return null;
  }
@@ -37,10 +37,10 @@ class ASTAliasDecl : ASTSymbolDeclaration
  parser.commit;
  auto locRange = parser.to(from);
  (nullable ASTSymbolDeclaration | fail Error) parseRest() {
- parser.expect("=")?;
+ parser.expectToken(TokenType.equal)?;
  mut nullable ASTSymbol target = lexicalContext.compiler.parseType(parser, lexicalContext)?;
  if (!target) target = lexicalContext.compiler.parseExpression(parser, lexicalContext)?;
- parser.expect(";")?;
+ parser.expectToken(TokenType.semicolon)?;
  return new ASTAliasDecl(name, target.notNull, locRange);
  }
  if (auto ret = lexicalContext.compiler.parseTemplateStub(parser, lexicalContext, name, comment, &parseRest)?)