Skip to content

Commit

Permalink
refactor: unify all operators under a single token kind
Browse files Browse the repository at this point in the history
  • Loading branch information
@Huy-DNA
Huy-DNA committed Aug 2, 2023
1 parent 1bfb755 commit fa85b8d
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 190 deletions.
124 changes: 40 additions & 84 deletions packages/dbml-core/src/parse/dbml/src/lib/lexer/lexer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { ParsingError, ParsingErrorCode } from '../errors';
import Result from '../result';
import { isAlpha, isAlphaNumeric, isDigit } from '../utils';
import { SyntaxToken, SyntaxTokenKind, isTriviaToken } from './tokens';
import {
SyntaxToken, SyntaxTokenKind, isOp, isTriviaToken,
} from './tokens';

export default class Lexer {
private start: number = 0;
Expand Down Expand Up @@ -46,10 +48,6 @@ export default class Lexer {
this.start = this.current;
}

private closeGap() {
this.start = this.current;
}

private addToken(kind: SyntaxTokenKind) {
this.tokens.push(
SyntaxToken.create(
Expand Down Expand Up @@ -85,36 +83,6 @@ export default class Lexer {
this.advance();
this.addToken(SyntaxTokenKind.COMMA);
break;
case '.':
this.advance();
this.addToken(SyntaxTokenKind.DOT);
break;
case '/':
if (this.peek(1) === '/') {
this.singleLineComment();
} else if (this.peek(1) === '*') {
this.multilineComment();
} else {
this.advance();
this.addToken(SyntaxTokenKind.FORWARDSLASH);
}
break;
case '%':
this.advance();
this.addToken(SyntaxTokenKind.PERCENT);
break;
case '*':
this.advance();
this.addToken(SyntaxTokenKind.ASTERISK);
break;
case '+':
this.advance();
this.addToken(SyntaxTokenKind.CROSS);
break;
case '-':
this.advance();
this.addToken(SyntaxTokenKind.MINUS);
break;
case '(':
this.advance();
this.addToken(SyntaxTokenKind.LPAREN);
Expand Down Expand Up @@ -147,42 +115,6 @@ export default class Lexer {
this.advance();
this.addToken(SyntaxTokenKind.COLON);
break;
case '<':
this.advance();
if (this.peek() !== '=') {
this.addToken(SyntaxTokenKind.LT);
} else {
this.advance();
this.addToken(SyntaxTokenKind.LE);
}
break;
case '>':
this.advance();
if (this.peek() !== '=') {
this.addToken(SyntaxTokenKind.GT);
} else {
this.advance();
this.addToken(SyntaxTokenKind.GE);
}
break;
case '=':
this.advance();
if (this.peek() !== '=') {
this.addToken(SyntaxTokenKind.EQUAL);
} else {
this.advance();
this.addToken(SyntaxTokenKind.DOUBLE_EQUAL);
}
break;
case '!':
this.advance();
if (this.peek() !== '=') {
this.addToken(SyntaxTokenKind.EXCLAMATION);
} else {
this.advance();
this.addToken(SyntaxTokenKind.NOT_EQUAL);
}
break;
case "'":
if (this.checkTripleQuote()) {
this.multilineStringLiteral();
Expand All @@ -199,27 +131,40 @@ export default class Lexer {
case '#':
this.colorLiteral();
break;
case '/':
if (this.peek(1) === '/') {
this.singleLineComment();
} else if (this.peek(1) === '*') {
this.multilineComment();
} else {
this.operator();
}
break;
default:
if (isOp(c)) {
this.operator();
break;
}
if (isAlpha(c)) {
this.identifier();
break;
} else if (isDigit(c)) {
}
if (isDigit(c)) {
this.numericLiteral();
break;
} else {
this.advance();
this.errors.push(
new ParsingError(
ParsingErrorCode.EXPECTED_THINGS,
`Unexpected token ${c}`,
this.start,
this.current - 1,
),
);
break;
}
this.advance();
this.errors.push(
new ParsingError(
ParsingErrorCode.EXPECTED_THINGS,
`Unexpected token ${c}`,
this.start,
this.current - 1,
),
);
break;
}
this.closeGap();
this.start = this.current;
}
this.tokens.push(SyntaxToken.create(SyntaxTokenKind.EOF, this.start, 0));
this.gatherTrivia();
Expand Down Expand Up @@ -412,6 +357,17 @@ export default class Lexer {
this.addToken(SyntaxTokenKind.IDENTIFIER);
}

operator() {
if (!isOp(this.peek())) {
return;
}

while (isOp(this.peek())) {
this.advance();
}
this.addToken(SyntaxTokenKind.OP);
}

numericLiteral() {
let isFloat = false;
const first = this.peek();
Expand Down
53 changes: 20 additions & 33 deletions packages/dbml-core/src/parse/dbml/src/lib/lexer/tokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export enum SyntaxTokenKind {
LANGLE = '<langle>',
RANGLE = '<rangle>',

DOT = '<dot>',
OP = '<OP>',
EOF = '<eof>',
NUMERIC_LITERAL = '<number>',
STRING_LITERAL = '<string>',
Expand All @@ -25,20 +25,6 @@ export enum SyntaxTokenKind {
QUOTED_STRING = '<variable>',
IDENTIFIER = '<identifier>',

ASTERISK = '<asterisk>',
CROSS = '<cross>',
MINUS = '<minus>',
BACKSLASH = '<backslash>',
FORWARDSLASH = '<forwardslash>',
PERCENT = '<percent>',
EXCLAMATION = '<exclamation>',
EQUAL = '<equal>',
DOUBLE_EQUAL = '<double-equal>',
NOT_EQUAL = '<not-equal>',
LT = '<lt>',
LE = '<le>',
GT = '<gt>',
GE = '<ge>',
SEMICOLON = '<semicolon>',
COLON = '<colon>',

Expand Down Expand Up @@ -66,33 +52,34 @@ export function isTriviaToken(token: SyntaxToken): boolean {
}
}

export function isOpToken(token?: SyntaxToken): boolean {
if (!token) {
export function isOp(c?: string): boolean {
if (!c) {
return false;
}

switch (token.kind) {
case SyntaxTokenKind.CROSS:
case SyntaxTokenKind.ASTERISK:
case SyntaxTokenKind.MINUS:
case SyntaxTokenKind.FORWARDSLASH:
case SyntaxTokenKind.PERCENT:
case SyntaxTokenKind.LT:
case SyntaxTokenKind.LE:
case SyntaxTokenKind.GT:
case SyntaxTokenKind.GE:
case SyntaxTokenKind.EQUAL:
case SyntaxTokenKind.DOUBLE_EQUAL:
case SyntaxTokenKind.NOT_EQUAL:
case SyntaxTokenKind.EXCLAMATION:
case SyntaxTokenKind.DOT:
case SyntaxTokenKind.LPAREN:
switch (c) {
case '+':
case '-':
case '*':
case '/':
case '%':
case '<':
case '>':
case '=':
case '!':
case '.':
case '&':
case '|':
return true;
default:
return false;
}
}

export function isOpToken(token?: SyntaxToken): boolean {
return token !== undefined && token.kind === SyntaxTokenKind.OP;
}

export class SyntaxToken {
kind: SyntaxTokenKind;

Expand Down
Loading

0 comments on commit fa85b8d

Please sign in to comment.