From 425ef7e3e0f13a7d95b7da61d3ff579b10745e63 Mon Sep 17 00:00:00 2001 From: Kayne Ruse Date: Mon, 14 Oct 2024 12:41:53 +1100 Subject: [PATCH] Moved keywords into the lexer --- source/toy_keywords.c | 77 -------------------------------------- source/toy_keywords.h | 16 -------- source/toy_lexer.c | 87 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 97 deletions(-) delete mode 100644 source/toy_keywords.c delete mode 100644 source/toy_keywords.h diff --git a/source/toy_keywords.c b/source/toy_keywords.c deleted file mode 100644 index b105882..0000000 --- a/source/toy_keywords.c +++ /dev/null @@ -1,77 +0,0 @@ -#include "toy_keywords.h" - -#include - -const Toy_KeywordTypeTuple Toy_private_keywords[] = { - //null - {TOY_TOKEN_NULL, "null"}, - - //types - {TOY_TOKEN_TYPE_TYPE, "type"}, - {TOY_TOKEN_TYPE_BOOLEAN, "bool"}, - {TOY_TOKEN_TYPE_INTEGER, "int"}, - {TOY_TOKEN_TYPE_FLOAT, "float"}, - {TOY_TOKEN_TYPE_STRING, "string"}, - // TOY_TOKEN_TYPE_ARRAY, - // TOY_TOKEN_TYPE_DICTIONARY, - // TOY_TOKEN_TYPE_FUNCTION, - {TOY_TOKEN_TYPE_OPAQUE, "opaque"}, - {TOY_TOKEN_TYPE_ANY, "any"}, - - //keywords and reserved words - {TOY_TOKEN_KEYWORD_AS, "as"}, - {TOY_TOKEN_KEYWORD_ASSERT, "assert"}, - {TOY_TOKEN_KEYWORD_BREAK, "break"}, - {TOY_TOKEN_KEYWORD_CLASS, "class"}, - {TOY_TOKEN_KEYWORD_CONST, "const"}, - {TOY_TOKEN_KEYWORD_CONTINUE, "continue"}, - {TOY_TOKEN_KEYWORD_DO, "do"}, - {TOY_TOKEN_KEYWORD_ELSE, "else"}, - {TOY_TOKEN_KEYWORD_EXPORT, "export"}, - {TOY_TOKEN_KEYWORD_FOR, "for"}, - {TOY_TOKEN_KEYWORD_FOREACH, "foreach"}, - {TOY_TOKEN_KEYWORD_FUNCTION, "fn"}, - {TOY_TOKEN_KEYWORD_IF, "if"}, - {TOY_TOKEN_KEYWORD_IMPORT, "import"}, - {TOY_TOKEN_KEYWORD_IN, "in"}, - {TOY_TOKEN_KEYWORD_OF, "of"}, - {TOY_TOKEN_KEYWORD_PRINT, "print"}, - {TOY_TOKEN_KEYWORD_RETURN, "return"}, - {TOY_TOKEN_KEYWORD_TYPEAS, "typeas"}, - {TOY_TOKEN_KEYWORD_TYPEOF, "typeof"}, - {TOY_TOKEN_KEYWORD_VAR, "var"}, - {TOY_TOKEN_KEYWORD_WHILE, "while"}, - {TOY_TOKEN_KEYWORD_YIELD, "yield"}, - - //literal values - {TOY_TOKEN_LITERAL_TRUE, "true"}, - {TOY_TOKEN_LITERAL_FALSE, "false"}, - - {TOY_TOKEN_EOF, NULL}, -}; - -const char* Toy_private_findKeywordByType(const Toy_TokenType type) { - if (type == TOY_TOKEN_EOF) { - return "EOF"; - } - - for(int i = 0; Toy_private_keywords[i].keyword; i++) { - if (Toy_private_keywords[i].type == type) { - return Toy_private_keywords[i].keyword; - } - } - - return NULL; -} - -Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword) { - const int length = strlen(keyword); - - for (int i = 0; Toy_private_keywords[i].keyword; i++) { - if (!strncmp(keyword, Toy_private_keywords[i].keyword, length)) { - return Toy_private_keywords[i].type; - } - } - - return TOY_TOKEN_EOF; -} diff --git a/source/toy_keywords.h b/source/toy_keywords.h deleted file mode 100644 index e3d908f..0000000 --- a/source/toy_keywords.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include "toy_token_types.h" -#include "toy_common.h" - -typedef struct { - const Toy_TokenType type; - const char* keyword; -} Toy_KeywordTypeTuple; - -extern const Toy_KeywordTypeTuple Toy_private_keywords[]; - -//access -const char* Toy_private_findKeywordByType(const Toy_TokenType type); -Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword); - diff --git a/source/toy_lexer.c b/source/toy_lexer.c index 15de6d8..3520bb3 100644 --- a/source/toy_lexer.c +++ b/source/toy_lexer.c @@ -1,11 +1,90 @@ #include "toy_lexer.h" -#include "toy_keywords.h" #include "toy_console_colors.h" #include #include #include +//keyword data +typedef struct { + const Toy_TokenType type; + const char* keyword; +} Toy_KeywordTypeTuple; + +const Toy_KeywordTypeTuple keywordTuples[] = { + //null + {TOY_TOKEN_NULL, "null"}, + + //types + {TOY_TOKEN_TYPE_TYPE, "type"}, + {TOY_TOKEN_TYPE_BOOLEAN, "bool"}, + {TOY_TOKEN_TYPE_INTEGER, "int"}, + {TOY_TOKEN_TYPE_FLOAT, "float"}, + {TOY_TOKEN_TYPE_STRING, "string"}, + // TOY_TOKEN_TYPE_ARRAY, + // TOY_TOKEN_TYPE_DICTIONARY, + // TOY_TOKEN_TYPE_FUNCTION, + {TOY_TOKEN_TYPE_OPAQUE, "opaque"}, + {TOY_TOKEN_TYPE_ANY, "any"}, + + //keywords and reserved words + {TOY_TOKEN_KEYWORD_AS, "as"}, + {TOY_TOKEN_KEYWORD_ASSERT, "assert"}, + {TOY_TOKEN_KEYWORD_BREAK, "break"}, + {TOY_TOKEN_KEYWORD_CLASS, "class"}, + {TOY_TOKEN_KEYWORD_CONST, "const"}, + {TOY_TOKEN_KEYWORD_CONTINUE, "continue"}, + {TOY_TOKEN_KEYWORD_DO, "do"}, + {TOY_TOKEN_KEYWORD_ELSE, "else"}, + {TOY_TOKEN_KEYWORD_EXPORT, "export"}, + {TOY_TOKEN_KEYWORD_FOR, "for"}, + {TOY_TOKEN_KEYWORD_FOREACH, "foreach"}, + {TOY_TOKEN_KEYWORD_FUNCTION, "fn"}, + {TOY_TOKEN_KEYWORD_IF, "if"}, + {TOY_TOKEN_KEYWORD_IMPORT, "import"}, + {TOY_TOKEN_KEYWORD_IN, "in"}, + {TOY_TOKEN_KEYWORD_OF, "of"}, + {TOY_TOKEN_KEYWORD_PRINT, "print"}, + {TOY_TOKEN_KEYWORD_RETURN, "return"}, + {TOY_TOKEN_KEYWORD_TYPEAS, "typeas"}, + {TOY_TOKEN_KEYWORD_TYPEOF, "typeof"}, + {TOY_TOKEN_KEYWORD_VAR, "var"}, + {TOY_TOKEN_KEYWORD_WHILE, "while"}, + {TOY_TOKEN_KEYWORD_YIELD, "yield"}, + + //literal values + {TOY_TOKEN_LITERAL_TRUE, "true"}, + {TOY_TOKEN_LITERAL_FALSE, "false"}, + + {TOY_TOKEN_EOF, NULL}, +}; + +const char* Toy_private_findKeywordByType(const Toy_TokenType type) { + if (type == TOY_TOKEN_EOF) { + return "EOF"; + } + + for(int i = 0; keywordTuples[i].keyword; i++) { + if (keywordTuples[i].type == type) { + return keywordTuples[i].keyword; + } + } + + return NULL; +} + +Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword) { + const int length = strlen(keyword); + + for (int i = 0; keywordTuples[i].keyword; i++) { + if (!strncmp(keyword, keywordTuples[i].keyword, length)) { + return keywordTuples[i].type; + } + } + + return TOY_TOKEN_EOF; +} + //static generic utility functions static void cleanLexer(Toy_Lexer* lexer) { lexer->start = 0; @@ -203,13 +282,13 @@ static Toy_Token makeKeywordOrName(Toy_Lexer* lexer) { } //scan for a keyword - for (int i = 0; Toy_private_keywords[i].keyword; i++) { + for (int i = 0; keywordTuples[i].keyword; i++) { //WONTFIX: could squeeze miniscule performance gain from this, but ROI isn't worth it - if (strlen(Toy_private_keywords[i].keyword) == (lexer->current - lexer->start) && !strncmp(Toy_private_keywords[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) { + if (strlen(keywordTuples[i].keyword) == (lexer->current - lexer->start) && !strncmp(keywordTuples[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) { //make token (keyword) Toy_Token token; - token.type = Toy_private_keywords[i].type; + token.type = keywordTuples[i].type; token.length = lexer->current - lexer->start; token.line = lexer->line; token.lexeme = &lexer->source[lexer->start];