Skip to content

Commit

Permalink
Moved keywords into the lexer
Browse files Browse the repository at this point in the history
  • Loading branch information
Ratstail91 committed Oct 14, 2024
1 parent 694e262 commit 425ef7e
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 97 deletions.
77 changes: 0 additions & 77 deletions source/toy_keywords.c

This file was deleted.

16 changes: 0 additions & 16 deletions source/toy_keywords.h

This file was deleted.

87 changes: 83 additions & 4 deletions source/toy_lexer.c
Original file line number Diff line number Diff line change
@@ -1,11 +1,90 @@
#include "toy_lexer.h"
#include "toy_keywords.h"
#include "toy_console_colors.h"

#include <stdio.h>
#include <string.h>
#include <ctype.h>

//keyword data
typedef struct {
const Toy_TokenType type;
const char* keyword;
} Toy_KeywordTypeTuple;

const Toy_KeywordTypeTuple keywordTuples[] = {
//null
{TOY_TOKEN_NULL, "null"},

//types
{TOY_TOKEN_TYPE_TYPE, "type"},
{TOY_TOKEN_TYPE_BOOLEAN, "bool"},
{TOY_TOKEN_TYPE_INTEGER, "int"},
{TOY_TOKEN_TYPE_FLOAT, "float"},
{TOY_TOKEN_TYPE_STRING, "string"},
// TOY_TOKEN_TYPE_ARRAY,
// TOY_TOKEN_TYPE_DICTIONARY,
// TOY_TOKEN_TYPE_FUNCTION,
{TOY_TOKEN_TYPE_OPAQUE, "opaque"},
{TOY_TOKEN_TYPE_ANY, "any"},

//keywords and reserved words
{TOY_TOKEN_KEYWORD_AS, "as"},
{TOY_TOKEN_KEYWORD_ASSERT, "assert"},
{TOY_TOKEN_KEYWORD_BREAK, "break"},
{TOY_TOKEN_KEYWORD_CLASS, "class"},
{TOY_TOKEN_KEYWORD_CONST, "const"},
{TOY_TOKEN_KEYWORD_CONTINUE, "continue"},
{TOY_TOKEN_KEYWORD_DO, "do"},
{TOY_TOKEN_KEYWORD_ELSE, "else"},
{TOY_TOKEN_KEYWORD_EXPORT, "export"},
{TOY_TOKEN_KEYWORD_FOR, "for"},
{TOY_TOKEN_KEYWORD_FOREACH, "foreach"},
{TOY_TOKEN_KEYWORD_FUNCTION, "fn"},
{TOY_TOKEN_KEYWORD_IF, "if"},
{TOY_TOKEN_KEYWORD_IMPORT, "import"},
{TOY_TOKEN_KEYWORD_IN, "in"},
{TOY_TOKEN_KEYWORD_OF, "of"},
{TOY_TOKEN_KEYWORD_PRINT, "print"},
{TOY_TOKEN_KEYWORD_RETURN, "return"},
{TOY_TOKEN_KEYWORD_TYPEAS, "typeas"},
{TOY_TOKEN_KEYWORD_TYPEOF, "typeof"},
{TOY_TOKEN_KEYWORD_VAR, "var"},
{TOY_TOKEN_KEYWORD_WHILE, "while"},
{TOY_TOKEN_KEYWORD_YIELD, "yield"},

//literal values
{TOY_TOKEN_LITERAL_TRUE, "true"},
{TOY_TOKEN_LITERAL_FALSE, "false"},

{TOY_TOKEN_EOF, NULL},
};

const char* Toy_private_findKeywordByType(const Toy_TokenType type) {
if (type == TOY_TOKEN_EOF) {
return "EOF";
}

for(int i = 0; keywordTuples[i].keyword; i++) {
if (keywordTuples[i].type == type) {
return keywordTuples[i].keyword;
}
}

return NULL;
}

Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword) {
const int length = strlen(keyword);

for (int i = 0; keywordTuples[i].keyword; i++) {
if (!strncmp(keyword, keywordTuples[i].keyword, length)) {
return keywordTuples[i].type;
}
}

return TOY_TOKEN_EOF;
}

//static generic utility functions
static void cleanLexer(Toy_Lexer* lexer) {
lexer->start = 0;
Expand Down Expand Up @@ -203,13 +282,13 @@ static Toy_Token makeKeywordOrName(Toy_Lexer* lexer) {
}

//scan for a keyword
for (int i = 0; Toy_private_keywords[i].keyword; i++) {
for (int i = 0; keywordTuples[i].keyword; i++) {
//WONTFIX: could squeeze miniscule performance gain from this, but ROI isn't worth it
if (strlen(Toy_private_keywords[i].keyword) == (lexer->current - lexer->start) && !strncmp(Toy_private_keywords[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
if (strlen(keywordTuples[i].keyword) == (lexer->current - lexer->start) && !strncmp(keywordTuples[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
//make token (keyword)
Toy_Token token;

token.type = Toy_private_keywords[i].type;
token.type = keywordTuples[i].type;
token.length = lexer->current - lexer->start;
token.line = lexer->line;
token.lexeme = &lexer->source[lexer->start];
Expand Down

0 comments on commit 425ef7e

Please sign in to comment.