From 2f7fc1f1d9e04a481591378fce7bfebc22b5484a Mon Sep 17 00:00:00 2001 From: Ole Morud Date: Mon, 29 Jan 2024 14:54:23 +0100 Subject: [PATCH] Use infix-to-postfix to parse expressions --- src/Makefile | 7 +- src/common.h | 6 + src/error.c | 2 +- src/file_stream.c | 47 +++--- src/file_stream.h | 7 +- src/parser.c | 317 ++++++++++++++++++++++++--------------- src/stack.h | 44 ++++++ src/test/expressions.txt | 6 +- src/tokenizer.c | 57 ++++--- src/tokenizer.h | 37 +++-- 10 files changed, 345 insertions(+), 185 deletions(-) create mode 100644 src/common.h create mode 100644 src/stack.h diff --git a/src/Makefile b/src/Makefile index 32081b1..3e6d27d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,3 +1,6 @@ -all: - gcc -o lang parser.c tokenizer.c error.c file_stream.c +CC = gcc +CFLAGS = -Wall -Wextra -g -O0 + +lang : parser.c tokenizer.c error.c file_stream.c | tokenizer.h error.h common.h file_stream.h + $(CC) $(CFLAGS) -o $@ $^ diff --git a/src/common.h b/src/common.h new file mode 100644 index 0000000..7e502ce --- /dev/null +++ b/src/common.h @@ -0,0 +1,6 @@ + +#pragma once + + +#define FATAL_NOT_IMPLEMENTED 123 +#define FATAL_EXPR_TOO_BIG 101 diff --git a/src/error.c b/src/error.c index d8e9085..5d59fdc 100644 --- a/src/error.c +++ b/src/error.c @@ -83,9 +83,9 @@ static void error_msg_free(struct error_msg* msg) return; error_msg_free(msg->next); free(msg->message); - free(msg); msg->message = NULL; msg->next = NULL; + free(msg); } void error_clear(Error* err) diff --git a/src/file_stream.c b/src/file_stream.c index 13c3e07..5307af3 100644 --- a/src/file_stream.c +++ b/src/file_stream.c @@ -11,8 +11,6 @@ #include "error.h" #include "file_stream.h" -char* mfile_overflow_slope = NULL; - Mfile* mfile_open(Error* err, char* filename) { Mfile* s = malloc(sizeof *s); @@ -58,51 +56,54 @@ void mfile_close(Error* err, Mfile* s) ok = munmap(s->data, s->size); if (ok == -1) { error_push(err, "failed to munmap file: %s", strerror(errno)); - free(s); - return; } close(s->fd); if (ok == -1) { error_push(err, "failed to close file: %s", strerror(errno)); - free(s); - return; } free(s); } -int mfile_get(Mfile* s) +inline size_t mfile_inc_pos(Mfile* m) { - if (s->pos >= s->size) { + return m->pos++; +} +inline size_t mfile_decr_pos(Mfile* m) +{ + return m->pos--; +} + +inline int mfile_get(Mfile* m) +{ + if (m->pos >= m->size) { return EOF; } - return s->data[s->pos++]; + return m->data[mfile_inc_pos(m)]; } -bool mfile_eof(Mfile* s) +inline bool mfile_eof(Mfile* m) { - return s->pos >= s->size; + return m->pos >= m->size; } -char* mfile_cur(Mfile* s) +inline char* mfile_cur(Mfile* m) { - if (s->pos >= s->size) { - return mfile_overflow_slope; + static char eof = EOF; + if (m->pos >= m->size) { + return &eof; } - return s->data + s->pos; + return m->data + m->pos; } -int mfile_curchar(Mfile* s) +inline int mfile_curchar(Mfile* m) { - if (s->pos >= s->size) { - return EOF; - } - return *(s->data + s->pos); + return *(mfile_cur(m)); } -void mfile_skip(Mfile* s, int (*f)(int)) +void mfile_skip(Mfile* m, int (*f)(int)) { - while (f(*(s->data + s->pos))) - s->pos += 1; + while (f(mfile_curchar(m))) + mfile_inc_pos(m); } diff --git a/src/file_stream.h b/src/file_stream.h index 38b8fae..3bc9528 100644 --- a/src/file_stream.h +++ b/src/file_stream.h @@ -8,7 +8,7 @@ extern char* mfile_overflow_slope; typedef struct mfile { char* data; - off_t size; + size_t size; size_t pos; // internal @@ -37,3 +37,8 @@ void mfile_skip(Mfile* s, int (*f)(int)); /* Get current char */ int mfile_curchar(Mfile* s); + +/* Skip the current char */ +size_t mfile_inc_pos(Mfile* m); + +size_t mfile_decr_pos(Mfile* m); diff --git a/src/parser.c b/src/parser.c index 2f9ac3a..9aaafbc 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,8 +1,15 @@ +/* + * TODO: + * line 312 + * */ + #include "error.h" #include "file_stream.h" #include "tokenizer.h" #include "printable.h" +#include "common.h" +#include "stack.h" #include #include @@ -11,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +26,7 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b)) + /* ======= Grammar Rules ======= statement: @@ -35,37 +44,51 @@ expr | FLOAT OPERATOR FLOAT {$$ = binary_op_float($1, $3, $2);}; | INT | FLOAT + ============================= */ enum value_type { VALUE_INTEGER, - VALUE_OPERATOR, VALUE_FLOATING, VALUE_TYPE_COUNT, }; static const char* value_type_str[VALUE_TYPE_COUNT] = { [VALUE_INTEGER] = "VALUE_INTEGER", - [VALUE_OPERATOR] = "VALUE_OPERATOR", [VALUE_FLOATING] = "VALUE_FLOATING", }; -static const int operator_precedence[256] = { - ['+'] = 0, - ['-'] = 0, - ['*'] = 10, - ['/'] = 10, -}; - typedef struct parser { Token* cur; Token* next; Mfile* m; -} Parser; +} ParserState; -bool parser_advance(Error* err, Parser* p) + +void parser_print_position(ParserState* p) +{ + Mfile m = *(p->m); + + m.pos = 0; + int linecount = 0; + int col = 0; + while (!mfile_eof(&m) && m.pos <= p->m->pos) { + char c = mfile_get(&m); + if (c == '\n') { + linecount++; + col = 0; + } else { + col++; + } + } + fprintf(stderr, "\nLine: %d\nCol: %d\n", linecount, col); +} + +bool parser_advance(Error* err, ParserState* p) { p->cur = p->next; + //if (p->cur) + // token_print(NULL, p->cur); mfile_skip(p->m, isspace); p->next = token_read(err, p->m); if (!error_empty(err)) { @@ -85,31 +108,31 @@ typedef struct value { }; } Value; -static Value* parse_operator(Error* err, Token* t) +static void value_print(FILE* out, Value* v) { - if (t->type != TOKEN_OPERATOR) { - error_push(err, "%s: unexpected token type: %s", __func__, token_type_str[t->type]); - return NULL; + switch (v->type) { + case VALUE_INTEGER: + fprintf(out, "%" PRId64, v->i64); + break; + case VALUE_FLOATING: + fprintf(out, "%lf", v->f64); + break; + default: + fprintf(out, "(bad value)"); + break; } - Value* v = calloc(1, sizeof *v); - if (!v) { - error_push(err, "%s: failed to allocate value: %s", __func__, strerror(errno)); - return NULL; - } - v->type = VALUE_OPERATOR; - strncpy(v->op, t->start, MIN(t->end - t->start, sizeof(v->op))); - return v; } -static Value* parse_int(Error* err, Token* t) +static Value* parse_int(Error* err, ParserState* p) { + Token* t = p->cur; if (t->type != TOKEN_INTEGER) { - error_push(err, "%s: unexpected token type: %s", __func__, token_type_str[t->type]); + error_push(err, "(%s) unexpected token type: %s", __func__, token_type_str[t->type]); return NULL; } Value* v = calloc(1, sizeof *v); if (!v) { - error_push(err, "%s: failed to allocate value: %s", __func__, strerror(errno)); + error_push(err, "(%s) failed to allocate value: %s", __func__, strerror(errno)); return NULL; } v->type = VALUE_INTEGER; @@ -117,14 +140,16 @@ static Value* parse_int(Error* err, Token* t) errno = 0; v->i64= strtol(t->start, NULL, 10); if (errno != 0) { - error_push(err, "%s: failed to parse int: %s", __func__, strerror(errno)); + error_push(err, "(%s) failed to parse int: %s", __func__, strerror(errno)); return NULL; } + parser_advance(err, p); return v; } -static Value* parse_floating(Error* err, Token* t) +static Value* parse_floating(Error* err, ParserState* p) { + Token* t = p->cur; if (t->type != TOKEN_FLOATING) { error_push(err, "(%s) unexpected token type: %s", __func__, token_type_str[t->type]); return NULL; @@ -142,20 +167,12 @@ static Value* parse_floating(Error* err, Token* t) error_push(err, "(%s) failed to parse float: %s", __func__, strerror(errno)); return NULL; } - return v; -} - -static Value* parse_number(Error* err, Token* t) -{ - switch (t->type) { - case TOKEN_FLOATING: - return parse_floating(err, t); - case TOKEN_INTEGER: - return parse_int(err, t); - default: - error_push(err, "(%s) unexpected token type %s", __func__, token_type_str[t->type]); + parser_advance(err, p); + if (!error_empty(err)) { + error_push(err, "(%s) couldn't advance parser", __func__); return NULL; } + return v; } static void conv_int_to_float(Error* err, Value* val) @@ -169,15 +186,17 @@ static void conv_int_to_float(Error* err, Value* val) val->type = VALUE_FLOATING; } -static Value* parse_binary_expr(Error* err, Value* lval, Value* rval, Value* op) +static Value* binary_op(Error* err, Value* lval, Value* rval, Token* op) { if ((lval->type != VALUE_INTEGER && lval->type != VALUE_FLOATING) || (rval->type != VALUE_INTEGER && rval->type != VALUE_FLOATING) - || op->type != VALUE_OPERATOR) + || op->type != TOKEN_OPERATOR) { error_push(err, "%s: unexpected token types: %s %s %s", __func__, value_type_str[lval->type], - value_type_str[rval->type], value_type_str[op->type]); + value_type_str[rval->type], token_type_str[op->type]); + fprintf(stderr, "\n####\n"); + token_print(err, op); return NULL; } @@ -197,10 +216,10 @@ static Value* parse_binary_expr(Error* err, Value* lval, Value* rval, Value* op) return NULL; } - fprintf(stderr, "\ndoing op: %s %c %s", value_type_str[lval->type], op->op[0], value_type_str[rval->type]); + //fprintf(stderr, "\ndoing op: %s %c %s", value_type_str[lval->type], op->start[0], value_type_str[rval->type]); if (rval->type == VALUE_INTEGER && lval->type == VALUE_INTEGER) { result->type = VALUE_INTEGER; - switch (op->op[0]) { + switch (op->start[0]) { case '+': result->i64 = lval->i64 + rval->i64; break; @@ -214,11 +233,11 @@ static Value* parse_binary_expr(Error* err, Value* lval, Value* rval, Value* op) result->i64 = lval->i64 / rval->i64; break; } - printf("\nCALCULATED EXPRESSION %ld %c %ld = %ld\n", lval->i64, - op->op[0], rval->i64, result->i64); + //fprintf(stderr, "\nCALCULATED EXPRESSION %ld %c %ld = %ld\n", lval->i64, + // op->start[0], rval->i64, result->i64); } else if (rval->type == VALUE_FLOATING && lval->type == VALUE_FLOATING) { result->type = VALUE_FLOATING; - switch (op->op[0]) { + switch (op->start[0]) { case '+': result->f64 = lval->f64 + rval->f64; break; @@ -232,90 +251,157 @@ static Value* parse_binary_expr(Error* err, Value* lval, Value* rval, Value* op) result->f64 = lval->f64 / rval->f64; break; } - printf("\nCALCULATED EXPRESSION %lf %c %lf = %lf\n", lval->f64, - op->op[0], rval->f64, result->f64); + //fprintf(stderr, "\nCALCULATED EXPRESSION %lf %c %lf = %lf\n", lval->f64, + // op->start[0], rval->f64, result->f64); } return result; } -static Value* parse_expr(Error* err, Parser* p) + +static inline int8_t operator_precedence(Token* op) { - Value* lval = parse_number(err, p->cur); - if (!error_empty(err) || !lval) { - goto generic_error; - } + // wrapped in a function for now because operators longer than 1 char might + // be implemented + static const int8_t lookup[256] = { + ['+'] = -20, ['-'] = -20, + ['*'] = -10, ['/'] = -10, + ['('] = 128, [')'] = 128 + }; + return lookup[(size_t)(op->start[0])]; +} - if (!error_empty(err)) { - goto generic_error; - } else if (p->next->type != TOKEN_OPERATOR) { - parser_advance(err, p); - if (!error_empty(err)) { - goto syntax_error; +static Value* parse_expr(Error* err, ParserState* p) +{ + FixedStack op_stack = STACK_INIT; + FixedStack value_stack = STACK_INIT; + + fprintf(stderr, "PRATT START\n"); + + while (1) { + token_print(NULL, p->cur); + switch (p->cur->type) { + case TOKEN_INTEGER: + stack_push(&value_stack, parse_int(err, p)); + if (!error_empty(err)) + goto fail; + break; + + case TOKEN_FLOATING: + stack_push(&value_stack, parse_floating(err, p)); + if (!error_empty(err)) + goto fail; + break; + + case TOKEN_IDENTIFIER: + fprintf(stderr, "identifiers not implemented yet\n"); + exit(FATAL_NOT_IMPLEMENTED); + break; + + case TOKEN_PAREN_OPEN: + stack_push(&op_stack, p->cur); + parser_advance(err, p); + if (!error_empty(err)) + goto fail; + break; + + case TOKEN_PAREN_CLOSE: + while (!stack_empty(&op_stack) && ((Token*)stack_top(&op_stack))->type != TOKEN_PAREN_OPEN) { + Value* rval = stack_pop(&value_stack); + Value* lval = stack_pop(&value_stack); + Token* op = stack_pop(&op_stack); + Value* result = binary_op(err, lval, rval, op); + if (!error_empty(err)) + goto fail; + stack_push(&value_stack, result); + } + if (((Token*)stack_top(&op_stack))->type != TOKEN_PAREN_OPEN) { + error_push(err, "%s: mismatched parentheses", __func__); + return NULL; + } else { + stack_pop(&op_stack); + } + parser_advance(err, p); + if (!error_empty(err)) + goto fail; + + break; + + case TOKEN_OPERATOR: { + Token* new_op = p->cur; + if (!stack_empty(&op_stack)) { + while (operator_precedence(new_op) < operator_precedence(stack_top(&op_stack))) { + Value* rval = stack_pop(&value_stack); + Value* lval = stack_pop(&value_stack); + Token* op = stack_pop(&op_stack); + Value* result = binary_op(err, lval, rval, op); + if (!error_empty(err)) + goto fail; + stack_push(&value_stack, result); + } + } + parser_advance(err, p); + if (!error_empty(err)) + goto fail; + stack_push(&op_stack, new_op); + break;} + + default: + goto end; } - return lval; } - - Value* op = parse_operator(err, p->next); - if (!error_empty(err)) { - goto generic_error; +end: + while (!stack_empty(&value_stack) && !stack_empty(&op_stack)) { + Token* op = stack_pop(&op_stack); + Value* rval = stack_pop(&value_stack); + Value* lval = stack_pop(&value_stack); + Value* result = binary_op(err, lval, rval, op); + if (!error_empty(err)) + goto fail; + stack_push(&value_stack, result); } - parser_advance(err, p); - if (!error_empty(err)) { - goto generic_error; + if (stack_len(&value_stack) != 1 && stack_len(&op_stack) != 0) { + error_push(err, "(%s) bad expression", __func__); + goto fail; } - parser_advance(err, p); - if (!error_empty(err)) { - goto generic_error; - } - - if (!error_empty(err)) { - goto generic_error; - } else if (p->cur->type != TOKEN_INTEGER - && p->cur->type != TOKEN_FLOATING) - { - goto syntax_error; - } - Value* rval = parse_expr(err, p); - if (!error_empty(err)) { - goto generic_error; - } - - Value* result = parse_binary_expr(err, lval, rval, op); - if (!error_empty(err)) { - goto generic_error; - } - return result; - -syntax_error: - error_push(err, "%s: syntax error, expected binary expression", __func__); -generic_error: + fprintf(stderr, "PRATT END\n"); + return stack_top(&value_stack); + +fail: return NULL; } -static Value* parser_next(Error* err, Parser* p) +static Value* parser_next(Error* err, ParserState* p) { - if (p->cur->type == EOF || !error_empty(err)) { + if (p->cur->type == TOKEN_EOF || !error_empty(err)) { return NULL; } Value* result; switch (p->cur->type) { - case TOKEN_INTEGER: - case TOKEN_FLOATING: - { - result = parse_expr(err, p); - if (!error_empty(err)) { - goto syntax_error; - } - break; - } - default: - syntax_error: - { - error_push(err, "(%s) syntax error: unexpected token %s", __func__, token_type_str[p->cur->type]); - return NULL; + case TOKEN_INTEGER: + case TOKEN_FLOATING: + result = parse_expr(err, p); + if (!error_empty(err) || result == NULL) { + goto syntax_error; } + fprintf(stderr, "result: "); + value_print(stderr, result); + fprintf(stderr, "\n"); + + break; + + default: syntax_error: + error_push(err, "(%s) syntax error: unexpected token %s (%s)", __func__, + token_type_str[p->cur->type], token_str(p->cur)); + return NULL; } + + if (p->cur->type != TOKEN_STATEMENT_END) { + error_push(err, "(%s) expected semicolon", __func__); + return NULL; + } + parser_advance(err, p); + return result; } @@ -330,12 +416,6 @@ int main(int argc, char** argv) return EXIT_FAILURE; } - // create a protected page for debugging purposes - size_t pagesize = sysconf(_SC_PAGESIZE); - void* protected_page = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); - mfile_overflow_slope = protected_page + 0; - mprotect(protected_page, pagesize, PROT_NONE); - Error err = ERROR_INIT; Mfile* m = mfile_open(&err, argv[1]); if (!error_empty(&err)) { @@ -344,7 +424,7 @@ int main(int argc, char** argv) return EXIT_FAILURE; } - Parser p = { + ParserState p = { .cur = NULL, .next = NULL, .m = m, @@ -356,6 +436,7 @@ int main(int argc, char** argv) parser_next(&err, &p); if (!error_empty(&err)) { error_print(&err); + parser_print_position(&p); return EXIT_FAILURE; } } diff --git a/src/stack.h b/src/stack.h new file mode 100644 index 0000000..1c6a69b --- /dev/null +++ b/src/stack.h @@ -0,0 +1,44 @@ + +#pragma once + +#include +#include + +#define STACK_MAX 256 + +typedef struct fixed_stack { + void* vals[STACK_MAX]; + size_t top; +} FixedStack; + +#define STACK_INIT { 0 } + +#define stack_push(s, v) stack_push_((s), (void*)(v)) +static inline void stack_push_(FixedStack* s, void* val) +{ + if (s->top >= STACK_MAX) { + fprintf(stderr, "static stack capacity exceeded"); + exit(EXIT_FAILURE); + } + s->vals[s->top++] = val; +} + +static inline void* stack_top(FixedStack* s) +{ + return s->vals[s->top - 1]; +} + +static inline void* stack_pop(FixedStack* s) +{ + return s->vals[--s->top]; +} + +static inline bool stack_empty(FixedStack* s) +{ + return s->top == 0; +} + +static inline size_t stack_len(FixedStack* s) +{ + return s->top; +} diff --git a/src/test/expressions.txt b/src/test/expressions.txt index 8f331d9..147203c 100644 --- a/src/test/expressions.txt +++ b/src/test/expressions.txt @@ -1,4 +1,2 @@ -1+2 -1 + 4 + 3 -5 / 2 -1.3 * 2 +1+2*2; +2 * (3+4); diff --git a/src/tokenizer.c b/src/tokenizer.c index e6bc6f0..df48d77 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -24,27 +24,29 @@ static const bool is_operator[256] = { static void token_read_operator(Error* err, Mfile* m, Token* t) { + (void)err; t->type = TOKEN_OPERATOR; t->start = mfile_cur(m); while (!mfile_eof(m) && is_operator[(unsigned char)mfile_get(m)]) /* NOOP */; - m->pos -= 1; + mfile_decr_pos(m); t->end = mfile_cur(m); } static void token_read_number(Error* err, Mfile* m, Token* t) { + (void)err; t->start = mfile_cur(m); if (mfile_curchar(m) == '-') { - m->pos += 1; + mfile_inc_pos(m); } mfile_skip(m, isdigit); if (mfile_curchar(m) == '.') { t->type = TOKEN_FLOATING; - m->pos += 1; + mfile_inc_pos(m); mfile_skip(m, isdigit); } else { t->type = TOKEN_INTEGER; @@ -82,7 +84,7 @@ static void token_read_string(Error* err, Mfile* m, Token* t) t->type = TOKEN_STRING; t->start = mfile_cur(m); - m->pos += 1; + mfile_inc_pos(m); bool escaped = false; while (!mfile_eof(m) && (mfile_curchar(m) != '"' || escaped)) { escaped = mfile_get(m) == '\\'; @@ -91,7 +93,7 @@ static void token_read_string(Error* err, Mfile* m, Token* t) error_push(err, "%s: expected '\"', got %c", __func__, PRINTABLE(*(mfile_cur(m)))); return; } - m->pos += 1; + mfile_inc_pos(m); t->end = mfile_cur(m); } @@ -102,20 +104,14 @@ static void token_read_identifier(Error* err, Mfile* m, Token* t) t->start = mfile_cur(m); if (!isalpha(*(t->start))) { - error_push(err, "%s, expected identifier, got %c", __func__, PRINTABLE(*(t->start))); + error_push(err, "(%s), expected alphanumeric character, got %c", + __func__, PRINTABLE(*(t->start))); return; } mfile_skip(m, isalnum); t->end = mfile_cur(m); - - return; -} - -static void token_eval_identifier(Token* t) -{ - } Token* token_read(Error* err, Mfile* m) @@ -132,12 +128,27 @@ Token* token_read(Error* err, Mfile* m) token_read_identifier(err, m, t); } else if (c == '"') { token_read_string(err, m, t); - } else if (isdigit(c) || c == '-') { + } else if (c == ';') { + t->type = TOKEN_STATEMENT_END; + t->start = mfile_cur(m); + mfile_inc_pos(m); + t->end = mfile_cur(m); + } else if (c == '(') { + t->type = TOKEN_PAREN_OPEN; + t->start = mfile_cur(m); + mfile_inc_pos(m); + t->end = mfile_cur(m); + } else if (c == ')') { + t->type = TOKEN_PAREN_CLOSE; + t->start = mfile_cur(m); + mfile_inc_pos(m); + t->end = mfile_cur(m); + } else if (isdigit(c)) { // signs are handled by the parser grammar token_read_number(err, m, t); } else if (is_operator[c]) { token_read_operator(err, m, t); } else if ( c == EOF ) { - t->type = EOF; + t->type = TOKEN_EOF; } else { error_push(err, "unexpected character: %s (0x%02x)", PRINTABLE(c), c); } @@ -149,7 +160,7 @@ void token_print(Error* err, Token* t) { int ok; char* start = t->start; - ok = fprintf(stderr, "{\n\t\""); + ok = fprintf(stderr, "[%s:%ld \"", token_type_str[t->type], t->type); if (ok < 0) { error_push(err, "failed to print token: %s", strerror(errno)); return; @@ -162,10 +173,14 @@ void token_print(Error* err, Token* t) } start += 1; } - ok = fprintf(stderr, "\"\n\ttype: %s\n}\n", token_type_str[t->type]); - if (ok < 0) { - error_push(err, "failed to print token: %s", strerror(errno)); - return; - } + fprintf(stderr, "\"]\n"); +} + +char* token_str(Token* t) +{ + static __thread char buf[256]; + memcpy(buf, t->start, t->end - t->start); + buf[t->start - t->end] = '\0'; + return buf; } diff --git a/src/tokenizer.h b/src/tokenizer.h index 2bd2017..11f385b 100644 --- a/src/tokenizer.h +++ b/src/tokenizer.h @@ -6,24 +6,30 @@ #include enum token_type { - TOKEN_IDENTIFIER = 0, // [a-zA-Z][a-zA-Z0-9_]* - TOKEN_STRING = 1, // "[^"]*" - TOKEN_INTEGER = 2, // [0-9]+ - TOKEN_FLOATING = 3, // [0-9]+\.[0-9]* - TOKEN_OPERATOR = 4, // + - * / - TOKEN_EOF = 5, - TOKEN_UNKNOWN = 6, - TOKEN_TYPE_COUNT = 7 + TOKEN_IDENTIFIER, // [a-zA-Z][a-zA-Z0-9_]* + TOKEN_STRING, // "[^"]*" + TOKEN_INTEGER, // [0-9]+ + TOKEN_FLOATING, // [0-9]+\.[0-9]* + TOKEN_OPERATOR, // '+' | '-' | '*' | '/' + TOKEN_STATEMENT_END, // ';' + TOKEN_PAREN_OPEN, + TOKEN_PAREN_CLOSE, + TOKEN_EOF, + TOKEN_UNKNOWN, + TOKEN_TYPE_COUNT }; static const char* token_type_str[TOKEN_TYPE_COUNT] = { - "TOKEN_IDENTIFIER", - "TOKEN_STRING", - "TOKEN_INTEGER", - "TOKEN_FLOATING", - "TOKEN_OPERATOR", - "TOKEN_EOF", - "TOKEN_UNKNOWN", + [TOKEN_IDENTIFIER] = "TOKEN_IDENTIFIER", + [TOKEN_STRING] = "TOKEN_STRING", + [TOKEN_INTEGER] = "TOKEN_INTEGER", + [TOKEN_FLOATING] = "TOKEN_FLOATING", + [TOKEN_OPERATOR] = "TOKEN_OPERATOR", + [TOKEN_PAREN_OPEN] = "TOKEN_PAREN_OPEN", + [TOKEN_PAREN_CLOSE] = "TOKEN_PAREN_CLOSE", + [TOKEN_STATEMENT_END] = "TOKEN_STATEMENT_END", + [TOKEN_EOF] = "TOKEN_EOF", + [TOKEN_UNKNOWN] = "TOKEN_UNKNOWN", }; typedef struct token { @@ -42,4 +48,5 @@ void token_print(Error* err, Token* t); int64_t token_eval_int(Error* err, Token* t); double token_eval_float(Error* err, Token* t); +char* token_str(Token* t);