jack-compiler/tokenizer/tokenizer.c

258 lines
5.4 KiB
C
Raw Normal View History

2020-11-30 16:44:22 -05:00
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include "tokenizer.h"
2020-12-21 16:11:23 -05:00
#include "tokenizer-tables.h"
2020-11-30 16:44:22 -05:00
2020-12-21 16:11:23 -05:00
// Data types
2020-11-30 16:44:22 -05:00
typedef enum {
common, charsymbol, space
} CHARTYPE;
typedef struct {
char* str;
int size;
int count;
} STRING;
2020-12-21 16:11:23 -05:00
// String manipulation
STRING* mkstring(int size);
void append(STRING* s, char c);
void freestr(STRING* str);
// Token manipulation;
TOKEN* appendtokenraw(TOKEN* curitem, STRING* token, int definedat, TOKENTYPE type);
2020-12-29 19:27:48 -05:00
TOKEN* appendtoken(TOKEN* curitem, STRING* token, char* file, int definedat);
2020-12-21 16:11:23 -05:00
#define mktoken() (TOKEN*)malloc(sizeof(TOKEN))
// Char types
CHARTYPE getchartype(unsigned char c);
bool iskeyword(STRING* tk);
bool issymbol(STRING* tk);
bool isint(char* str);
bool isintcons(STRING* tk);
bool isidentifier(STRING* tk);
2020-12-29 19:27:48 -05:00
TOKENTYPE gettokentype(STRING* tk, char* file, int definedat);
2020-12-21 16:11:23 -05:00
// Stream handling
void skipln(FILE* input);
void skipmultiln(FILE* input, int* lnscount);
bool handlecomment(FILE* input, int* lnscount);
void readstr(FILE* input, STRING* tmp, int definedat);
// String manipulation
STRING* mkstring(int size) {
STRING* str = (STRING*)malloc(sizeof(STRING));
str->size = sizeof(char) * size; // initial size
str->str = (char*)malloc(str->size);
str->count = 0;
return str;
2020-11-30 16:44:22 -05:00
}
void append(STRING* s, char c) {
int targsize = sizeof(char) * (s->count + 1);
if(s->size <= targsize) {
s->size = targsize * 2;
s->str = (char*)realloc(s->str, s->size);
}
s->str[s->count] = c;
s->count++;
}
2020-12-21 16:11:23 -05:00
void freestr(STRING* str) {
free(str->str);
free(str);
}
// Token manipulation;
TOKEN* appendtokenraw(TOKEN* curitem, STRING* token, int definedat, TOKENTYPE type) {
curitem->token = (char*)malloc(sizeof(char)*token->count);
strcpy(curitem->token, token->str);
curitem->definedat = definedat;
curitem->type = type;
TOKEN* nextitem = mktoken();
curitem->next = nextitem;
token->count = 0;
return nextitem;
}
2021-01-03 14:08:54 -05:00
void freetokens(TOKEN* t) {
free(t->token);
TOKEN* next = t->next;
free(t);
if(next != NULL)
freetokens(next);
}
2020-12-29 19:27:48 -05:00
TOKEN* appendtoken(TOKEN* curitem, STRING* token, char* file, int definedat) {
2020-12-21 16:11:23 -05:00
append(token, '\0');
2020-12-29 19:27:48 -05:00
return appendtokenraw(curitem, token, definedat, gettokentype(token, file, definedat));
2020-12-21 16:11:23 -05:00
}
// Char types
CHARTYPE getchartype(unsigned char c) {
if(isspace(c)) return space;
if(isalnum(c) || c == '_' || c == '"') return common;
return charsymbol;
2020-11-30 16:44:22 -05:00
}
bool iskeyword(STRING* tk) {
2020-12-22 14:00:49 -05:00
return existsinarray(&keywords, tk->str);
2020-11-30 16:44:22 -05:00
}
bool issymbol(STRING* tk) {
if(tk->count != 2)
return false;
2020-12-22 14:00:49 -05:00
return existsinarray(&symbols, tk->str);
2020-11-30 16:44:22 -05:00
}
bool isint(char* str) {
int i = 0;
while(str[i] != '\0') {
if(!isdigit(str[i]))
return false;
i++;
}
return true;
}
bool isintcons(STRING* tk) {
if(!isint(tk->str))
return false;
int val = atoi(tk->str);
2021-01-06 11:31:17 -05:00
return val <= 65535;
2020-11-30 16:44:22 -05:00
}
bool isidentifier(STRING* tk) {
if(isdigit(tk->str[0]))
return false;
int count = tk->count - 1;
for(int i = 0; i < count; i++)
if(!isalnum(tk->str[i]) && tk->str[i] != '_')
return false;
return true;
}
2020-12-29 19:27:48 -05:00
TOKENTYPE gettokentype(STRING* tk, char* file, int definedat) {
2020-11-30 16:44:22 -05:00
if(iskeyword(tk)) return keyword;
if(issymbol(tk)) return symbol;
if(isintcons(tk)) return integer;
if(isidentifier(tk)) return identifier;
2020-12-29 19:27:48 -05:00
eprintf("Unexpected token '%s'; file '%s', line %i\n", tk->str, file, definedat);
2020-11-30 16:44:22 -05:00
exit(1);
}
2020-12-21 16:11:23 -05:00
// Stream handling
2020-11-30 16:44:22 -05:00
void skipln(FILE* input) {
unsigned char c;
while(c = fgetc(input), c != '\0')
if(c == '\n')
break;
}
void skipmultiln(FILE* input, int* lnscount) {
unsigned char c;
while(c = fgetc(input), c != '\0')
if(c == '\n')
(*lnscount)++;
else if(c == '*')
if(fgetc(input) == '/')
break;
}
bool handlecomment(FILE* input, int* lnscount) {
unsigned char nextc = fgetc(input);
if(nextc == '/') {
skipln(input);
(*lnscount)++;
return true;
}
else if(nextc == '*') {
unsigned char furtherc = fgetc(input);
if(furtherc == '*') {
skipmultiln(input, lnscount);
return true;
}
ungetc(furtherc, input);
}
ungetc(nextc, input);
return false;
}
2020-12-21 16:11:23 -05:00
void readstr(FILE* input, STRING* tmp, int definedat) {
2020-11-30 16:44:22 -05:00
unsigned char c;
while(c = fgetc(input), c != '\0') {
if(c == '\n') {
2020-12-21 16:11:23 -05:00
eprintf("Unexpected end of line; line %i", definedat);
2020-11-30 16:44:22 -05:00
exit(1);
}
if(c == '"')
break;
append(tmp, c);
}
append(tmp, '\0');
}
2020-12-29 19:27:48 -05:00
TOKEN* tokenize(char* file) {
2020-12-21 16:11:23 -05:00
TOKEN* head = mktoken();
2020-12-14 14:12:20 -05:00
TOKEN* lastitem = head;
TOKEN* curitem = head;
2020-11-30 16:44:22 -05:00
STRING* tmptoken = mkstring(200);
CHARTYPE lasttype = space;
CHARTYPE curtype;
int lnscount = 1;
2020-12-29 19:27:48 -05:00
FILE* input = fopen(file, "r");
2020-11-30 16:44:22 -05:00
unsigned char c;
2020-12-31 15:38:41 -05:00
while(!feof(input)) {
c = fgetc(input);
if(c == '\n')
2020-11-30 16:44:22 -05:00
lnscount++;
else if(c == '/' && handlecomment(input, &lnscount))
continue;
else if(c == '"') {
if(lasttype != space)
2020-12-29 19:27:48 -05:00
curitem = appendtoken(curitem, tmptoken, file, lnscount);
2020-11-30 16:44:22 -05:00
readstr(input, tmptoken, lnscount);
lastitem = curitem;
curitem = appendtokenraw(curitem, tmptoken, lnscount, string);
lasttype = space;
continue;
}
curtype = getchartype(c);
if(curtype == common) {
if(lasttype == charsymbol) {
lastitem = curitem;
2020-12-29 19:27:48 -05:00
curitem = appendtoken(curitem, tmptoken, file, lnscount);
2020-11-30 16:44:22 -05:00
}
append(tmptoken, c);
} else {
if(lasttype != space){
lastitem = curitem;
2020-12-29 19:27:48 -05:00
curitem = appendtoken(curitem, tmptoken, file, lnscount);
2020-11-30 16:44:22 -05:00
}
if(curtype == charsymbol)
append(tmptoken, c);
}
lasttype = curtype;
}
2021-01-03 14:24:01 -05:00
if(curitem == head) {
eprintf("File '%s' is empty\n", file);
exit(1);
}
2020-11-30 16:44:22 -05:00
lastitem->next = NULL;
free(curitem);
freestr(tmptoken);
fclose(input);
return head;
}