22 lines
377 B
C
22 lines
377 B
C
#ifndef TOKENIZER_H
|
|
#define TOKENIZER_H
|
|
#include <stdio.h>
|
|
|
|
/* tokenizer
|
|
* Simple tool that splits a stream into many tokens. */
|
|
|
|
typedef enum {
|
|
keyword, identifier, symbol, integer, string
|
|
} TOKENTYPE;
|
|
|
|
typedef struct token {
|
|
char* token;
|
|
TOKENTYPE type;
|
|
int definedat;
|
|
struct token* next;
|
|
} TOKEN;
|
|
|
|
TOKEN* tokenize(FILE* input);
|
|
void freetokenlist(TOKEN* list);
|
|
#endif
|