tokenizer.c (926B)
1 #include <stdlib.h> 2 #include <assert.h> 3 #include <ctype.h> 4 5 #include "tokenizer.h" 6 7 struct tokenizer { 8 char *buf; 9 size_t cap; 10 size_t len; 11 FILE *src; 12 }; 13 14 bool 15 isdelim(char c) 16 { 17 return isspace(c) || iscntrl(c); 18 } 19 20 tokenizer* 21 new_tokenizer(FILE* src) 22 { 23 tokenizer *t = malloc(sizeof(tokenizer)); 24 t->cap = 1; 25 t->len = 0; 26 t->buf = malloc(sizeof(char)*t->cap); 27 t->src = src; 28 return t; 29 } 30 31 void 32 buf_append(tokenizer *t, char c) 33 { 34 if (t->len == t->cap) 35 t->buf = realloc(t->buf, (t->cap *= 2)*sizeof(char)); 36 assert(t->buf); 37 t->buf[t->len++] = c; 38 } 39 40 symbol 41 next_token(tokenizer* t) 42 { 43 char c; 44 symbol res; 45 46 assert(t->len == 0); 47 48 for (c = fgetc(t->src); !isdelim(c) && c != EOF; c = fgetc(t->src)) 49 buf_append(t, c); 50 buf_append(t, '\0'); 51 52 while (isdelim(c)) c = fgetc(t->src); 53 if (!feof(t->src)) ungetc(c, t->src); 54 55 res = intern(t->buf); 56 t->len = 0; 57 return res; 58 } 59 60 bool 61 eos(tokenizer* t) 62 { 63 return feof(t->src); 64 }