commit e7f076f7403be64cb516e9b330d388735eebc69d
parent 8bdabb9fb89eb5407b644ff4e2069dea455e3496
Author: Andres Navarro <canavarro82@gmail.com>
Date: Sat, 19 Feb 2011 21:30:19 -0300
Added a reader (including source info tracking & srfi-38 style shared defs/refs).
Diffstat:
11 files changed, 617 insertions(+), 64 deletions(-)
diff --git a/src/Makefile b/src/Makefile
@@ -1,5 +1,5 @@
CC=gcc
-CFLAGS=-O2 -std=c99 -Wall -m32 $(MYCFLAGS)
+CFLAGS=-O2 -g -std=c99 -Wall -m32 $(MYCFLAGS)
RM=rm -f
LIBS=-lm $(MYLIBS)
@@ -7,7 +7,7 @@ MYCFLAGS=
MYLDFLAGS=
MYLIBS=
-CORE_O= kobject.o ktoken.o kpair.o kstring.o ksymbol.o
+CORE_O= kobject.o ktoken.o kpair.o kstring.o ksymbol.o kread.o
KRN_T= klisp
KRN_O= klisp.o
@@ -38,3 +38,4 @@ kpair.o: kpair.c kpair.h kobject.h
kstring.o: kstring.c kstring.h kobject.h
# XXX: kpair.h because of use of list as symbol table
ksymbol.o: ksymbol.c ksymbol.h kobject.h kpair.h
+kread.o: kread.c kread.h kobject.h ktoken.h kpair.h
diff --git a/src/klisp.c b/src/klisp.c
@@ -6,28 +6,28 @@
#include <stdio.h>
+#include <inttypes.h>
+#include <math.h>
+
#include "kobject.h"
-#include "ktoken.h"
+#include "kread.h"
int main(int argc, char *argv[])
{
/*
- ** Simple tokenizer loop
+ ** Simple read loop
*/
- printf("Tokenizer Type Test\n");
-
- ktok_file = stdin;
- ktok_init();
-
- TValue tok = KNIL;
-
- while(!ttiseof(tok)) {
- tok = ktok_read_token();
- if (ttisnil(tok)) {
- /* there was an error */
- break;
- }
- printf("\nToken Type: %s\n", ttname(tok));
+ printf("Read Type Test\n");
+
+ kread_file = stdin;
+ kread_filename = "*STDIN*";
+ kread_init();
+
+ TValue obj = KNIL;
+
+ while(!ttiseof(obj)) {
+ obj = kread();
+ printf("\nRead Object Type: %s\n", ttname(obj));
}
return 0;
diff --git a/src/kpair.c b/src/kpair.c
@@ -12,7 +12,7 @@
#include "kobject.h"
/* TODO: Out of memory errors */
-/* XXX: for now all pairs are mutable */
+/* TEMP: for now all pairs are mutable */
TValue kcons(TValue car, TValue cdr)
{
Pair *new_pair = malloc(sizeof(Pair));
diff --git a/src/kpair.h b/src/kpair.h
@@ -11,15 +11,18 @@
/* TODO: add type assertions */
/* TODO: add more kc[ad]*r combinations */
-#define kcar(p_) (((Pair *)(p_.tv.v.gc))->car)
-#define kcdr(p_) (((Pair *)(p_.tv.v.gc))->cdr)
+#define kcar(p_) (tv2pair(p_)->car)
+#define kcdr(p_) (tv2pair(p_)->cdr)
#define kset_car(p_, v_) (kcar(p_) = v_)
#define kset_cdr(p_, v_) (kcdr(p_) = v_)
-#define kdummy_cons (kcons(KNIL, KNIL))
+#define kdummy_cons() (kcons(KNIL, KNIL))
-/* XXX: for now all pairs are mutable */
+/* TEMP: for now all pairs are mutable */
TValue kcons(TValue, TValue);
+#define kget_source_info(p_) (tv2pair(p_)->si)
+#define kset_source_info(p_, si_) (kget_source_info(p_) = si_)
+
#endif
diff --git a/src/kread.c b/src/kread.c
@@ -0,0 +1,461 @@
+/*
+** kread.c
+** Reader for the Kernel Programming Language
+** See Copyright Notice in klisp.h
+*/
+
+
+/*
+** TODO:
+**
+** - Read mutable/immutable objects (cons function should be a parameter)
+** this is needed because some functions (like load) return immutable objs
+** - Decent error handling mechanism
+**
+*/
+
+#include <stdio.h>
+/* XXX for malloc */
+#include <stdlib.h>
+/* TODO: use a generalized alloc function */
+/* TEMP: for out of mem errors */
+#include <assert.h>
+
+#include "kread.h"
+#include "kobject.h"
+#include "kpair.h"
+#include "ktoken.h"
+
+/* TODO: move to the global state */
+/* TODO: replace the list with a hashtable */
+TValue shared_dict = KNIL_;
+FILE *kread_file = NULL;
+char *kread_filename = NULL;
+
+
+/*
+** Stacks for the read FSM
+**
+** The state stack is never empty while read is in process and
+** selects the action to be performed on the next read token.
+**
+** The data saved in the data stack changes according to state:
+** ST_FIRST_LIST: pair representing the first pair of the list
+** with source info of the '(' token.
+** ST_MIDDLE_LIST, ST_LAST_ILIST: two elements, first below, second on top:
+** - a pair with car: first pair of the list (with source info
+** corrected to car of list) and cdr: source info of the '(' token that
+** started the [i]list.
+** - another pair, that is the last pair of the list so far.
+** ST_PAST_LAST_ILIST: a pair with car: first pair and cdr: source
+** info as above (but no pair with last pair).
+** ST_SHARED_DEF: a pair with car: shared def token and cdr: source
+** info of the shared def token.
+**
+*/
+
+/* TODO: move to the global state */
+typedef enum {
+ ST_READ, ST_SHARED_DEF, ST_LAST_ILIST, ST_PAST_LAST_ILIST,
+ ST_FIRST_LIST, ST_MIDDLE_LIST
+} state_t;
+
+state_t *sstack;
+int sstack_size;
+int sstack_i;
+
+TValue *dstack;
+int dstack_size;
+int dstack_i;
+
+/* TEMP: for now stacks are fixed size, use asserts to check */
+#define STACK_INIT_SIZE 1024
+
+#define push_state(st_) ({ assert(sstack_i < sstack_size); \
+ sstack[sstack_i++] = st_; })
+#define pop_state() (--sstack_i)
+#define get_state() (sstack[sstack_i-1])
+#define clear_state() (sstack_i = 0)
+
+#define push_data(data_) ({ assert(dstack_i < dstack_size); \
+ dstack[dstack_i++] = data_; })
+#define pop_data() (--dstack_i)
+#define get_data() (dstack[dstack_i-1])
+#define clear_data() (dstack_i = 0)
+
+/*
+** Error management
+*/
+TValue kread_error(char *str)
+{
+ /* TODO: Decide on error handling mechanism for reader (& tokenizer) */
+ printf("READ ERROR: %s\n", str);
+ return KEOF;
+}
+
+/*
+** Reader initialization
+*/
+void kread_init()
+{
+ assert(kread_file != NULL);
+ assert(kread_filename != NULL);
+
+ ktok_file = kread_file;
+ ktok_source_info.filename = kread_filename;
+ /* XXX: For now just hardcode it to 8 spaces tab-stop */
+ ktok_source_info.tab_width = 8;
+ ktok_init();
+ ktok_reset_source_info();
+
+ /* XXX: for now use a fixed size for stacks */
+ sstack_size = STACK_INIT_SIZE;
+ clear_state();
+ sstack = malloc(STACK_INIT_SIZE*sizeof(state_t));
+ assert(sstack != NULL);
+
+ dstack_size = STACK_INIT_SIZE;
+ clear_data();
+ dstack = malloc(STACK_INIT_SIZE*sizeof(TValue));
+ assert(dstack != NULL);
+}
+
+/*
+** Shared Reference Management (srfi-38)
+*/
+
+/* This is called after kread to clear the shared alist */
+void clear_shared_dict()
+{
+ shared_dict = KNIL;
+}
+
+TValue try_shared_ref(TValue ref_token)
+{
+ /* TEMP: for now, only allow fixints in shared tokens */
+ int32_t ref_num = ivalue(kcdr(ref_token));
+ TValue tail = shared_dict;
+ while (!ttisnil(tail)) {
+ TValue head = kcar(tail);
+ if (ref_num == ivalue(kcar(head)))
+ return kcdr(head);
+ tail = kcdr(tail);
+ }
+ return kread_error("undefined shared ref found");
+}
+
+TValue try_shared_def(TValue def_token, TValue value)
+{
+ /* TEMP: for now, only allow fixints in shared tokens */
+ int32_t ref_num = ivalue(kcdr(def_token));
+ TValue tail = shared_dict;
+ while (!ttisnil(tail)) {
+ TValue head = kcar(tail);
+ if (ref_num == ivalue(kcar(head)))
+ return kread_error("duplicate shared def found");
+ tail = kcdr(tail);
+ }
+
+ /* XXX: what happens on out of mem? & gc?(inner cons is not rooted) */
+ shared_dict = kcons(kcons(kcdr(def_token), value),
+ shared_dict);
+ return KINERT;
+}
+
+/* This overwrites a previouly made def, it is used in '() */
+/* NOTE: the shared def is guaranteed to exist */
+void change_shared_def(TValue def_token, TValue value)
+{
+ /* TEMP: for now, only allow fixints in shared tokens */
+ int32_t ref_num = ivalue(kcdr(def_token));
+ TValue tail = shared_dict;
+ while (!ttisnil(tail)) {
+ TValue head = kcar(tail);
+ if (ref_num == ivalue(kcar(head))) {
+ kset_cdr(head, value);
+ return;
+ }
+ tail = kcdr(tail);
+ }
+ /* NOTE: can't really happen */
+ return;
+}
+
+/*
+** Reader FSM
+*/
+/* TEMP: For now we'll use just one big function */
+TValue kread_fsm()
+{
+ push_state(ST_READ);
+
+ /* read next token or process obj */
+ bool read_next_token = true;
+ /* the obj just read/completed */
+ TValue obj;
+ /* the source code information of that obj */
+ TValue obj_si;
+
+ while (!(get_state() == ST_READ && !read_next_token)) {
+ if (read_next_token) {
+ TValue tok = ktok_read_token();
+ if (ttispair(tok)) { /* special token */
+ switch (chvalue(kcar(tok))) {
+ case '(': {
+ if (get_state() == ST_PAST_LAST_ILIST)
+ return kread_error("open paren found after "
+ "last element of improper list");
+ TValue np = kdummy_cons();
+ /*
+ ** NOTE: the source info of the '(' is temporarily saved
+ ** in np (later it will be replace by the source info
+ ** of the car of the list
+ */
+ kset_source_info(np, ktok_get_source_info());
+
+ /* update the shared def to point to the new list */
+ /* NOTE: this is necessary for self referrencing lists */
+ /* NOTE: the shared def was already checked for errors */
+ if (get_state() == ST_SHARED_DEF)
+ change_shared_def(kcar(get_data()), np);
+
+ /* start reading elements of the new list */
+ push_state(ST_FIRST_LIST);
+ push_data(np);
+ read_next_token = true;
+ break;
+ }
+ case ')': {
+ switch(get_state()) {
+ case ST_FIRST_LIST: { /* empty list */
+ /*
+ ** Discard the pair in sdata but
+ ** retain the source info
+ ** Return () for processing
+ */
+ TValue fp_with_old_si = get_data();
+ pop_data();
+ pop_state();
+ obj = KNIL;
+ obj_si = kget_source_info(fp_with_old_si);
+ read_next_token = false;
+ break;
+ }
+ case ST_MIDDLE_LIST: /* end of list */
+ case ST_PAST_LAST_ILIST: { /* end of ilist */
+ /* discard info on last pair */
+ pop_data();
+ pop_state();
+ TValue fp_old_si = get_data();
+ pop_data();
+ pop_state();
+ /* list read ok, process it in next iteration */
+ obj = kcar(fp_old_si);
+ obj_si = kcdr(fp_old_si);
+ read_next_token = false;
+ break;
+ }
+ case ST_LAST_ILIST:
+ return kread_error("missing last element in "
+ "improper list");
+ case ST_SHARED_DEF:
+ return kread_error("unmatched closing paren found "
+ "in shared def");
+ case ST_READ:
+ return kread_error("unmatched closing paren found");
+ default:
+ /* shouldn't happen */
+ assert(0);
+ }
+ break;
+ }
+ case '.': {
+ switch(get_state()) {
+ case (ST_MIDDLE_LIST):
+ /* tok ok, read next obj for cdr of ilist */
+ pop_state();
+ push_state(ST_LAST_ILIST);
+ read_next_token = true;
+ break;
+ case ST_FIRST_LIST:
+ return kread_error("missing first element of "
+ "improper list");
+ case ST_LAST_ILIST:
+ case ST_PAST_LAST_ILIST:
+ return kread_error("double dot in improper list");
+ case ST_SHARED_DEF:
+ return kread_error("dot found in shared def");
+ case ST_READ:
+ return kread_error("dot found outside list");
+ default:
+ /* shouldn't happen */
+ assert(0);
+ }
+ break;
+ }
+ case '=': { /* srfi-38 shared def */
+ switch (get_state()) {
+ case ST_SHARED_DEF:
+ return kread_error("shared def found in "
+ "shared def");
+ case ST_PAST_LAST_ILIST:
+ return kread_error("shared def found after "
+ "last element of improper list");
+ default: {
+ TValue res = try_shared_def(tok, KNIL);
+ /* TEMP: while error returns EOF */
+ if (ttiseof(res)) {
+ return res;
+ } else {
+ /* token ok, read defined object */
+ push_state(ST_SHARED_DEF);
+ /* NOTE: save the source info to return it
+ after the defined object is read */
+ push_data(kcons(tok, ktok_get_source_info()));
+ read_next_token = true;
+ }
+ }
+ }
+ break;
+ }
+ case '#': { /* srfi-38 shared ref */
+ switch(get_state()) {
+ case ST_SHARED_DEF:
+ return kread_error("shared ref found in "
+ "shared def");
+ case ST_PAST_LAST_ILIST:
+ return kread_error("shared ref found after "
+ "last element of improper list");
+ default: {
+ TValue res = try_shared_ref(tok);
+ /* TEMP: while error returns EOF */
+ if (ttiseof(res)) {
+ return res;
+ } else {
+ /* token ok, process it in next iteration */
+ obj = tok;
+ /* NOTE: use source info of ref token */
+ obj_si = ktok_get_source_info();
+ read_next_token = false;
+ }
+ }
+ }
+ break;
+ }
+ default:
+ /* shouldn't happen */
+ assert(0);
+ }
+ } else if (ttiseof(tok)) {
+ switch (get_state()) {
+ case ST_READ:
+ /* will exit in next loop */
+ obj = tok;
+ obj_si = ktok_get_source_info();
+ read_next_token = false;
+ break;
+ case ST_FIRST_LIST:
+ case ST_MIDDLE_LIST:
+ return kread_error("EOF found while reading list");
+ case ST_LAST_ILIST:
+ case ST_PAST_LAST_ILIST:
+ return kread_error("EOF found while reading "
+ "improper list");
+ case ST_SHARED_DEF:
+ return kread_error("EOF found in shared def");
+ default:
+ /* shouldn't happen */
+ assert(0);
+ }
+ } else { /* this can only be a complete token */
+ if (get_state() == ST_PAST_LAST_ILIST) {
+ return kread_error("Non paren found after last "
+ "element of improper list");
+ } else {
+ /* token ok, process it in next iteration */
+ obj = tok;
+ obj_si = ktok_get_source_info();
+ read_next_token = false;
+ }
+ }
+ } else { /* if(read_next_token) */
+ /* process the object just read */
+ switch(get_state()) {
+ case ST_FIRST_LIST: {
+ TValue fp = get_data();
+ /* replace source info in fp with the saved one */
+ /* NOTE: the old one will be returned when list is complete */
+ TValue fp_old_si = kget_source_info(fp);
+ kset_source_info(fp, obj_si);
+ kset_car(fp, obj);
+
+ /* continue reading objects of list */
+ push_state(ST_MIDDLE_LIST);
+ pop_data();
+ /* save first & last pair of the (still incomplete) list */
+ push_data(kcons (fp, fp_old_si));
+ push_data(fp);
+ read_next_token = true;
+ break;
+ }
+ case ST_MIDDLE_LIST: {
+ TValue np = kcons(obj, KNIL);
+ kset_source_info(np, obj_si);
+ kset_cdr(get_data(), np);
+ /* replace last pair of the (still incomplete) read next obj */
+ pop_data();
+ push_data(np);
+ read_next_token = true;
+ break;
+ }
+ case ST_LAST_ILIST:
+ pop_state();
+ push_state(ST_PAST_LAST_ILIST);
+ read_next_token = true;
+ break;
+ case ST_SHARED_DEF: {
+ /* shared def completed, continue processing obj */
+ TValue def_si = get_data();
+ pop_state();
+ pop_data();
+
+ change_shared_def(kcar(def_si), obj);
+
+ /* obj = obj; */
+ /* the source info returned is the one from the shared def */
+ obj_si = kcdr(def_si);
+ read_next_token = false;
+ break;
+ }
+ case ST_READ:
+ /* this shouldn't happen, should've exited the while */
+ assert(0);
+ default:
+ /* shouldn't happen */
+ assert(0);
+ }
+ }
+ }
+
+ return obj;
+}
+
+/*
+** Reader Main Function
+*/
+TValue kread()
+{
+ TValue obj;
+
+ /* TEMP: for now assume we are in the repl: reset source info */
+ ktok_reset_source_info();
+
+ obj = kread_fsm();
+
+ /* NOTE: clear after function to allow earlier gc */
+ clear_shared_dict();
+ clear_state();
+ clear_data();
+
+ return obj;
+}
diff --git a/src/kread.h b/src/kread.h
@@ -0,0 +1,23 @@
+/*
+** kread.h
+** Reader for the Kernel Programming Language
+** See Copyright Notice in klisp.h
+*/
+
+#ifndef kread_h
+#define kread_h
+
+#include "kobject.h"
+
+/*
+** Reader interface
+*/
+void kread_init();
+TValue kread();
+
+/* TODO: move this to the global state */
+FILE *kread_file;
+char *kread_filename;
+
+#endif
+
diff --git a/src/kstring.c b/src/kstring.c
@@ -14,7 +14,7 @@
#include "kobject.h"
/* TODO: Out of memory errors */
-/* XXX: for now all strings are mutable */
+/* TEMP: for now all strings are mutable */
TValue kstring_new(const char *buf, uint32_t size)
{
String *new_str = malloc(sizeof(String) + size + 1);
diff --git a/src/kstring.h b/src/kstring.h
@@ -9,7 +9,7 @@
#include "kobject.h"
-/* XXX: for now all strings are mutable */
+/* TEMP: for now all strings are mutable */
TValue kstring_new(const char *, uint32_t);
#endif
diff --git a/src/ksymbol.h b/src/ksymbol.h
@@ -13,7 +13,7 @@
/* TODO: move to global state */
TValue ksymbol_table;
-/* XXX: for now all symbols are interned */
+/* TEMP: for now all symbols are interned */
TValue ksymbol_new(const char *);
#endif
diff --git a/src/ktoken.c b/src/ktoken.c
@@ -12,7 +12,6 @@
** - Support for complete number syntax (exactness, radix, etc)
** - Support for unicode (strings, char and symbols).
** - Error handling
-** - Source code tracking
**
*/
#include <stdio.h>
@@ -20,6 +19,9 @@
#include <stdlib.h>
/* TODO: use a generalized alloc function */
+/* TEMP: for out of mem errors */
+#include <assert.h>
+
#include <string.h>
#include <ctype.h>
#include <stdint.h>
@@ -108,28 +110,34 @@ kcharset ktok_delimiter, ktok_extended, ktok_subsequent;
** instead of creating an otherwise useless special token type
** lparen, rparen and dot are represented as a pair with the corresponding
** char in the car and nil in the cdr.
-** srfi-38 tokens are represented with a boolean in the indicating if it's a
-** defining token and the number in the cdr
+** srfi-38 tokens are also represented with a char in the car indicating if
+** it's a defining token ('=') or a referring token ('#') and the number in
+** the cdr. This way a special token can be easily tested for (with ttispair)
+** and easily classified (with switch(chvalue(kcar(tok)))).
**
*/
TValue ktok_lparen, ktok_rparen, ktok_dot;
/* TODO: move this to the global state */
char *ktok_buffer;
-/* WORKAROUND: for stdin line buffering & reading of EOF */
-bool ktok_seen_eof;
uint32_t ktok_buffer_size;
#define KTOK_BUFFER_INITIAL_SIZE 1024
+/* WORKAROUND: for stdin line buffering & reading of EOF */
+bool ktok_seen_eof;
void ktok_init()
{
+ assert(ktok_file != NULL);
+ assert(ktok_source_info.filename != NULL);
+
/* WORKAROUND: for stdin line buffering & reading of EOF */
ktok_seen_eof = false;
/* string buffer */
- /* XXX: for now use a fixed size */
+ /* TEMP: for now use a fixed size */
ktok_buffer_size = KTOK_BUFFER_INITIAL_SIZE;
- /* TODO: Out of memory errors */
ktok_buffer = malloc(KTOK_BUFFER_INITIAL_SIZE);
+ /* TEMP: while there is no error handling code */
+ assert(ktok_buffer != NULL);
/* Character sets */
kcharset_fill(ktok_alphabetic, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -154,28 +162,74 @@ void ktok_init()
}
/*
-** Underlying stream interface
+** Underlying stream interface & source code location tracking
*/
-int ktok_getc();
-int ktok_peekc();
-
int ktok_getc() {
- /* TODO: add location tracking */
/* WORKAROUND: for stdin line buffering & reading of EOF */
if (ktok_seen_eof) {
return EOF;
} else {
int chi = getc(ktok_file);
- ktok_seen_eof = (chi == EOF);
- return chi;
+ if (chi == EOF) {
+ /* NOTE: eof doesn't change source code location info */
+ ktok_seen_eof = true;
+ return EOF;
+ }
+
+ /* track source code location before returning the char */
+ if (chi == '\t') {
+ /* align column to next tab stop */
+ ktok_source_info.col =
+ (ktok_source_info.col + ktok_source_info.tab_width) -
+ (ktok_source_info.col % ktok_source_info.tab_width);
+ return '\t';
+ } else if (chi == '\n') {
+ ktok_source_info.line++;
+ ktok_source_info.col = 0;
+ return '\n';
+ } else {
+ return chi;
+ }
}
}
int ktok_peekc() {
- int chi = ktok_getc();
- if (chi != EOF)
- ungetc(chi, ktok_file);
- return chi;
+ /* WORKAROUND: for stdin line buffering & reading of EOF */
+ if (ktok_seen_eof) {
+ return EOF;
+ } else {
+ int chi = getc(ktok_file);
+ if (chi == EOF)
+ ktok_seen_eof = true;
+ else
+ ungetc(chi, ktok_file);
+ return chi;
+ }
+}
+
+void ktok_reset_source_info()
+{
+ /* line is 1-base and col is 0-based */
+ ktok_source_info.line = 1;
+ ktok_source_info.col = 0;
+}
+
+void ktok_save_source_info()
+{
+ ktok_source_info.saved_filename = ktok_source_info.filename;
+ ktok_source_info.saved_line = ktok_source_info.line;
+ ktok_source_info.saved_col = ktok_source_info.col;
+}
+
+TValue ktok_get_source_info()
+{
+ /* XXX: what happens on gc? (unrooted objects) */
+ /* NOTE: the filename doesn't contains embedded '\0's */
+ TValue filename_str = kstring_new(ktok_source_info.saved_filename,
+ strlen(ktok_source_info.saved_filename));
+ /* TEMP: for now, lines and column names are fixints */
+ return kcons(filename_str, kcons(i2tv(ktok_source_info.saved_line),
+ i2tv(ktok_source_info.saved_col)));
}
/*
@@ -184,8 +238,9 @@ int ktok_peekc() {
TValue ktok_error(char *str)
{
/* TODO: Decide on error handling mechanism for reader (& tokenizer) */
+ /* TEMP: Use eof object */
printf("TOK ERROR: %s\n", str);
- return KNIL;
+ return KEOF;
}
@@ -213,8 +268,8 @@ TValue ktok_read_token ()
** in any case we save the location of the port
*/
- /* TODO: add location tracking */
- /* SCHEME VERSION (kport-save-loc! kport) */
+ /* save the source info of the start of the next token */
+ ktok_save_source_info();
int chi = ktok_peekc();
@@ -316,14 +371,12 @@ int ktok_read_until_delimiter()
int i = 0;
while (!ktok_check_delimiter()) {
+ /* TODO: allow buffer to grow */
+ assert(i + 1 < ktok_buffer_size);
+
/* NOTE: can't be eof, because eof is a delimiter */
char ch = (char) ktok_getc();
ktok_buffer[i++] = ch;
-
- if (i + 1 == ktok_buffer_size) {
- /* TODO: allow buffer to grow */
- break;
- }
}
ktok_buffer[i] = '\0';
return i;
@@ -331,7 +384,7 @@ int ktok_read_until_delimiter()
/*
** Numbers
-** XXX: for now, only fixints in base 10
+** TEMP: for now, only fixints in base 10
*/
TValue ktok_read_number(bool is_pos)
{
@@ -397,12 +450,10 @@ TValue ktok_read_string()
"while reading a string");
}
}
+ /* TODO: allow buffer to grow */
+ assert(i+1 < ktok_buffer_size);
+
ktok_buffer[i++] = ch;
-
- if (i + 1 == ktok_buffer_size)
- /* TODO: allow buffer to grow */
- return ktok_error("Implementation restriction: "
- "String too long");
}
}
return kstring_new(ktok_buffer, i);
@@ -505,7 +556,7 @@ TValue ktok_read_special()
if (chi == EOF)
return ktok_error("EOF found while reading a srfi-38 token");
}
- return kcons(b2tv(ch == '='), i2tv(res));
+ return kcons(ch2tv(ch), i2tv(res));
}
/* TODO: add real with no primary value and undefined */
default:
@@ -521,6 +572,9 @@ TValue ktok_read_identifier()
int i = 0;
while (!ktok_check_delimiter()) {
+ /* TODO: allow buffer to grow */
+ assert(i+1 < ktok_buffer_size);
+
/* NOTE: can't be eof, because eof is a delimiter */
char ch = (char) ktok_getc();
@@ -529,11 +583,6 @@ TValue ktok_read_identifier()
ktok_buffer[i++] = ch;
else
return ktok_error("Invalid char in identifier");
-
- /* TODO: allow buffer to grow */
- if (i + 1 == ktok_buffer_size) {
- break;
- }
}
ktok_buffer[i] = '\0';
return ksymbol_new(ktok_buffer);
diff --git a/src/ktoken.h b/src/ktoken.h
@@ -14,8 +14,24 @@
*/
void ktok_init();
TValue ktok_read_token();
+void ktok_reset_source_info();
+TValue ktok_get_source_info();
/* TODO: move this to the global state */
FILE *ktok_file;
+/* XXX: for now, lines and column names are fixints */
+typedef struct {
+ char *filename;
+ int32_t tab_width;
+ int32_t line;
+ int32_t col;
+
+ char *saved_filename;
+ int32_t saved_line;
+ int32_t saved_col;
+} ksource_info_t;
+
+ksource_info_t ktok_source_info;
+
#endif
