klisp

an open source interpreter for the Kernel Programming Language.
git clone http://git.hanabi.in/repos/klisp.git
Log | Files | Refs | README

ktoken.h (2547B)

      1 /*
      2 ** ktoken.h
      3 ** Tokenizer for the Kernel Programming Language
      4 ** See Copyright Notice in klisp.h
      5 */
      6 
      7 #ifndef ktoken_h
      8 #define ktoken_h
      9 
     10 #include <stdio.h>
     11 #include <ctype.h>
     12 
     13 #include "kobject.h"
     14 #include "kstate.h"
     15 
     16 /*
     17 ** Tokenizer interface
     18 */
     19 void ktok_init(klisp_State *K);
     20 TValue ktok_read_token(klisp_State *K);
     21 
     22 /* return a fresh ilist of the form (filename line . col) */
     23 TValue ktok_get_source_info(klisp_State *K);
     24 void ktok_set_source_info(klisp_State *K, TValue filename, int32_t line,
     25                           int32_t col);
     26 
     27 /* This is needed here to allow cleanup of shared dict from tokenizer */
     28 void clear_shared_dict(klisp_State *K);
     29 
     30 /* These are used in peek-char, peek-u8, read-char & read-u8 */
     31 int ktok_peekc_getc(klisp_State *K, bool peekp);
     32 static inline int ktok_getc(klisp_State *K) { return ktok_peekc_getc(K, false); }
     33 static inline int ktok_peekc(klisp_State *K) { return ktok_peekc_getc(K, true); }
     34 
     35 /* needed by the repl */
     36 void ktok_ignore_whitespace(klisp_State *K);
     37 
     38 /* This is needed for kwrite to check if a symbol has external
     39    representation as an identifier */
     40 /* REFACTOR: think out a better interface to all this */
     41 /*
     42 ** Char set contains macro interface
     43 */
     44 #define KCHS_OCTANT(ch) ((ch) >> 5)
     45 #define KCHS_BIT(ch) (1 << ((ch) & 0x1f))
     46 
     47 /* Each bit correspond to a char in the 0-255 range */
     48 typedef uint32_t kcharset[8];
     49 
     50 extern kcharset ktok_alphabetic, ktok_numeric, ktok_whitespace;
     51 extern kcharset ktok_delimiter, ktok_extended;
     52 extern kcharset ktok_subsequent, ktok_initial;
     53 
     54 #define ktok_is_alphabetic(chi_) kcharset_contains(ktok_alphabetic, chi_)
     55 #define ktok_is_numeric(chi_) kcharset_contains(ktok_numeric, chi_)
     56 
     57 #define ktok_is_whitespace(chi_) kcharset_contains(ktok_whitespace, chi_)
     58 #define ktok_is_delimiter(chi_) ((chi_) == EOF ||                       \
     59                                  kcharset_contains(ktok_delimiter, chi_))
     60 #define ktok_is_initial(chi_) kcharset_contains(ktok_initial, chi_)
     61 #define ktok_is_subsequent(chi_) kcharset_contains(ktok_subsequent, chi_)
     62 
     63 #define kcharset_contains(kch_, ch_)                    \
     64     ({ unsigned char ch__ = (unsigned char) (ch_);      \
     65         kch_[KCHS_OCTANT(ch__)] & KCHS_BIT(ch__); })
     66 
     67 
     68 static inline bool ktok_is_digit(char ch, int32_t radix)
     69 {
     70     ch = tolower(ch);
     71     return (ktok_is_numeric(ch) && (ch - '0') < radix) ||
     72         (ktok_is_alphabetic(ch) && (10 + (ch - 'a')) < radix);
     73 }
     74 
     75 static inline int32_t ktok_digit_value(char ch)
     76 {
     77     ch = tolower(ch);
     78     return (ch <= '9')? ch - '0' : 10 + (ch - 'a');
     79 }
     80 
     81 #endif