generalize grammar analysis to k>1

src/backends/llk.c

@@ -18,12 +18,18 @@ typedef struct HLLkTable_ {
   HAllocator *mm__;
 } HLLkTable;
 
+
+// XXX adaptation to LL(1), to be removed
+typedef HCharKey HCFToken;
+static const HCFToken end_token = 0x200;
+#define char_token char_key
+
 /* Interface to look up an entry in the parse table. */
 const HCFSequence *h_llk_lookup(const HLLkTable *table, const HCFChoice *x,
                                 HInputStream lookahead)
 {
-  // note the lookahead stream is passed by value, i.e. a copy
+  // note the lookahead stream is passed by value, i.e. a copy.
-  // reading bits from it does not consume them from the input
+  // reading bits from it does not consume them from the real input.
   HCFToken tok;
   uint8_t c = h_read_bits(&lookahead, 8, false);
   if(lookahead.overrun)
@@ -71,15 +77,21 @@ HHashSet *h_predict(HCFGrammar *g, const HCFChoice *A, const HCFSequence *rhs)
 {
   // predict(A -> rhs) = first(rhs) u follow(A)  if "" can be derived from rhs
   // predict(A -> rhs) = first(rhs)              otherwise
-  HHashSet *first_rhs = h_first_sequence(g, rhs->items);
+  const HCFStringMap *first_rhs = h_first_seq(1, g, rhs->items);
-  if(h_sequence_derives_epsilon(g, rhs->items)) {
+  const HCFStringMap *follow_A = h_follow(1, g, A);
   HHashSet *ret = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
-    h_hashset_put_all(ret, first_rhs);
+
-    h_hashset_put_all(ret, h_follow(g, A));
+  h_hashset_put_all(ret, first_rhs->char_branches);
-    return ret;
+  if(first_rhs->end_branch)
-  } else {
+    h_hashset_put(ret, (void *)end_token);
-    return first_rhs;
+
+  if(h_derives_epsilon_seq(g, rhs->items)) {
+    h_hashset_put_all(ret, follow_A->char_branches);
+    if(follow_A->end_branch)
+      h_hashset_put(ret, (void *)end_token);
   }
+
+  return ret;
 }
 
 /* Generate entries for the production "A -> rhs" in the given table row. */
@@ -360,9 +372,9 @@ int test_llk(void)
   printf("generate epsilon: ");
   h_pprint_symbolset(stdout, g, g->geneps, 0);
   printf("first(A) = ");
-  h_pprint_tokenset(stdout, g, h_first_symbol(g, g->start), 0);
+  h_pprint_stringset(stdout, g, h_first(1, g, g->start), 0);
   printf("follow(C) = ");
-  h_pprint_tokenset(stdout, g, h_follow(g, h_desugar(&system_allocator, c)), 0);
+  h_pprint_stringset(stdout, g, h_follow(1, g, h_desugar(&system_allocator, c)), 0);
 
   h_compile(p, PB_LLk, NULL);
 

src/cfgrammar.c

@@ -6,6 +6,10 @@
 #include <ctype.h>
 
 
+// a special map value for use when the map is used to represent a set
+static void * const INSET = (void *)(uintptr_t)1;
+
+
 HCFGrammar *h_cfgrammar_new(HAllocator *mm__)
 {
   HCFGrammar *g = h_new(HCFGrammar, 1);
@@ -15,15 +19,17 @@ HCFGrammar *h_cfgrammar_new(HAllocator *mm__)
   g->arena  = h_new_arena(mm__, 0);     // default blocksize
   g->nts    = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
   g->geneps = NULL;
-  g->first  = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->first  = NULL;
-  g->follow = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->follow = NULL;
+  g->kmax   = 0;    // will be increased as needed by ensure_k
+
+  HCFStringMap *eps = h_stringmap_new(g->arena);
+  h_stringmap_put_epsilon(eps, INSET);
+  g->singleton_epsilon = eps;
 
   return g;
 }
 
-/* Frees the given grammar and associated data.
- * Does *not* free parsers' CFG forms as created by h_desugar.
- */
 void h_cfgrammar_free(HCFGrammar *g)
 {
   HAllocator *mm__ = g->mm__;
@@ -37,10 +43,6 @@ static void collect_nts(HCFGrammar *grammar, HCFChoice *symbol);
 static void collect_geneps(HCFGrammar *grammar);
 
 
-/* Convert 'parser' into CFG representation by desugaring and compiling the set
- * of nonterminals.
- * A NULL return means we are unable to represent the parser as a CFG.
- */
 HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser)
 {
   // convert parser to CFG form ("desugar").
@@ -114,9 +116,26 @@ static void collect_nts(HCFGrammar *grammar, HCFChoice *symbol)
   }
 }
 
+/* Increase g->kmax if needed, allocating enough first/follow slots. */
+static void ensure_k(HCFGrammar *g, size_t k)
+{
+  if(k <= g->kmax) return;
 
-/* Does the given symbol derive the empty string (under g)? */
+  // NB: we don't actually use first/follow[0] but allocate it anyway
-bool h_symbol_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol)
+  // so indices of the array correspond neatly to values of k
+
+  assert(k==1);   // XXX
+  g->first  = h_arena_malloc(g->arena, (k+1)*sizeof(HHashTable *));
+  g->follow = h_arena_malloc(g->arena, (k+1)*sizeof(HHashTable *));
+  g->first[0]  = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->follow[0] = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->first[1]  = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->follow[1] = h_hashtable_new(g->arena, h_eq_ptr, h_hash_ptr);
+  g->kmax = k;
+}
+
+
+bool h_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol)
 {
   assert(g->geneps != NULL);
 
@@ -130,12 +149,11 @@ bool h_symbol_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol)
   }
 }
 
-/* Does the sentential form s derive the empty string? s NULL-terminated. */
+bool h_derives_epsilon_seq(HCFGrammar *g, HCFChoice **s)
-bool h_sequence_derives_epsilon(HCFGrammar *g, HCFChoice **s)
 {
   // return true iff all symbols in s derive epsilon
   for(; *s; s++) {
-    if(!h_symbol_derives_epsilon(g, *s))
+    if(!h_derives_epsilon(g, *s))
       return false;
   }
   return true;
@@ -165,10 +183,10 @@ static void collect_geneps(HCFGrammar *g)
         const HCFChoice *symbol = hte->key;
         assert(symbol->type == HCF_CHOICE);
 
-        // this NT derives epsilon if any of its productions does.
+        // this NT derives epsilon if any one of its productions does.
         HCFSequence **p;
         for(p = symbol->seq; *p != NULL; p++) {
-          if(h_sequence_derives_epsilon(g, (*p)->items)) {
+          if(h_derives_epsilon_seq(g, (*p)->items)) {
             h_hashset_put(g->geneps, symbol);
             break;
           }
@@ -179,44 +197,118 @@ static void collect_geneps(HCFGrammar *g)
 }
 
 
-/* Compute first set of sentential form s. s NULL-terminated. */
+HCFStringMap *h_stringmap_new(HArena *a)
-HHashSet *h_first_sequence(HCFGrammar *g, HCFChoice **s);
-
-/* Compute first set of symbol x. Memoized. */
-HHashSet *h_first_symbol(HCFGrammar *g, const HCFChoice *x)
 {
-  HHashSet *ret;
+  HCFStringMap *m = h_arena_malloc(a, sizeof(HCFStringMap));
+  m->char_branches = h_hashtable_new(a, h_eq_ptr, h_hash_ptr);
+  m->arena = a;
+  return m;
+}
+
+void h_stringmap_put_end(HCFStringMap *m, void *v)
+{
+  m->end_branch = v;
+}
+
+void h_stringmap_put_epsilon(HCFStringMap *m, void *v)
+{
+  m->epsilon_branch = v;
+}
+
+void h_stringmap_put_char(HCFStringMap *m, uint8_t c, void *v)
+{
+  HCFStringMap *node = h_stringmap_new(m->arena);
+  h_stringmap_put_epsilon(node, v);
+  h_hashtable_put(m->char_branches, (void *)char_key(c), node);
+}
+
+void h_stringmap_update(HCFStringMap *m, const HCFStringMap *n)
+{
+  if(n->epsilon_branch)
+    m->epsilon_branch = n->epsilon_branch;
+
+  if(n->end_branch)
+    m->end_branch = n->end_branch;
+
+  h_hashtable_update(m->char_branches, n->char_branches);
+}
+
+void *h_stringmap_get(const HCFStringMap *m, const uint8_t *str, size_t n, bool end)
+{
+  for(size_t i=0; i<n; i++) {
+    if(i==n-1 && end && m->end_branch)
+      return m->end_branch;
+    m = h_hashtable_get(m->char_branches, (void *)char_key(str[i]));
+    if(!m)
+      return NULL;
+  }
+  return m->epsilon_branch;
+}
+
+bool h_stringmap_present(const HCFStringMap *m, const uint8_t *str, size_t n, bool end)
+{
+  return (h_stringmap_get(m, str, n, end) != NULL);
+}
+
+
+const HCFStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x)
+{
+  HCFStringMap *ret;
   HCFSequence **p;
   uint8_t c;
 
-  // memoize via g->first
+  // shortcut: first_0(X) is always {""}
-  assert(g->first != NULL);
+  if(k==0)
-  ret = h_hashtable_get(g->first, x);
+    return g->singleton_epsilon;
-  if(ret != NULL)
+#if 0
-    return ret;
+  // XXX this is bullshit?
-  ret = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
+  // shortcut: first_0(X) is {""} if X derives anything
-  assert(ret != NULL);
+  if(k==0) {
-  h_hashtable_put(g->first, x, ret);
-
     switch(x->type) {
     case HCF_END:
-    h_hashset_put(ret, (void *)end_token);
-    break;
     case HCF_CHAR:
-    h_hashset_put(ret, (void *)char_token(x->chr));
+      return g->singleton_epsilon;
-    break;
     case HCF_CHARSET:
       c=0;
       do {
         if(charset_isset(x->charset, c))
-        h_hashset_put(ret, (void *)char_token(c));
+          return g->singleton_epsilon;
+      } while(c++ < 255);
+      break;
+    // HCF_CHOICE is handled by the general case below
+    }
+  }
+#endif
+
+  // memoize via g->first
+  ensure_k(g, k);
+  ret = h_hashtable_get(g->first[k], x);
+  if(ret != NULL)
+    return ret;
+  ret = h_stringmap_new(g->arena);
+  assert(ret != NULL);
+  h_hashtable_put(g->first[k], x, ret);
+
+  switch(x->type) {
+  case HCF_END:
+    h_stringmap_put_end(ret, INSET);
+    break;
+  case HCF_CHAR:
+    h_stringmap_put_char(ret, x->chr, INSET);
+    break;
+  case HCF_CHARSET:
+    c=0;
+    do {
+      if(charset_isset(x->charset, c)) {
+        h_stringmap_put_char(ret, c, INSET);
+      }
     } while(c++ < 255);
     break;
   case HCF_CHOICE:
     // this is a nonterminal
     // return the union of the first sets of all productions
     for(p=x->seq; *p; ++p)
-      h_hashset_put_all(ret, h_first_sequence(g, (*p)->items));
+      h_stringmap_update(ret, h_first_seq(k, g, (*p)->items));
     break;
   default:  // should not be reached
     assert_message(0, "unknown HCFChoice type");
@@ -225,58 +317,155 @@ HHashSet *h_first_symbol(HCFGrammar *g, const HCFChoice *x)
   return ret;
 }
 
-HHashSet *h_first_sequence(HCFGrammar *g, HCFChoice **s)
+// helpers for h_first_seq, definitions below
-{
+static void first_extend(HCFGrammar *g, HCFStringMap *ret,
-  // the first set of the empty sequence is empty
+                         size_t k, const HCFStringMap *as, HCFChoice **tail);
-  if(*s == NULL)
+static bool is_singleton_epsilon(const HCFStringMap *m);
-    return h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
+static bool any_string_shorter(size_t k, const HCFStringMap *m);
 
-  // first(X tail) = first(X)                if X does not derive epsilon
+const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s)
-  //               = first(X) u first(tail)  otherwise
+{
+  // shortcut: the first set of the empty sequence, for any k, is {""}
+  if(*s == NULL)
+    return g->singleton_epsilon;
+
+  // first_k(X tail) = { a b | a <- first_k(X), b <- first_l(tail), l=k-|a| }
 
   HCFChoice *x = s[0];
   HCFChoice **tail = s+1;
 
-  HHashSet *first_x = h_first_symbol(g, x);
+  const HCFStringMap *first_x = h_first(k, g, x);
-  if(h_symbol_derives_epsilon(g, x)) {
+
-    // return the union of first(x) and first(tail)
+  // shortcut: if first_k(X) = {""}, just return first_k(tail)
-    HHashSet *first_tail = h_first_sequence(g, tail);
+  if(is_singleton_epsilon(first_x))
-    HHashSet *ret = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
+    return h_first_seq(k, g, tail);
-    h_hashset_put_all(ret, first_x);
+
-    h_hashset_put_all(ret, first_tail);
+  // shortcut: if no elements of first_k(X) have length <k, just return first_k(X)
-    return ret;
+  if(!any_string_shorter(k, first_x))
-  } else {
     return first_x;
+
+  // create a new result set and build up the set described above
+  HCFStringMap *ret = h_stringmap_new(g->arena);
+
+  // extend the elements of first_k(X) up to length k from tail
+  first_extend(g, ret, k, first_x, tail);
+
+  return ret;
+}
+
+// add the set { a b | a <- as, b <- first_l(tail), l=k-|a| } to ret
+static void first_extend(HCFGrammar *g, HCFStringMap *ret,
+                         size_t k, const HCFStringMap *as, HCFChoice **tail)
+{
+  if(as->epsilon_branch) {
+    // for a="", add first_k(tail) to ret
+    h_stringmap_update(ret, h_first_seq(k, g, tail));
+  }
+
+  if(as->end_branch) {
+    // for a="$", nothing can follow; just add "$" to ret
+    // NB: formally, "$" is considered to be of length k
+    h_stringmap_put_end(ret, INSET);
+  }
+
+  // iterate over as->char_branches
+  const HHashTable *ht = as->char_branches;
+  for(size_t i=0; i < ht->capacity; i++) {
+    for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) {
+      if(hte->key == NULL)
+        continue;
+      uint8_t c = key_char((HCharKey)hte->key);
+      
+      // follow the branch to find the set { a' | t a' <- as }
+      HCFStringMap *as_ = (HCFStringMap *)hte->value;
+
+      // now the elements of ret that begin with t are given by
+      // t { a b | a <- as_, b <- first_l(tail), l=k-|a|-1 }
+      // so we can use recursion over k
+      HCFStringMap *ret_ = h_stringmap_new(g->arena);
+      h_stringmap_put_char(ret, c, ret_);
+
+      first_extend(g, ret_, k-1, as_, tail);
+    }
   }
 }
 
+static bool is_singleton_epsilon(const HCFStringMap *m)
+{
+  return ( m->epsilon_branch
+           && !m->end_branch
+           && h_hashtable_empty(m->char_branches) );
+}
 
-/* Compute follow set of symbol x. Memoized. */
+static bool any_string_shorter(size_t k, const HCFStringMap *m)
-HHashSet *h_follow(HCFGrammar *g, const HCFChoice *x)
+{
+  if(k==0)
+    return false;
+
+  if(m->epsilon_branch)
+    return true;
+
+  // iterate over m->char_branches
+  const HHashTable *ht = m->char_branches;
+  for(size_t i=0; i < ht->capacity; i++) {
+    for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) {
+      if(hte->key == NULL)
+        continue;
+      HCFStringMap *m_ = hte->value;
+
+      // check subtree for strings shorter than k-1
+      if(any_string_shorter(k-1, m_))
+        return true;
+    }
+  }
+
+  return false;
+}
+
+const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x);
+
+// pointer to functions like h_first_seq
+typedef const HCFStringMap *(*StringSetFun)(size_t, HCFGrammar *, HCFChoice const* const*);
+
+static void stringset_extend(HCFGrammar *g, HCFStringMap *ret,
+                             size_t k, const HCFStringMap *as,
+                             StringSetFun f, HCFChoice const * const *tail);
+
+// h_follow adapted to the signature of StringSetFun
+static inline const HCFStringMap *h_follow_(size_t k, HCFGrammar *g, HCFChoice const* const*s)
+{
+  return h_follow(k, g, *s);
+}
+
+const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x)
 {
   // consider all occurances of X in g
   // the follow set of X is the union of:
   //   {$} if X is the start symbol
   //   given a production "A -> alpha X tail":
-  //   if tail derives epsilon:
+  //     first_k(tail follow_k(A))
-  //     first(tail) u follow(A)
-  //   else:
-  //     first(tail)
 
-  HHashSet *ret;
+  // first_k(tail follow_k(A)) =
+  //   { a b | a <- first_k(tail), b <- follow_l(A), l=k-|a| }
+
+  HCFStringMap *ret;
+
+  // shortcut: follow_0(X) is always {""}
+  if(k==0)
+    return g->singleton_epsilon;
 
   // memoize via g->follow
-  assert(g->follow != NULL);
+  ensure_k(g, k);
-  ret = h_hashtable_get(g->follow, x);
+  ret = h_hashtable_get(g->follow[k], x);
   if(ret != NULL)
     return ret;
-  ret = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr);
+  ret = h_stringmap_new(g->arena);
   assert(ret != NULL);
-  h_hashtable_put(g->follow, x, ret);
+  h_hashtable_put(g->follow[k], x, ret);
 
   // if X is the start symbol, the end token is in its follow set
   if(x == g->start)
-    h_hashset_put(ret, (void *)end_token);
+    h_stringmap_put_end(ret, INSET);
 
   // iterate over g->nts
   size_t i;
@@ -285,7 +474,7 @@ HHashSet *h_follow(HCFGrammar *g, const HCFChoice *x)
     for(hte = &g->nts->contents[i]; hte; hte = hte->next) {
       if(hte->key == NULL)
         continue;
-      const HCFChoice *a = hte->key;        // production's left-hand symbol
+      HCFChoice const * const a = hte->key; // production's left-hand symbol
       assert(a->type == HCF_CHOICE);
 
       // iterate over the productions for A
@@ -297,9 +486,12 @@ HHashSet *h_follow(HCFGrammar *g, const HCFChoice *x)
           if(*s == x) { // occurance found
             HCFChoice **tail = s+1;
 
-            h_hashset_put_all(ret, h_first_sequence(g, tail));
+            const HCFStringMap *first_tail = h_first_seq(k, g, tail);
-            if(h_sequence_derives_epsilon(g, tail))
+
-              h_hashset_put_all(ret, h_follow(g, a));
+            //h_stringmap_update(ret, first_tail);
+
+            // extend the elems of first_k(tail) up to length k from follow(A)
+            stringset_extend(g, ret, k, first_tail, h_follow_, &a);
           }
         }
       }
@@ -309,6 +501,44 @@ HHashSet *h_follow(HCFGrammar *g, const HCFChoice *x)
   return ret;
 }
 
+// add the set { a b | a <- as, b <- f_l(S), l=k-|a| } to ret
+static void stringset_extend(HCFGrammar *g, HCFStringMap *ret,
+                             size_t k, const HCFStringMap *as,
+                             StringSetFun f, HCFChoice const * const *tail)
+{
+  if(as->epsilon_branch) {
+    // for a="", add f_k(tail) to ret
+    h_stringmap_update(ret, f(k, g, tail));
+  }
+
+  if(as->end_branch) {
+    // for a="$", nothing can follow; just add "$" to ret
+    // NB: formally, "$" is considered to be of length k
+    h_stringmap_put_end(ret, INSET);
+  }
+
+  // iterate over as->char_branches
+  const HHashTable *ht = as->char_branches;
+  for(size_t i=0; i < ht->capacity; i++) {
+    for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) {
+      if(hte->key == NULL)
+        continue;
+      uint8_t c = key_char((HCharKey)hte->key);
+      
+      // follow the branch to find the set { a' | t a' <- as }
+      HCFStringMap *as_ = (HCFStringMap *)hte->value;
+
+      // now the elements of ret that begin with t are given by
+      // t { a b | a <- as_, b <- f_l(tail), l=k-|a|-1 }
+      // so we can use recursion over k
+      HCFStringMap *ret_ = h_stringmap_new(g->arena);
+      h_stringmap_put_char(ret, c, ret_);
+
+      stringset_extend(g, ret_, k-1, as_, f, tail);
+    }
+  }
+}
+
 
 static void pprint_char(FILE *f, char c)
 {
@@ -344,7 +574,7 @@ static void pprint_charset(FILE *f, const HCharset cs)
 
   fputc('[', f);
   for(i=0; i<256; i++) {
-    if(charset_isset(cs, i))
+    if(charset_isset(cs, i)) {
       pprint_charset_char(f, i);
 
       // detect ranges
@@ -355,6 +585,7 @@ static void pprint_charset(FILE *f, const HCharset cs)
         pprint_charset_char(f, i);
       }
     }
+  }
   fputc(']', f);
 }
 
@@ -400,6 +631,7 @@ static void pprint_symbol(FILE *f, const HCFGrammar *g, const HCFChoice *x)
     break;
   case HCF_CHARSET:
     pprint_charset(f, x->charset);
+    break;
   default:
     fputs(nonterminal_name(g, x), f);
   }
@@ -507,30 +739,66 @@ void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, in
   fputs("}\n", file);
 }
 
-void h_pprint_tokenset(FILE *file, const HCFGrammar *g, const HHashSet *set, int indent)
+#define BUFSIZE 512
+
+void pprint_stringset_elems(FILE *file, char *prefix, size_t n, const HCFStringMap *set)
+{
+  assert(n < BUFSIZE-4);
+
+  if(set->epsilon_branch) {
+    if(n==0) {
+      fputs("''", file);
+    } else {
+      fputc(',', file);
+      fwrite(prefix, 1, n, file);
+    }
+  }
+
+  if(set->end_branch) {
+    fputc(',', file);
+    fwrite(prefix, 1, n, file);
+    fputc('$', file);
+  }
+
+  // iterate over set->char_branches
+  HHashTable *ht = set->char_branches;
+  size_t i;
+  HHashTableEntry *hte;
+  for(i=0; i < ht->capacity; i++) {
+    for(hte = &ht->contents[i]; hte; hte = hte->next) {
+      if(hte->key == NULL)
+        continue;
+      uint8_t c = key_char((HCharKey)hte->key);
+      HCFStringMap *ends = hte->value;
+
+      size_t n_ = n;
+      switch(c) {
+      case '$':  prefix[n_++] = '\\'; prefix[n_++] = '$'; break;
+      case '"':  prefix[n_++] = '\\'; prefix[n_++] = '"'; break;
+      case '\\': prefix[n_++] = '\\'; prefix[n_++] = '\\'; break;
+      case '\b': prefix[n_++] = '\\'; prefix[n_++] = 'b'; break;
+      case '\t': prefix[n_++] = '\\'; prefix[n_++] = 't'; break;
+      case '\n': prefix[n_++] = '\\'; prefix[n_++] = 'n'; break;
+      case '\r': prefix[n_++] = '\\'; prefix[n_++] = 'r'; break;
+      default:
+        if(isprint(c))
+          prefix[n_++] = c;
+        else
+          n_ += sprintf(prefix+n_, "\\x%.2X", c);
+      }
+
+      pprint_stringset_elems(file, prefix, n_, ends);
+    }
+  }
+}
+
+void h_pprint_stringset(FILE *file, const HCFGrammar *g, const HCFStringMap *set, int indent)
 {
   int j;
   for(j=0; j<indent; j++) fputc(' ', file);
 
-  fputc('[', file);
+  char buf[BUFSIZE];
-
+  fputc('{', file);
-  // iterate over set
+  pprint_stringset_elems(file, buf, 0, set);
-  size_t i;
+  fputs("}\n", file);
-  HHashTableEntry *hte;
-  for(i=0; i < set->capacity; i++) {
-    for(hte = &set->contents[i]; hte; hte = hte->next) {
-      if(hte->key == NULL)
-        continue;
-      HCFToken a = (HCFToken)hte->key;
-      
-      if(a == end_token)
-        fputc('$', file);
-      else if(token_char(a) == '$')
-        fputs("\\$", file);
-      else
-        pprint_char(file, token_char(a));
-    }
-  }
-
-  fputs("]\n", file);
 }

src/cfgrammar.h

@@ -8,20 +8,44 @@ typedef struct HCFGrammar_ {
   HHashSet    *nts;     // HCFChoices, each representing the alternative
                         // productions for one nonterminal
   HHashSet    *geneps;  // set of NTs that can generate the empty string
-  HHashTable  *first;   // memoized first sets of the grammar's symbols
+  HHashTable  **first;  // memoized first sets of the grammar's symbols
-  HHashTable  *follow;  // memoized follow sets of the grammar's NTs
+  HHashTable  **follow; // memoized follow sets of the grammar's NTs
+  size_t      kmax;     // maximum lookahead depth allocated
   HArena      *arena;
   HAllocator  *mm__;
+
+  // constant set containing only the empty string.
+  // this is only a member of HCFGrammar because it needs a pointer to arena.
+  const struct HCFStringMap_ *singleton_epsilon;
 } HCFGrammar;
 
-/* mapping input bytes or end to tokens
+
- * we want to use these, cast to void *, as elements in hashsets
+/* Representing input characters (bytes) in HHashTables.
- * therefore we must avoid 0 as a token value because NULL means "not in set".
+ * To use these as keys, we must avoid 0 as because NULL means "not set".
  */
-typedef uintptr_t HCFToken;
+typedef uintptr_t HCharKey;
-static inline HCFToken char_token(uint8_t c) { return (0x100 | c); }
+static inline HCharKey char_key(uint8_t c) { return (0x100 | c); }
-static inline uint8_t token_char(HCFToken t) { return (0xFF & t); }
+static inline uint8_t key_char(HCharKey k) { return (0xFF & k); }
-static const HCFToken end_token = 0x200;
+
+/* Mapping strings of input tokens to arbitrary values (or serving as a set).
+ * Common prefixes are folded into a tree of HHashTables, branches labeled with
+ * input tokens.
+ * Each path through the tree represents the string along its branches.
+ */
+typedef struct HCFStringMap_ {
+  void *epsilon_branch;         // points to leaf value
+  void *end_branch;             // points to leaf value
+  HHashTable *char_branches;    // maps to inner nodes (HCFStringMaps)
+  HArena *arena;
+} HCFStringMap;
+
+HCFStringMap *h_stringmap_new(HArena *a);
+void h_stringmap_put_end(HCFStringMap *m, void *v);
+void h_stringmap_put_epsilon(HCFStringMap *m, void *v);
+void h_stringmap_put_char(HCFStringMap *m, uint8_t c, void *v);
+void h_stringmap_update(HCFStringMap *m, const HCFStringMap *n);
+void *h_stringmap_get(const HCFStringMap *m, const uint8_t *str, size_t n, bool end);
+bool h_stringmap_present(const HCFStringMap *m, const uint8_t *str, size_t n, bool end);
 
 
 /* Convert 'parser' into CFG representation by desugaring and compiling the set
@@ -36,22 +60,22 @@ HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser);
 void h_cfgrammar_free(HCFGrammar *g);
 
 /* Does the given symbol derive the empty string (under g)? */
-bool h_symbol_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol);
+bool h_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol);
 
 /* Does the sentential form s derive the empty string? s NULL-terminated. */
-bool h_sequence_derives_epsilon(HCFGrammar *g, HCFChoice **s);
+bool h_derives_epsilon_seq(HCFGrammar *g, HCFChoice **s);
 
-/* Compute first set of sentential form s. s NULL-terminated. */
+/* Compute first_k set of symbol x. Memoized. */
-HHashSet *h_first_sequence(HCFGrammar *g, HCFChoice **s);
+const HCFStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x);
 
-/* Compute first set of symbol x. Memoized. */
+/* Compute first_k set of sentential form s. s NULL-terminated. */
-HHashSet *h_first_symbol(HCFGrammar *g, const HCFChoice *x);
+const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s);
 
-/* Compute follow set of symbol x. Memoized. */
+/* Compute follow_k set of symbol x. Memoized. */
-HHashSet *h_follow(HCFGrammar *g, const HCFChoice *x);
+const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x);
 
 
 /* Pretty-printers for grammars and associated data. */
 void h_pprint_grammar(FILE *file, const HCFGrammar *g, int indent);
 void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, int indent);
-void h_pprint_tokenset(FILE *file, const HCFGrammar *g, const HHashSet *set, int indent);
+void h_pprint_stringset(FILE *file, const HCFGrammar *g, const HCFStringMap *set, int indent);

src/datastructures.c

@@ -241,5 +241,6 @@ bool h_eq_ptr(const void *p, const void *q) {
 }
 
 HHashValue h_hash_ptr(const void *p) {
+  // XXX just djbhash it
   return (uintptr_t)p >> 4;
 }

src/t_grammar.c

@@ -28,13 +28,13 @@ static void test_example_1(void) {
   g_check_derives_epsilon_not(g, q);
   g_check_derives_epsilon_not(g, p);
 
-  g_check_firstset_present(g, p, end_token);
+  g_check_firstset_present(1, g, p, "$");
-  g_check_firstset_present(g, p, char_token('x'));
+  g_check_firstset_present(1, g, p, "x");
-  g_check_firstset_present(g, p, char_token('y'));
+  g_check_firstset_present(1, g, p, "y");
 
-  g_check_followset_absent(g, c, end_token);
+  g_check_followset_absent(1, g, c, "$");
-  g_check_followset_absent(g, c, char_token('x'));
+  g_check_followset_absent(1, g, c, "x");
-  g_check_followset_present(g, c, char_token('y'));
+  g_check_followset_present(1, g, c, "y");
 }
 
 void register_grammar_tests(void) {

src/test_suite.h

@@ -84,6 +84,11 @@
   } while(0)
 
 #define g_check_parse_failed(parser, input, inp_len) do {		\
+    int skip = h_compile((HParser *)(parser), PB_LLk, NULL); \
+    if(skip != 0) {	\
+      g_test_message("Backend not applicable, skipping test");	\
+      break;	\
+    }	\
     const HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \
     if (NULL != result) {						\
       g_test_message("Check failed: shouldn't have succeeded, but did"); \
@@ -92,6 +97,11 @@
   } while(0)
 
 #define g_check_parse_ok(parser, input, inp_len, result) do {		\
+    int skip = h_compile((HParser *)(parser), PB_LLk, NULL); \
+    if(skip) {	\
+      g_test_message("Backend not applicable, skipping test");	\
+      break;	\
+    }	\
     HParseResult *res = h_parse(parser, (const uint8_t*)input, inp_len); \
     if (!res) {								\
       g_test_message("Parse failed on line %d", __LINE__);		\
@@ -134,6 +144,23 @@
     }									\
   } while(0)
 
+#define g_check_stringmap_present(table, key) do {			\
+    bool end = (key[strlen(key)-1] == '$');				\
+    if(!h_stringmap_present(table, (uint8_t *)key, strlen(key), end)) {	\
+      g_test_message("Check failed: \"%s\" should have been in map, but wasn't", key); \
+      g_test_fail();							\
+    }									\
+  } while(0)
+
+#define g_check_stringmap_absent(table, key) do {			\
+    bool end = (key[strlen(key)-2] == '$');				\
+    if(h_stringmap_present(table, (uint8_t *)key, strlen(key), end)) {	\
+      g_test_message("Check failed: \"%s\" shouldn't have been in map, but was", key); \
+      g_test_fail();							\
+    }									\
+  } while(0)
+
+
 #define g_check_terminal(grammar, parser) \
   g_check_hashtable_absent(grammar->nts, h_desugar(&system_allocator, parser))
 
@@ -146,17 +173,17 @@
 #define g_check_derives_epsilon_not(grammar, parser) \
   g_check_hashtable_absent(grammar->geneps, h_desugar(&system_allocator, parser))
 
-#define g_check_firstset_present(grammar, parser, token) \
+#define g_check_firstset_present(k, grammar, parser, str) \
-  g_check_hashtable_present(h_first_symbol(grammar, h_desugar(&system_allocator, parser)), (void *)token)
+  g_check_stringmap_present(h_first(k, grammar, h_desugar(&system_allocator, parser)), str)
 
-#define g_check_firstset_absent(grammar, parser, token) \
+#define g_check_firstset_absent(k, grammar, parser, str) \
-  g_check_hashtable_absent(h_first_symbol(grammar, h_desugar(&system_allocator, parser)), (void *)token)
+  g_check_stringmap_absent(h_first(k, grammar, h_desugar(&system_allocator, parser)), str)
 
-#define g_check_followset_present(grammar, parser, token) \
+#define g_check_followset_present(k, grammar, parser, str) \
-  g_check_hashtable_present(h_follow(grammar, h_desugar(&system_allocator, parser)), (void *)token)
+  g_check_stringmap_present(h_follow(k, grammar, h_desugar(&system_allocator, parser)), str)
 
-#define g_check_followset_absent(grammar, parser, token) \
+#define g_check_followset_absent(k, grammar, parser, str) \
-  g_check_hashtable_absent(h_follow(grammar, h_desugar(&system_allocator, parser)), (void *)token)
+  g_check_stringmap_absent(h_follow(k, grammar, h_desugar(&system_allocator, parser)), str)