From e6eb2efa882d70c2511c9d76405e0437a51cd2ed Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Sun, 13 May 2012 01:18:18 +0100
Subject: [PATCH 1/6] Left-recursion caching strategy about halfway
 implemented. Compiles, DOES NOT RUN.

---
 src/hammer.c   | 73 +++++++++++++++++++++++++++++++++++++-------------
 src/hammer.h   |  1 +
 src/internal.h | 24 +++++++++++------
 3 files changed, 71 insertions(+), 27 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index 0e48b85..b306073 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -35,36 +35,70 @@ guint djbhash(const uint8_t *buf, size_t len) {
   return hash;
 }
 
+void setupLR(const parser_t *p, GQueue *stack, LR_t *recDetect) {
+  
+}
+
+parse_result_t* lr_answer(const parser_t *p, parse_state_t *state, LR_t *growable) {
+  return NULL;
+}
+
+parse_result_t* grow(const parser_t *p, parse_state_t *state, head_t *head) {
+  return NULL;
+}
+
 parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
   // TODO(thequux): add caching here.
-  parser_cache_key_t key = {
-    .input_pos = state->input_stream,
-    .parser = parser
-  };
+  parser_cache_key_t *key = a_new(parser_cache_key_t, 1);
+  key->input_pos = state->input_stream;
+  key->parser = parser;
   
   // check to see if there is already a result for this object...
-  if (g_hash_table_contains(state->cache, &key)) {
-    // it exists!
-    // TODO(thequux): handle left recursion case
-    return g_hash_table_lookup(state->cache, &key);
-  } else {
-    // It doesn't exist... run the 
-    parse_result_t *res;
+  if (!g_hash_table_contains(state->cache, key)) {
+    // It doesn't exist, so create a dummy result to cache
+    LR_t *base = a_new(LR_t, 1);
+    base->seed = NULL; base->rule = parser; base->head = NULL;
+    g_queue_push_head(state->input_stream.lr_stack, base);
+    // cache it
+    parser_cache_value_t *dummy = a_new(parser_cache_value_t, 1);
+    dummy->value_type = PC_LEFT; dummy->left = base;
+    g_hash_table_replace(state->cache, key, dummy);
+    // parse the input
+    parse_result_t *tmp_res;
     if (parser)
-      res = parser->fn(parser->env, state);
+      tmp_res = parser->fn(parser->env, state);
     else
-      res = NULL;
+      tmp_res = NULL;
     if (state->input_stream.overrun)
-      res = NULL; // overrun is always failure.
-    // update the cache
-    g_hash_table_replace(state->cache, &key, res);
+      return NULL; // overrun is always failure.
 #ifdef CONSISTENCY_CHECK
-    if (!res) {
+    if (!tmp_res) {
       state->input_stream = INVALID;
-      state->input_stream.input = key.input_pos.input;
+      state->input_stream.input = key->input_pos.input;
     }
 #endif
-    return res;
+    // the base variable has passed equality tests with the cache
+    g_queue_pop_head(state->input_stream.lr_stack);
+    // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
+    if (NULL == base->head) {
+      parser_cache_value_t *right = a_new(parser_cache_value_t, 1);
+      right->value_type = PC_RIGHT; right->right = tmp_res;
+      g_hash_table_replace(state->cache, key, right);
+      return tmp_res;
+    } else {
+      base->seed = tmp_res;
+      parse_result_t *res = lr_answer(parser, state, base);
+      return res;
+    }
+  } else {
+    // it exists!
+    parser_cache_value_t *value = g_hash_table_lookup(state->cache, key);
+    if (PC_LEFT == value->value_type) {
+      setupLR(parser, state->input_stream.lr_stack, value->left);
+      return value->left->seed; // BUG: this might not be correct
+    } else {
+      return value->right;
+    }
   }
 }
 
@@ -489,6 +523,7 @@ parse_result_t* parse(const parser_t* parser, const uint8_t* input, size_t lengt
   parse_state->input_stream.overrun = 0;
   parse_state->input_stream.endianness = BIT_BIG_ENDIAN | BYTE_BIG_ENDIAN;
   parse_state->input_stream.length = length;
+  g_queue_init(parse_state->input_stream.lr_stack);
   parse_state->arena = arena;
   parse_result_t *res = do_parse(parser, parse_state);
   // tear down the parse state. For now, leak like a sieve.
diff --git a/src/hammer.h b/src/hammer.h
index a353641..421c9f3 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -43,6 +43,7 @@ typedef struct input_stream {
   char bit_offset;
   char endianness;
   char overrun;
+  GQueue *lr_stack;
 } input_stream_t;
   
 typedef struct parse_state {
diff --git a/src/internal.h b/src/internal.h
index 8a756c4..e08d5fc 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -29,19 +29,27 @@ typedef struct parser_cache_key {
 } parser_cache_key_t;
 
 typedef enum parser_cache_value_type {
-  PC_BASE,
-  PC_IN_RECURSION,
-  PC_LRESULT,
-  PC_RESULT
+  PC_LEFT,
+  PC_RIGHT
 } parser_cache_value_type_t;
 
+typedef struct head {
+  parser_t *head_parser;
+  GSList *involved_set;
+  GSList *eval_set;
+} head_t;
+
+typedef struct LR {
+  parse_result_t *seed;
+  const parser_t *rule;
+  head_t *head;
+} LR_t;
+
 typedef struct parser_cache_value {
   parser_cache_value_type_t value_type;
   union {
-    int base;
-    parse_result_t *in_recursion;
-    parse_result_t *lresult;
-    parse_result_t *result;
+    LR_t *left;
+    parse_result_t *right;
   };
 } parser_cache_value_t;
 

From 756f0a6573b2453c32321ae1d75af69f7f4e60ab Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Sun, 13 May 2012 01:25:45 +0100
Subject: [PATCH 2/6] moved lr_queue from input_stream to parse_state

---
 src/hammer.c | 8 ++++----
 src/hammer.h | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index b306073..2f2c1c6 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -58,7 +58,7 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
     // It doesn't exist, so create a dummy result to cache
     LR_t *base = a_new(LR_t, 1);
     base->seed = NULL; base->rule = parser; base->head = NULL;
-    g_queue_push_head(state->input_stream.lr_stack, base);
+    g_queue_push_head(state->lr_stack, base);
     // cache it
     parser_cache_value_t *dummy = a_new(parser_cache_value_t, 1);
     dummy->value_type = PC_LEFT; dummy->left = base;
@@ -78,7 +78,7 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
     }
 #endif
     // the base variable has passed equality tests with the cache
-    g_queue_pop_head(state->input_stream.lr_stack);
+    g_queue_pop_head(state->lr_stack);
     // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
     if (NULL == base->head) {
       parser_cache_value_t *right = a_new(parser_cache_value_t, 1);
@@ -94,7 +94,7 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
     // it exists!
     parser_cache_value_t *value = g_hash_table_lookup(state->cache, key);
     if (PC_LEFT == value->value_type) {
-      setupLR(parser, state->input_stream.lr_stack, value->left);
+      setupLR(parser, state->lr_stack, value->left);
       return value->left->seed; // BUG: this might not be correct
     } else {
       return value->right;
@@ -523,7 +523,7 @@ parse_result_t* parse(const parser_t* parser, const uint8_t* input, size_t lengt
   parse_state->input_stream.overrun = 0;
   parse_state->input_stream.endianness = BIT_BIG_ENDIAN | BYTE_BIG_ENDIAN;
   parse_state->input_stream.length = length;
-  g_queue_init(parse_state->input_stream.lr_stack);
+  parse_state->lr_stack = g_queue_new();
   parse_state->arena = arena;
   parse_result_t *res = do_parse(parser, parse_state);
   // tear down the parse state. For now, leak like a sieve.
diff --git a/src/hammer.h b/src/hammer.h
index 421c9f3..dfa7ee0 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -43,13 +43,13 @@ typedef struct input_stream {
   char bit_offset;
   char endianness;
   char overrun;
-  GQueue *lr_stack;
 } input_stream_t;
   
 typedef struct parse_state {
   GHashTable *cache; 
   input_stream_t input_stream;
   arena_t arena;
+  GQueue *lr_stack;
 } parse_state_t;
 
 typedef enum token_type {

From c6f2dcc257b72f88edff823355a6dda278173b30 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Thu, 17 May 2012 13:22:56 +0200
Subject: [PATCH 3/6] Moved parse_state_t into internal.h, had to add some
 things for Warth's recursion. More documentary comments.

---
 src/hammer.c   | 35 ++++++++++++++++++++++++++++--
 src/hammer.h   | 28 +++---------------------
 src/internal.h | 59 +++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 94 insertions(+), 28 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index 94a3998..0b342e6 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -35,7 +35,36 @@ guint djbhash(const uint8_t *buf, size_t len) {
   return hash;
 }
 
-void setupLR(const parser_t *p, GQueue *stack, LR_t *recDetect) {
+parser_cache_value_t* recall(parser_cache_key_t *k, parse_state_t *state) {
+  parser_cache_value_t *cached = g_hash_table_lookup(state->cache, k);
+  head_t *head = g_hash_table_lookup(state->recursion_heads, &(state->input_stream));
+  if (!head) { // No heads found
+    return cached;
+  } else { // Some heads found
+    if (!cached && head->head_parser != k->parser && !g_slist_find(head->involved_set, k->parser)) {
+      // Nothing in the cache, and the key parser is not involved
+      return /* TODO(mlp): figure out what to return here instead of Some(MemoEntry(Right(Failure("dummy", in")))) */ NULL;
+    }
+    if (g_slist_find(head->eval_set, k->parser)) {
+      // Something is in the cache, and the key parser is in the eval set. Remove the key parser from the eval set of the head. 
+      head->eval_set = g_slist_remove_all(head->eval_set, k->parser);
+      parse_result_t *tmp_res = k->parser->fn(k->parser->env, state);
+      if (tmp_res)
+	tmp_res->arena = state->arena;
+      // we know that cached has an entry here, modify it
+      cached->value_type = PC_RIGHT;
+      cached->right = tmp_res;
+    }
+    return cached;
+  }
+}
+
+void setupLR(const parser_t *p, GQueue *stack, LR_t *rec_detect) {
+  if (!rec_detect->head) {
+    head_t *some = g_new(head_t, 1);
+    some->head_parser = p; some->involved_set = NULL; some->eval_set = NULL;
+    rec_detect->head = some;
+  }
   
 }
 
@@ -47,6 +76,7 @@ parse_result_t* grow(const parser_t *p, parse_state_t *state, head_t *head) {
   return NULL;
 }
 
+/* Warth's recursion. Hi Alessandro! */
 parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
   // TODO(thequux): add caching here.
   parser_cache_key_t *key = a_new(parser_cache_key_t, 1);
@@ -67,7 +97,8 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
     parse_result_t *tmp_res;
     if (parser) {
       tmp_res = parser->fn(parser->env, state);
-      tmp_res->arena = state->arena;
+      if (tmp_res)
+	tmp_res->arena = state->arena;
     } else
       tmp_res = NULL;
     if (state->input_stream.overrun)
diff --git a/src/hammer.h b/src/hammer.h
index 8eb901c..62a6f71 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -20,37 +20,15 @@
 #include <glib.h>
 #include <stdint.h>
 #include "allocator.h"
-/* The state of the parser.
- *
- * Members:
- *   input - the entire string being parsed
- *   index - current position in input
- *   length - size of input
- *   cache - a hash table describing the state of the parse, including partial parse_results. It's a hash table from parser_cache_key_t to parse_state_t. 
- *
- */
+
 #define BYTE_BIG_ENDIAN 0x1
 #define BIT_BIG_ENDIAN 0x2
 #define BIT_LITTLE_ENDIAN 0x0
 #define BYTE_LITTLE_ENDIAN 0x0
 
 typedef int bool;
-typedef struct input_stream {
-  // This should be considered to be a really big value type.
-  const uint8_t *input;
-  size_t index;
-  size_t length;
-  char bit_offset;
-  char endianness;
-  char overrun;
-} input_stream_t;
-  
-typedef struct parse_state {
-  GHashTable *cache; 
-  input_stream_t input_stream;
-  arena_t arena;
-  GQueue *lr_stack;
-} parse_state_t;
+
+typedef struct parse_state parse_state_t;
 
 typedef enum token_type {
   TT_NONE,
diff --git a/src/internal.h b/src/internal.h
index 8b6818e..1c88367 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -32,28 +32,85 @@
 #define false 0
 #define true 1
 
+typedef struct input_stream {
+  // This should be considered to be a really big value type.
+  const uint8_t *input;
+  size_t index;
+  size_t length;
+  char bit_offset;
+  char endianness;
+  char overrun;
+} input_stream_t;
+
+/* The state of the parser.
+ *
+ * Members:
+ *   cache - a hash table describing the state of the parse, including partial parse_results. It's a hash table from parser_cache_key_t to parser_cache_value_t. 
+ *   input_stream - the input stream at this state.
+ *   arena - the arena that has been allocated for the parse this state is in.
+ *   lr_stack - used in Warth's recursion
+ *   recursion_heads - used in Warth's recursion
+ *
+ */
+  
+typedef struct parse_state {
+  GHashTable *cache; 
+  input_stream_t input_stream;
+  arena_t arena;
+  GQueue *lr_stack;
+  GHashTable *recursion_heads;
+} parse_state_t;
+
+/* The (location, parser) tuple used to key the cache.
+ */
+
 typedef struct parser_cache_key {
   input_stream_t input_pos;
   const parser_t *parser;
 } parser_cache_key_t;
 
+/* A value in the cache is either of value Left or Right (this is a 
+ * holdover from Scala, which used Either here). Left corresponds to
+ * LR_t, which is for left recursion; Right corresponds to 
+ * parse_result_t.
+ */
+
 typedef enum parser_cache_value_type {
   PC_LEFT,
   PC_RIGHT
 } parser_cache_value_type_t;
 
+
+/* A recursion head.
+ *
+ * Members:
+ *   head_parser -
+ *   involved_set -
+ *   eval_set - 
+ */
 typedef struct head {
-  parser_t *head_parser;
+  const parser_t *head_parser;
   GSList *involved_set;
   GSList *eval_set;
 } head_t;
 
+
+/* A left recursion.
+ *
+ * Members:
+ *   seed -
+ *   rule -
+ *   head -
+ */
 typedef struct LR {
   parse_result_t *seed;
   const parser_t *rule;
   head_t *head;
 } LR_t;
 
+/* Tagged union for values in the cache: either LR's (Left) or 
+ * parse_result_t's (Right).
+ */
 typedef struct parser_cache_value {
   parser_cache_value_type_t value_type;
   union {

From e4593dad41adcfe19287ef70f8f70968b011646a Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Thu, 17 May 2012 14:05:10 +0200
Subject: [PATCH 4/6] Finished recall() and setupLR(). Documenting internal for
 the next asshole.

---
 src/hammer.c   | 12 +++++++++++-
 src/internal.h |  4 ++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index 0b342e6..2500a4a 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -59,13 +59,23 @@ parser_cache_value_t* recall(parser_cache_key_t *k, parse_state_t *state) {
   }
 }
 
+/* Setting up the left recursion. We have the LR for the rule head;
+ * we modify the involved_sets of all LRs in the stack, until we 
+ * see the current parser again.
+ */
+
 void setupLR(const parser_t *p, GQueue *stack, LR_t *rec_detect) {
   if (!rec_detect->head) {
     head_t *some = g_new(head_t, 1);
     some->head_parser = p; some->involved_set = NULL; some->eval_set = NULL;
     rec_detect->head = some;
   }
-  
+  size_t i = 0;
+  LR_t *lr = g_queue_peek_nth(stack, i);
+  while (lr && lr->rule != p) {
+    lr->head = rec_detect->head;
+    lr->head->involved_set = g_slist_prepend(lr->head->involved_set, (gpointer)lr->rule);
+  }
 }
 
 parse_result_t* lr_answer(const parser_t *p, parse_state_t *state, LR_t *growable) {
diff --git a/src/internal.h b/src/internal.h
index 1c88367..1135d22 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -84,8 +84,8 @@ typedef enum parser_cache_value_type {
 /* A recursion head.
  *
  * Members:
- *   head_parser -
- *   involved_set -
+ *   head_parser - the parse rule that started this recursion
+ *   involved_set - A list of rules (parser_t's) involved in the recursion
  *   eval_set - 
  */
 typedef struct head {

From 662cd0e5b42d2be3643a4dfc6e3490eca03d3e7b Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Thu, 17 May 2012 14:45:09 +0200
Subject: [PATCH 5/6] Added lr_answer

---
 src/hammer.c | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index 2500a4a..eb091c9 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -22,6 +22,7 @@
 #include <string.h>
 #include <stdarg.h>
 #include <ctype.h>
+#include <error.h>
 
 #define a_new_(arena, typ, count) ((typ*)arena_malloc((arena), sizeof(typ)*(count)))
 #define a_new(typ, count) a_new_(state->arena, typ, count)
@@ -78,14 +79,35 @@ void setupLR(const parser_t *p, GQueue *stack, LR_t *rec_detect) {
   }
 }
 
-parse_result_t* lr_answer(const parser_t *p, parse_state_t *state, LR_t *growable) {
-  return NULL;
-}
+/* If recall() returns NULL, we need to store a dummy failure in the cache and compute the
+ * future parse. 
+ */
 
 parse_result_t* grow(const parser_t *p, parse_state_t *state, head_t *head) {
   return NULL;
 }
 
+parse_result_t* lr_answer(parser_cache_key_t *k, parse_state_t *state, LR_t *growable) {
+  if (growable->head) {
+    if (growable->head->head_parser != k->parser) {
+      // not the head rule, so not growing
+      return growable->seed;
+    }
+    else {
+      // update cache
+      parser_cache_value_t *v = g_new(parser_cache_value_t, 1);
+      v->value_type = PC_RIGHT; v->right = growable->seed;
+      g_hash_table_replace(state->cache, k, v);
+      if (!growable->seed)
+	return NULL;
+      else
+	return grow(k->parser, state, growable->head);
+    }
+  } else {
+    errx(1, "lrAnswer with no head");
+  }
+}
+
 /* Warth's recursion. Hi Alessandro! */
 parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
   // TODO(thequux): add caching here.
@@ -129,7 +151,7 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
       return tmp_res;
     } else {
       base->seed = tmp_res;
-      parse_result_t *res = lr_answer(parser, state, base);
+      parse_result_t *res = lr_answer(key, state, base);
       return res;
     }
   } else {

From 9803b14ce87db040fe07f5e5e991d785897d17ca Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Thu, 17 May 2012 15:47:14 +0200
Subject: [PATCH 6/6] Warth's recursion finished. action() and and() still
 fail. Time to migrate in TQ's arena work.

---
 src/hammer.c   | 69 +++++++++++++++++++++++++++++++++++++++-----------
 src/hammer.h   |  2 ++
 src/internal.h |  4 +--
 3 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/src/hammer.c b/src/hammer.c
index eb091c9..e832d12 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -38,13 +38,17 @@ guint djbhash(const uint8_t *buf, size_t len) {
 
 parser_cache_value_t* recall(parser_cache_key_t *k, parse_state_t *state) {
   parser_cache_value_t *cached = g_hash_table_lookup(state->cache, k);
-  head_t *head = g_hash_table_lookup(state->recursion_heads, &(state->input_stream));
+  head_t *head = g_hash_table_lookup(state->recursion_heads, k);
   if (!head) { // No heads found
     return cached;
   } else { // Some heads found
     if (!cached && head->head_parser != k->parser && !g_slist_find(head->involved_set, k->parser)) {
       // Nothing in the cache, and the key parser is not involved
-      return /* TODO(mlp): figure out what to return here instead of Some(MemoEntry(Right(Failure("dummy", in")))) */ NULL;
+      parse_result_t *tmp = g_new(parse_result_t, 1);
+      tmp->ast = NULL; tmp->arena = state->arena;
+      parser_cache_value_t *ret = g_new(parser_cache_value_t, 1);
+      ret->value_type = PC_RIGHT; ret->right = tmp;
+      return ret;
     }
     if (g_slist_find(head->eval_set, k->parser)) {
       // Something is in the cache, and the key parser is in the eval set. Remove the key parser from the eval set of the head. 
@@ -83,8 +87,44 @@ void setupLR(const parser_t *p, GQueue *stack, LR_t *rec_detect) {
  * future parse. 
  */
 
-parse_result_t* grow(const parser_t *p, parse_state_t *state, head_t *head) {
-  return NULL;
+parse_result_t* grow(parser_cache_key_t *k, parse_state_t *state, head_t *head) {
+  // Store the head into the recursion_heads
+  g_hash_table_replace(state->recursion_heads, k, head);
+  parser_cache_value_t *old_cached = g_hash_table_lookup(state->cache, k);
+  if (!old_cached || PC_LEFT == old_cached->value_type)
+    errx(1, "impossible match");
+  parse_result_t *old_res = old_cached->right;
+  
+  // reset the eval_set of the head of the recursion at each beginning of growth
+  head->eval_set = head->involved_set;
+  parse_result_t *tmp_res;
+  if (k->parser) {
+    tmp_res = k->parser->fn(k->parser->env, state);
+    if (tmp_res)
+      tmp_res->arena = state->arena;
+  } else
+    tmp_res = NULL;
+  if (tmp_res) {
+    if ((old_res->ast->index < tmp_res->ast->index) || 
+	(old_res->ast->index == tmp_res->ast->index && old_res->ast->bit_offset < tmp_res->ast->bit_offset)) {
+      parser_cache_value_t *v = g_new(parser_cache_value_t, 1);
+      v->value_type = PC_RIGHT; v->right = tmp_res;
+      g_hash_table_replace(state->cache, k, v);
+      return grow(k, state, head);
+    } else {
+      // we're done with growing, we can remove data from the recursion head
+      g_hash_table_remove(state->recursion_heads, k);
+      parser_cache_value_t *cached = g_hash_table_lookup(state->cache, k);
+      if (cached && PC_RIGHT == cached->value_type) {
+	return cached->right;
+      } else {
+	errx(1, "impossible match");
+      }
+    }
+  } else {
+    g_hash_table_remove(state->recursion_heads, k);
+    return old_res;
+  }
 }
 
 parse_result_t* lr_answer(parser_cache_key_t *k, parse_state_t *state, LR_t *growable) {
@@ -101,7 +141,7 @@ parse_result_t* lr_answer(parser_cache_key_t *k, parse_state_t *state, LR_t *gro
       if (!growable->seed)
 	return NULL;
       else
-	return grow(k->parser, state, growable->head);
+	return grow(k, state, growable->head);
     }
   } else {
     errx(1, "lrAnswer with no head");
@@ -110,13 +150,11 @@ parse_result_t* lr_answer(parser_cache_key_t *k, parse_state_t *state, LR_t *gro
 
 /* Warth's recursion. Hi Alessandro! */
 parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
-  // TODO(thequux): add caching here.
   parser_cache_key_t *key = a_new(parser_cache_key_t, 1);
-  key->input_pos = state->input_stream;
-  key->parser = parser;
-  
+  key->input_pos = state->input_stream; key->parser = parser;
+  parser_cache_value_t *m = recall(key, state);
   // check to see if there is already a result for this object...
-  if (!g_hash_table_contains(state->cache, key)) {
+  if (!m) {
     // It doesn't exist, so create a dummy result to cache
     LR_t *base = a_new(LR_t, 1);
     base->seed = NULL; base->rule = parser; base->head = NULL;
@@ -156,12 +194,11 @@ parse_result_t* do_parse(const parser_t* parser, parse_state_t *state) {
     }
   } else {
     // it exists!
-    parser_cache_value_t *value = g_hash_table_lookup(state->cache, key);
-    if (PC_LEFT == value->value_type) {
-      setupLR(parser, state->lr_stack, value->left);
-      return value->left->seed; // BUG: this might not be correct
+    if (PC_LEFT == m->value_type) {
+      setupLR(parser, state->lr_stack, m->left);
+      return m->left->seed; // BUG: this might not be correct
     } else {
-      return value->right;
+      return m->right;
     }
   }
 }
@@ -749,6 +786,8 @@ parse_result_t* parse(const parser_t* parser, const uint8_t* input, size_t lengt
   parse_state->input_stream.endianness = BIT_BIG_ENDIAN | BYTE_BIG_ENDIAN;
   parse_state->input_stream.length = length;
   parse_state->lr_stack = g_queue_new();
+  parse_state->recursion_heads = g_hash_table_new(cache_key_hash,
+						  cache_key_equal);
   parse_state->arena = arena;
   parse_result_t *res = do_parse(parser, parse_state);
   // tear down the parse state. For now, leak like a sieve.
diff --git a/src/hammer.h b/src/hammer.h
index 62a6f71..f0d6863 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -53,6 +53,8 @@ typedef struct parsed_token {
     float flt;
     GSequence *seq; // a sequence of parsed_token_t's
   };
+  size_t index;
+  char bit_offset;
 } parsed_token_t;
 
 /* If a parse fails, the parse result will be NULL.
diff --git a/src/internal.h b/src/internal.h
index 1135d22..3e006a0 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -48,8 +48,8 @@ typedef struct input_stream {
  *   cache - a hash table describing the state of the parse, including partial parse_results. It's a hash table from parser_cache_key_t to parser_cache_value_t. 
  *   input_stream - the input stream at this state.
  *   arena - the arena that has been allocated for the parse this state is in.
- *   lr_stack - used in Warth's recursion
- *   recursion_heads - used in Warth's recursion
+ *   lr_stack - a stack of LRs, used in Warth's recursion
+ *   recursion_heads - table of recursion heads. Keys are parse_cache_key_t's with only an input_state_t (parser can be NULL), values are head_t.
  *
  */