From 85e1ca823560f9969745d26aaebad7c11ebcaff0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=A6gis?= <github@æg.is>
Date: Fri, 28 Sep 2012 17:57:46 +0200
Subject: [PATCH 001/104] Make enum values of HTokenType in hammer.h explicit

---
 src/hammer.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/hammer.h b/src/hammer.h
index a219dfe..4f21e29 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -32,14 +32,15 @@ typedef int bool;
 typedef struct HParseState_ HParseState;
 
 typedef enum HTokenType_ {
-  TT_NONE,
-  TT_BYTES,
-  TT_SINT,
-  TT_UINT,
-  TT_SEQUENCE,
+  // Before you change the explicit values of these, think of the poor bindings ;_;
+  TT_NONE = 1,
+  TT_BYTES = 2,
+  TT_SINT = 4,
+  TT_UINT = 8,
+  TT_SEQUENCE = 16,
+  TT_ERR = 32,
   TT_USER = 64,
-  TT_ERR,
-  TT_MAX
+  TT_MAX = 128 
 } HTokenType;
 
 typedef struct HCountedArray_ {

From aff27779cb065cb2d2ce78d1959a784e257bdb1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=A6gis?= <github@æg.is>
Date: Fri, 28 Sep 2012 17:58:21 +0200
Subject: [PATCH 002/104] Add draft JNI interface

---
 jni/NOTES                                     | 12 ++++
 jni/com/upstandinghackers/hammer/HAction.java |  8 +++
 .../hammer/HParseResult.java                  |  9 +++
 .../hammer/HParsedToken.java                  | 28 ++++++++
 jni/com/upstandinghackers/hammer/HParser.java |  6 ++
 .../upstandinghackers/hammer/HPredicate.java  |  6 ++
 jni/com/upstandinghackers/hammer/Hammer.java  | 66 +++++++++++++++++++
 7 files changed, 135 insertions(+)
 create mode 100644 jni/NOTES
 create mode 100644 jni/com/upstandinghackers/hammer/HAction.java
 create mode 100644 jni/com/upstandinghackers/hammer/HParseResult.java
 create mode 100644 jni/com/upstandinghackers/hammer/HParsedToken.java
 create mode 100644 jni/com/upstandinghackers/hammer/HParser.java
 create mode 100644 jni/com/upstandinghackers/hammer/HPredicate.java
 create mode 100644 jni/com/upstandinghackers/hammer/Hammer.java

diff --git a/jni/NOTES b/jni/NOTES
new file mode 100644
index 0000000..2e6e5bb
--- /dev/null
+++ b/jni/NOTES
@@ -0,0 +1,12 @@
+Compilation:
+javac com/upstandinghackers/hammer/*.java
+
+Conversion to JNI headers:
+find -name "*.class" | sed -e 's/.class$//' | tr '/' '.' | cut -c 3- | xargs javah
+
+Not working:
+enums aren't converted at all, no idea why
+
+TODO:
+Implement the entire JNI side
+Testing
diff --git a/jni/com/upstandinghackers/hammer/HAction.java b/jni/com/upstandinghackers/hammer/HAction.java
new file mode 100644
index 0000000..09aeaaf
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/HAction.java
@@ -0,0 +1,8 @@
+package com.upstandinghackers.hammer;
+
+import java.util.List;
+
+public interface HAction
+{
+    public List<HParsedToken> execute(HParseResult p);
+}
diff --git a/jni/com/upstandinghackers/hammer/HParseResult.java b/jni/com/upstandinghackers/hammer/HParseResult.java
new file mode 100644
index 0000000..ed5c85b
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/HParseResult.java
@@ -0,0 +1,9 @@
+package com.upstandinghackers.hammer;
+
+import java.util.List;
+
+public class HParseResult
+{
+    public native List<HParsedToken> getAst();
+    public native long getBitLength();
+}
diff --git a/jni/com/upstandinghackers/hammer/HParsedToken.java b/jni/com/upstandinghackers/hammer/HParsedToken.java
new file mode 100644
index 0000000..d8a70d6
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/HParsedToken.java
@@ -0,0 +1,28 @@
+package com.upstandinghackers.hammer;
+
+import java.util.List;
+
+public class HParsedToken
+{
+    public native Hammer.HTokenType getTokenType();
+    public native int getIndex();
+    public native byte getBitOffset();
+    public native byte[] getBytesValue();
+    public native long getSIntValue();
+    public native long getUIntValue();
+    public native double getDoubleValue();
+    public native float getFloatValue();
+    public native List<HParsedToken> getSeqValue();
+    public native Object getUserValue();
+
+    native void setTokenType(Hammer.HTokenType type);
+    native void setIndex(int index);
+    native void setBitOffset(byte offset);
+    native void setBytesValue(byte[] value);
+    native void setSIntValue(long value);
+    native void setUIntValue(long value);
+    native void setDoubleValue(double value);
+    native void setFloatValue(float value);
+    native void setSeqValue(List<HParsedToken> value);
+    native void setUserValue(Object value);
+}
diff --git a/jni/com/upstandinghackers/hammer/HParser.java b/jni/com/upstandinghackers/hammer/HParser.java
new file mode 100644
index 0000000..9f2387a
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/HParser.java
@@ -0,0 +1,6 @@
+package com.upstandinghackers.hammer;
+
+public class HParser
+{
+    public native void bindIndirect(HParser inner);
+}
diff --git a/jni/com/upstandinghackers/hammer/HPredicate.java b/jni/com/upstandinghackers/hammer/HPredicate.java
new file mode 100644
index 0000000..9c87875
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/HPredicate.java
@@ -0,0 +1,6 @@
+package com.upstandinghackers.hammer;
+
+public interface HPredicate
+{
+    public boolean apply(HParseResult p);
+}
diff --git a/jni/com/upstandinghackers/hammer/Hammer.java b/jni/com/upstandinghackers/hammer/Hammer.java
new file mode 100644
index 0000000..08b75b1
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/Hammer.java
@@ -0,0 +1,66 @@
+package com.upstandinghackers.hammer;
+
+public class Hammer
+{
+    public final static byte BYTE_BIG_ENDIAN = 0x1;
+    public final static byte BIT_BIG_ENDIAN = 0x2;
+    public final static byte BYTE_LITTLE_ENDIAN = 0x0;
+    public final static byte BIT_LITTLE_ENDIAN = 0x0;
+   
+    public enum HTokenType
+    {
+        TT_NONE(1),
+        TT_BYTES(2),
+        TT_SINT(4),
+        TT_UINT(8),
+        TT_SEQUENCE(16),
+        TT_ERR(32),
+        TT_USER(64),
+        TT_MAX(128);
+
+        private int value;
+        public int getValue() { return this.value; }
+        private HTokenType(int value) { this.value = value; }
+    }
+
+    public static native HParseResult hParse(HParser parser, byte[] input, int length);
+    public static native HParser hToken(byte[] str, int length);
+    public static native HParser hCh(byte c);
+    public static native HParser hChRange(byte from, byte to);
+    public static native HParser hIntRange(HParser p, int lower, int upper);
+    public static native HParser hBits(int len, boolean sign);
+    public static native HParser hInt64();
+    public static native HParser hInt32();
+    public static native HParser hInt16();
+    public static native HParser hInt8();
+    public static native HParser hUInt64();
+    public static native HParser hUInt32();
+    public static native HParser hUInt16();
+    public static native HParser hUInt8();
+    public static native HParser hWhitespace(HParser p);
+    public static native HParser hLeft(HParser p, HParser q);
+    public static native HParser hRight(HParser p, HParser q);
+    public static native HParser hMiddle(HParser p, HParser x, HParser q);
+    public static native HParser hAction(HParser p, HAction a);
+    public static native HParser hIn(byte[] charset, int length);
+    public static native HParser hEndP();
+    public static native HParser hNothingP();
+    public static native HParser hSequence(HParser[] parsers);
+    public static native HParser hChoice(HParser[] parsers);
+    public static native HParser hButNot(HParser p1, HParser p2);
+    public static native HParser hDifference(HParser p1, HParser p2);
+    public static native HParser hXor(HParser p1, HParser p2);
+    public static native HParser hMany(HParser p);
+    public static native HParser hMany1(HParser p);
+    public static native HParser hRepeatN(HParser p, int n);
+    public static native HParser hOptional(HParser p);
+    public static native HParser hIgnore(HParser p);
+    public static native HParser hSepBy(HParser p, HParser sep);
+    public static native HParser hSepBy1(HParser p, HParser sep);
+    public static native HParser hEpsilonP();
+    public static native HParser hLengthValue(HParser length, HParser value);
+    public static native HParser hAttrBool(HParser p, HPredicate pred);
+    public static native HParser hAnd(HParser p);
+    public static native HParser hNot(HParser p);
+    public static native HParser hIndirect();
+}

From 4d0ec07c1a9b34f5e11df598877c36fca1516164 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 7 Jan 2013 23:44:26 +0100
Subject: [PATCH 003/104] make TT_USER > TT_ERR, remove TT_MAX

---
 src/Makefile     |  1 +
 src/hammer.h     |  3 +--
 src/t_misc.c     | 16 ++++++++++++++++
 src/test_suite.c |  2 ++
 4 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 src/t_misc.c

diff --git a/src/Makefile b/src/Makefile
index 128de05..47e136d 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -45,6 +45,7 @@ TESTS := t_benchmark.o \
 	 t_bitreader.o \
 	 t_bitwriter.o \
 	 t_parser.o \
+	 t_misc.o \
 	 test_suite.o
 
 OUTPUTS := libhammer.a \
diff --git a/src/hammer.h b/src/hammer.h
index 0791769..5a0c625 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -43,9 +43,8 @@ typedef enum HTokenType_ {
   TT_SINT,
   TT_UINT,
   TT_SEQUENCE,
-  TT_USER = 64,
   TT_ERR,
-  TT_MAX
+  TT_USER = 64
 } HTokenType;
 
 typedef struct HCountedArray_ {
diff --git a/src/t_misc.c b/src/t_misc.c
new file mode 100644
index 0000000..5c08a2e
--- /dev/null
+++ b/src/t_misc.c
@@ -0,0 +1,16 @@
+#include <glib.h>
+#include "test_suite.h"
+#include "hammer.h"
+
+static void test_tt_user(void) {
+  g_check_cmpint(TT_USER, >, TT_NONE);
+  g_check_cmpint(TT_USER, >, TT_BYTES);
+  g_check_cmpint(TT_USER, >, TT_SINT);
+  g_check_cmpint(TT_USER, >, TT_UINT);
+  g_check_cmpint(TT_USER, >, TT_SEQUENCE);
+  g_check_cmpint(TT_USER, >, TT_ERR);
+}
+
+void register_misc_tests(void) {
+  g_test_add_func("/core/misc/tt_user", test_tt_user);
+}
diff --git a/src/test_suite.c b/src/test_suite.c
index 8d2913a..e01d020 100644
--- a/src/test_suite.c
+++ b/src/test_suite.c
@@ -22,6 +22,7 @@
 extern void register_bitreader_tests();
 extern void register_bitwriter_tests();
 extern void register_parser_tests();
+extern void register_misc_tests();
 extern void register_benchmark_tests();
 
 int main(int argc, char** argv) {
@@ -31,6 +32,7 @@ int main(int argc, char** argv) {
   register_bitreader_tests();
   register_bitwriter_tests();
   register_parser_tests();
+  register_misc_tests();
   register_benchmark_tests();
 
   g_test_run();

From bc486aa8401a0e5b23c0c3cd1c253ca6a3480e3b Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 8 Jan 2013 00:19:23 +0100
Subject: [PATCH 004/104] fix base64 example to parse more than the 2- and
 1-byte special cases

---
 examples/base64.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/examples/base64.c b/examples/base64.c
index beb2484..8ebe77b 100644
--- a/examples/base64.c
+++ b/examples/base64.c
@@ -19,9 +19,13 @@ void init_parser(void)
         h_ch('Y'), h_ch('c'), h_ch('g'), h_ch('k'), h_ch('o'), h_ch('s'),
         h_ch('w'), h_ch('0'), h_ch('4'), h_ch('8'), NULL);
     const HParser *bsfdig_2bit = h_choice(h_ch('A'), h_ch('Q'), h_ch('g'), h_ch('w'), NULL);
+    const HParser *base64_3 = h_repeat_n(bsfdig, 4);
     const HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL);
     const HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL);
-    const HParser *base64 = h_choice(base64_2, base64_1, NULL);
+    const HParser *base64 = h_sequence(h_many(base64_3),
+                                       h_optional(h_choice(base64_2,
+                                                           base64_1, NULL)),
+                                       NULL);
         // why does this parse "A=="?!
         // why does this parse "aaA=" but not "aA=="?!
 

From 87b5e668c327d55781b8960df59a0c5a177b56ee Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 8 Jan 2013 00:20:34 +0100
Subject: [PATCH 005/104] remove comments about bugs that no longer exist

---
 examples/base64.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/base64.c b/examples/base64.c
index 8ebe77b..7692532 100644
--- a/examples/base64.c
+++ b/examples/base64.c
@@ -26,8 +26,6 @@ void init_parser(void)
                                        h_optional(h_choice(base64_2,
                                                            base64_1, NULL)),
                                        NULL);
-        // why does this parse "A=="?!
-        // why does this parse "aaA=" but not "aA=="?!
 
     document = base64;
 }

From 177281289c45c8021fd256190f1be009feabd2b9 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 8 Jan 2013 00:24:13 +0100
Subject: [PATCH 006/104] use h_in for restricted base64 digit cases

---
 examples/base64.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/examples/base64.c b/examples/base64.c
index 7692532..6c4db9e 100644
--- a/examples/base64.c
+++ b/examples/base64.c
@@ -14,11 +14,8 @@ void init_parser(void)
     const HParser *equals = h_ch('=');
 
     const HParser *bsfdig = h_choice(alpha, digit, plus, slash, NULL);
-    const HParser *bsfdig_4bit = h_choice(
-        h_ch('A'), h_ch('E'), h_ch('I'), h_ch('M'), h_ch('Q'), h_ch('U'),
-        h_ch('Y'), h_ch('c'), h_ch('g'), h_ch('k'), h_ch('o'), h_ch('s'),
-        h_ch('w'), h_ch('0'), h_ch('4'), h_ch('8'), NULL);
-    const HParser *bsfdig_2bit = h_choice(h_ch('A'), h_ch('Q'), h_ch('g'), h_ch('w'), NULL);
+    const HParser *bsfdig_4bit = h_in((uint8_t *)"AEIMQUYcgkosw048", 16);
+    const HParser *bsfdig_2bit = h_in((uint8_t *)"AQgw", 4);
     const HParser *base64_3 = h_repeat_n(bsfdig, 4);
     const HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL);
     const HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL);

From 2bc03e4e9ffa8f6f2ee7f34cf4d993bde29c9074 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 9 Jan 2013 14:48:02 +0100
Subject: [PATCH 007/104] allow h_whitespace(p) to succeed on end of input if p
 succeeds

---
 src/parsers/whitespace.c | 2 +-
 src/t_parser.c           | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/parsers/whitespace.c b/src/parsers/whitespace.c
index 8e009a4..45c7dcb 100644
--- a/src/parsers/whitespace.c
+++ b/src/parsers/whitespace.c
@@ -8,7 +8,7 @@ static HParseResult* parse_whitespace(void* env, HParseState *state) {
     bak = state->input_stream;
     c = h_read_bits(&state->input_stream, 8, false);
     if (state->input_stream.overrun)
-      return NULL;
+      break;
   } while (isspace(c));
   state->input_stream = bak;
   return h_do_parse((HParser*)env, state);
diff --git a/src/t_parser.c b/src/t_parser.c
index ff05e71..b1f9b63 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -110,12 +110,17 @@ static void test_float32(void) {
 
 static void test_whitespace(void) {
   const HParser *whitespace_ = h_whitespace(h_ch('a'));
+  const HParser *whitespace_end = h_whitespace(h_end_p());
 
   g_check_parse_ok(whitespace_, "a", 1, "u0x61");
   g_check_parse_ok(whitespace_, " a", 2, "u0x61");
   g_check_parse_ok(whitespace_, "  a", 3, "u0x61");
   g_check_parse_ok(whitespace_, "\ta", 2, "u0x61");
   g_check_parse_failed(whitespace_, "_a", 2);
+
+  g_check_parse_ok(whitespace_end, "", 0, "NULL");
+  g_check_parse_ok(whitespace_end, "  ", 2, "NULL");
+  g_check_parse_failed(whitespace_end, "  x", 3);
 }
 
 static void test_left(void) {

From 0497cdb08b034dc30255ebd41316d0d5132df2d6 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 00:08:07 +0100
Subject: [PATCH 008/104] add trivial left-recursion test case

---
 src/t_parser.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/t_parser.c b/src/t_parser.c
index b1f9b63..6bbc409 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -365,6 +365,17 @@ static void test_not(void) {
   g_check_parse_ok(not_2, "a++b", 4, "(u0x61 <2b.2b> u0x62)");
 }
 
+static void test_leftrec(void) {
+  const HParser *a_ = h_ch('a');
+
+  HParser *lr_ = h_indirect();
+  h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL));
+
+  g_check_parse_ok(lr_, "a", 1, "(u0x61)");
+  g_check_parse_ok(lr_, "aa", 2, "(u0x61 u0x61)");
+  g_check_parse_ok(lr_, "aaa", 3, "((u0x61 u0x61) u0x61)");
+}
+
 void register_parser_tests(void) {
   g_test_add_func("/core/parser/token", test_token);
   g_test_add_func("/core/parser/ch", test_ch);
@@ -406,4 +417,5 @@ void register_parser_tests(void) {
   g_test_add_func("/core/parser/and", test_and);
   g_test_add_func("/core/parser/not", test_not);
   g_test_add_func("/core/parser/ignore", test_ignore);
+  g_test_add_func("/core/parser/leftrec", test_leftrec);
 }

From ce88a3f49d82a2c2e03e7061baa43eea33dea617 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 00:39:15 +0100
Subject: [PATCH 009/104] initialize involved_set to an empty HList, not NULL

---
 src/backends/packrat.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/backends/packrat.c b/src/backends/packrat.c
index d05129d..25a0966 100644
--- a/src/backends/packrat.c
+++ b/src/backends/packrat.c
@@ -77,7 +77,9 @@ HParserCacheValue* recall(HParserCacheKey *k, HParseState *state) {
 void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) {
   if (!rec_detect->head) {
     HRecursionHead *some = a_new(HRecursionHead, 1);
-    some->head_parser = p; some->involved_set = NULL; some->eval_set = NULL;
+    some->head_parser = p;
+    some->involved_set = h_slist_new(state->arena);
+    some->eval_set = NULL;
     rec_detect->head = some;
   }
   assert(state->lr_stack->head != NULL);
@@ -101,7 +103,7 @@ HParseResult* grow(HParserCacheKey *k, HParseState *state, HRecursionHead *head)
   HParseResult *old_res = old_cached->right->result;
   
   // reset the eval_set of the head of the recursion at each beginning of growth
-  head->eval_set = head->involved_set;
+  head->eval_set = head->involved_set;  // BUG: this must be a copy
   HParseResult *tmp_res = perform_lowlevel_parse(state, k->parser);
 
   if (tmp_res) {

From dffe47bf5775f35bc13cdd9bea856ac3e9cbdfbb Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 00:40:19 +0100
Subject: [PATCH 010/104] initialize eval_set to a proper copy of involved_set

---
 src/backends/packrat.c |  2 +-
 src/datastructures.c   | 20 ++++++++++++++++++++
 src/internal.h         |  1 +
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/backends/packrat.c b/src/backends/packrat.c
index 25a0966..c67c3e4 100644
--- a/src/backends/packrat.c
+++ b/src/backends/packrat.c
@@ -103,7 +103,7 @@ HParseResult* grow(HParserCacheKey *k, HParseState *state, HRecursionHead *head)
   HParseResult *old_res = old_cached->right->result;
   
   // reset the eval_set of the head of the recursion at each beginning of growth
-  head->eval_set = head->involved_set;  // BUG: this must be a copy
+  head->eval_set = h_slist_copy(head->involved_set);
   HParseResult *tmp_res = perform_lowlevel_parse(state, k->parser);
 
   if (tmp_res) {
diff --git a/src/datastructures.c b/src/datastructures.c
index b1e4f75..3d94804 100644
--- a/src/datastructures.c
+++ b/src/datastructures.c
@@ -41,6 +41,26 @@ HSlist* h_slist_new(HArena *arena) {
   return ret;
 }
 
+HSlist* h_slist_copy(HSlist *slist) {
+  HSlist *ret = h_slist_new(slist->arena);
+  HSlistNode *head = slist->head;
+  HSlistNode *tail;
+  if (head != NULL) {
+    h_slist_push(ret, head->elem);
+    tail = ret->head;
+    head = head->next;
+  }
+  while (head != NULL) {
+    // append head item to tail in a new node
+    HSlistNode *node = h_arena_malloc(slist->arena, sizeof(HSlistNode));
+    node->elem = head->elem;
+    node->next = NULL;
+    tail = tail->next = node;
+    head = head->next;
+  }
+  return ret;
+}
+
 void* h_slist_pop(HSlist *slist) {
   HSlistNode *head = slist->head;
   if (!head)
diff --git a/src/internal.h b/src/internal.h
index 67ecb22..0dcf857 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -209,6 +209,7 @@ HCountedArray *h_carray_new(HArena * arena);
 void h_carray_append(HCountedArray *array, void* item);
 
 HSlist* h_slist_new(HArena *arena);
+HSlist* h_slist_copy(HSlist *slist);
 void* h_slist_pop(HSlist *slist);
 void h_slist_push(HSlist *slist, void* item);
 bool h_slist_find(HSlist *slist, const void* item);

From 63dbf83b4de91ae22c6fe5c0b08b1831bc7c86fd Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 01:04:04 +0100
Subject: [PATCH 011/104] fix a loop in setupLR

---
 src/backends/packrat.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/backends/packrat.c b/src/backends/packrat.c
index c67c3e4..cc2a9db 100644
--- a/src/backends/packrat.c
+++ b/src/backends/packrat.c
@@ -83,10 +83,12 @@ void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) {
     rec_detect->head = some;
   }
   assert(state->lr_stack->head != NULL);
-  HLeftRec *lr = state->lr_stack->head->elem;
-  while (lr && lr->rule != p) {
+  HSlistNode *head = state->lr_stack->head;
+  HLeftRec *lr;
+  while (head && (lr = head->elem)->rule != p) {
     lr->head = rec_detect->head;
     h_slist_push(lr->head->involved_set, (void*)lr->rule);
+    head = head->next;
   }
 }
 

From 4e3084739e301a1412f59a1991d64b502e159c17 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 01:09:04 +0100
Subject: [PATCH 012/104] fix expected value of first leftrec test case

---
 src/t_parser.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_parser.c b/src/t_parser.c
index 6bbc409..daca1a3 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -371,7 +371,7 @@ static void test_leftrec(void) {
   HParser *lr_ = h_indirect();
   h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL));
 
-  g_check_parse_ok(lr_, "a", 1, "(u0x61)");
+  g_check_parse_ok(lr_, "a", 1, "u0x61");
   g_check_parse_ok(lr_, "aa", 2, "(u0x61 u0x61)");
   g_check_parse_ok(lr_, "aaa", 3, "((u0x61 u0x61) u0x61)");
 }

From 445913610da0734630b0affd512e1d6024b27284 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 01:25:37 +0100
Subject: [PATCH 013/104] parse whole input and allow surrounding whitespace in
 base64 example

---
 examples/base64.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/base64.c b/examples/base64.c
index 6c4db9e..cdb088e 100644
--- a/examples/base64.c
+++ b/examples/base64.c
@@ -24,7 +24,7 @@ void init_parser(void)
                                                            base64_1, NULL)),
                                        NULL);
 
-    document = base64;
+    document = h_sequence(h_whitespace(base64), h_whitespace(h_end_p()), NULL);
 }
 
 

From 2af0f9c47cdff7cdbc86fd4f22974974e08d4111 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 22:33:58 +0100
Subject: [PATCH 014/104] start a variant of the base64 example with semantic
 actions

---
 examples/Makefile      |  12 ++--
 examples/base64_sem1.c | 142 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 149 insertions(+), 5 deletions(-)
 create mode 100644 examples/base64_sem1.c

diff --git a/examples/Makefile b/examples/Makefile
index 6a054ca..a3be0ce 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -2,7 +2,9 @@
 OUTPUTS := dns.o \
 	   dns \
 	   base64.o \
-	   base64
+	   base64 \
+	   base64_sem1.o \
+	   base64_sem1
 
 TOPLEVEL := ../
 
@@ -12,7 +14,7 @@ LDFLAGS += $(pkg-config --libs glib-2.0)
 
 
 
-all: dns base64
+all: dns base64 base64_sem1
 
 dns: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
 dns: dns.o rr.o dns_common.o
@@ -24,8 +26,8 @@ rr.o: ../src/hammer.h rr.h dns_common.h
 
 dns_common.o: ../src/hammer.h dns_common.h
 
-base64: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
-base64: base64.o
+base64%: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
+base64%: base64%.o
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
 
-base64.o: ../src/hammer.h
+base64%.o: ../src/hammer.h
diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
new file mode 100644
index 0000000..83efc64
--- /dev/null
+++ b/examples/base64_sem1.c
@@ -0,0 +1,142 @@
+#include "../src/hammer.h"
+#include <assert.h>
+
+
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+
+
+///
+// Semantic action helpers.
+// These might be candidates for inclusion in the library.
+///
+
+// The action equivalent of h_ignore.
+const HParsedToken *act_ignore(const HParseResult *p)
+{
+    return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+const HParsedToken *act_index0(const HParseResult *p)
+{
+    return act_index(0, p);
+}
+
+
+///
+// Semantic actions for the grammar below, each corresponds to an "ARULE".
+// They must be named act_<rulename>.
+///
+
+const HParsedToken *act_bsfdig(const HParseResult *p)
+{
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+
+    assert(p->ast->token_type == TT_UINT);
+    uint8_t c = p->ast->uint;
+
+    res->token_type = TT_UINT;
+    if(c >= 0x40 && c <= 0x5A) // A-Z
+        res->uint = c - 0x41;
+    else if(c >= 0x60 && c <= 0x7A) // a-z
+        res->uint = c - 0x61 + 26;
+    else if(c >= 0x30 && c <= 0x39) // 0-9
+        res->uint = c - 0x30 + 52;
+    else if(c == '+')
+        res->uint = 62;
+    else if(c == '/')
+        res->uint = 63;
+
+    return res;
+}
+
+#define act_bsfdig_4bit  act_bsfdig
+#define act_bsfdig_2bit  act_bsfdig
+
+#define act_equals       act_ignore
+#define act_ws           act_ignore
+
+#define act_document     act_index0
+
+
+///
+// Set up the parser with the grammar to be recognized.
+///
+
+const HParser *init_parser(void)
+{
+    // CORE
+    H_RULE (digit,   h_ch_range(0x30, 0x39));
+    H_RULE (alpha,   h_choice(h_ch_range(0x41, 0x5a), h_ch_range(0x61, 0x7a), NULL));
+    H_RULE (space,   h_in((uint8_t *)" \t\n\r\f\v", 6));
+
+    // AUX.
+    H_RULE (plus,    h_ch('+'));
+    H_RULE (slash,   h_ch('/'));
+    H_ARULE(equals,  h_ch('='));
+
+    H_ARULE(bsfdig,       h_choice(alpha, digit, plus, slash, NULL));
+    H_ARULE(bsfdig_4bit,  h_in((uint8_t *)"AEIMQUYcgkosw048", 16));
+    H_ARULE(bsfdig_2bit,  h_in((uint8_t *)"AQgw", 4));
+    H_RULE (base64_3,     h_repeat_n(bsfdig, 4));
+    H_RULE (base64_2,     h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL));
+    H_RULE (base64_1,     h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL));
+    H_RULE (base64,       h_sequence(h_many(base64_3),
+                                     h_optional(h_choice(base64_2,
+                                                         base64_1, NULL)),
+                                     NULL));
+
+    H_ARULE(ws,           h_many(space));
+    H_ARULE(document,     h_sequence(ws, base64, ws, h_end_p(), NULL));
+
+    return document;
+}
+
+
+///
+// Main routine: print input, parse, print result, return success/failure.
+///
+
+#include <stdio.h>
+
+int main(int argc, char **argv)
+{
+    uint8_t input[102400];
+    size_t inputsize;
+    const HParser *parser;
+    const HParseResult *result;
+
+    parser = init_parser();
+
+    inputsize = fread(input, 1, sizeof(input), stdin);
+    fprintf(stderr, "inputsize=%lu\ninput=", inputsize);
+    fwrite(input, 1, inputsize, stderr);
+    result = h_parse(parser, input, inputsize);
+
+    if(result) {
+        fprintf(stderr, "parsed=%lld bytes\n", result->bit_length/8);
+        h_pprint(stdout, result->ast, 0, 0);
+        return 0;
+    } else {
+        return 1;
+    }
+}

From 54bd5a4a3832a2f4c85cc34799022f61d0ddcf86 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 10 Jan 2013 23:44:28 +0100
Subject: [PATCH 015/104] add semantic actions for base64_3, base64_2, base64_1

---
 examples/base64_sem1.c | 84 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 4 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 83efc64..7839435 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -1,4 +1,5 @@
 #include "../src/hammer.h"
+#include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
 
 
@@ -77,6 +78,81 @@ const HParsedToken *act_bsfdig(const HParseResult *p)
 
 #define act_document     act_index0
 
+// helper
+void carray_append_uint(HCountedArray *array, uint8_t value)
+{
+    HParsedToken *item = h_arena_malloc(array->arena, sizeof(HParsedToken));
+    item->token_type = TT_UINT;
+    item->uint = value;
+    h_carray_append(array, item);
+}
+
+const HParsedToken *act_base64_3(const HParseResult *p)
+{
+    assert(p->ast->token_type == TT_SEQUENCE);
+
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new_sized(p->arena, 4);
+
+    HParsedToken **digits = p->ast->seq->elements;
+    uint32_t x = digits[0]->uint;
+    x <<= 6; x |= digits[1]->uint;
+    x <<= 6; x |= digits[2]->uint;
+    x <<= 6; x |= digits[3]->uint;
+
+    carray_append_uint(res->seq, (x >> 16) & 0xFF);
+    carray_append_uint(res->seq, (x >> 8) & 0xFF);
+    carray_append_uint(res->seq, x & 0xFF);
+
+    return res;
+}
+
+const HParsedToken *act_base64_2(const HParseResult *p)
+{
+    assert(p->ast->token_type == TT_SEQUENCE);
+
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new_sized(p->arena, 4);
+
+    HParsedToken **digits = p->ast->seq->elements;
+    uint32_t x = digits[0]->uint;
+    x <<= 6; x |= digits[1]->uint;
+    x <<= 6; x |= digits[2]->uint;
+
+    carray_append_uint(res->seq, (x >> 10) & 0xFF);
+    carray_append_uint(res->seq, (x >> 2) & 0xFF);
+
+    return res;
+}
+
+const HParsedToken *act_base64_1(const HParseResult *p)
+{
+    assert(p->ast->token_type == TT_SEQUENCE);
+
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new_sized(p->arena, 4);
+
+    HParsedToken **digits = p->ast->seq->elements;
+    uint32_t x = digits[0]->uint;
+    x <<= 6; x |= digits[1]->uint;
+
+    carray_append_uint(res->seq, (x >> 4) & 0xFF);
+
+    return res;
+}
+
+#if 0
+const HParsedToken *act_base64(const HParseResult *p)
+{
+    // XXX
+    // concatenate base64_3 blocks
+    // append trailing base64_2 or _1 block
+}
+#endif
+
 
 ///
 // Set up the parser with the grammar to be recognized.
@@ -97,10 +173,10 @@ const HParser *init_parser(void)
     H_ARULE(bsfdig,       h_choice(alpha, digit, plus, slash, NULL));
     H_ARULE(bsfdig_4bit,  h_in((uint8_t *)"AEIMQUYcgkosw048", 16));
     H_ARULE(bsfdig_2bit,  h_in((uint8_t *)"AQgw", 4));
-    H_RULE (base64_3,     h_repeat_n(bsfdig, 4));
-    H_RULE (base64_2,     h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL));
-    H_RULE (base64_1,     h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL));
-    H_RULE (base64,       h_sequence(h_many(base64_3),
+    H_ARULE(base64_3,     h_repeat_n(bsfdig, 4));
+    H_ARULE(base64_2,     h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL));
+    H_ARULE(base64_1,     h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL));
+    H_RULE(base64,       h_sequence(h_many(base64_3),
                                      h_optional(h_choice(base64_2,
                                                          base64_1, NULL)),
                                      NULL));

From 97b13672ced335dd7a8c4c926a150ddffb63d401 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 11 Jan 2013 00:41:34 +0100
Subject: [PATCH 016/104] generalize act_base64_n

---
 examples/base64_sem1.c | 83 ++++++++++++++++--------------------------
 1 file changed, 32 insertions(+), 51 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 7839435..6dbb2fd 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -78,70 +78,51 @@ const HParsedToken *act_bsfdig(const HParseResult *p)
 
 #define act_document     act_index0
 
-// helper
-void carray_append_uint(HCountedArray *array, uint8_t value)
+// General-form action to turn a block of base64 digits into bytes.
+const HParsedToken *act_base64_n(int n, const HParseResult *p)
 {
-    HParsedToken *item = h_arena_malloc(array->arena, sizeof(HParsedToken));
-    item->token_type = TT_UINT;
-    item->uint = value;
-    h_carray_append(array, item);
+    assert(p->ast->token_type == TT_SEQUENCE);
+
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new_sized(p->arena, n);
+
+    HParsedToken **digits = p->ast->seq->elements;
+
+    uint32_t x = 0;
+    int bits = 0;
+    for(int i=0; i<n+1; i++) {
+        x <<= 6; x |= digits[i]->uint;
+        bits += 6;
+    }
+    x >>= bits%8;   // align, i.e. cut off extra bits
+
+    for(int i=0; i<n; i++) {
+        HParsedToken *item = h_arena_malloc(p->arena, sizeof(HParsedToken));
+        item->token_type = TT_UINT;
+        item->uint = x & 0xFF;
+
+        res->seq->elements[n-1-i] = item;   // output the last byte and
+        x >>= 8;                            // discard it
+    }
+    res->seq->used = n;
+
+    return res;
 }
 
 const HParsedToken *act_base64_3(const HParseResult *p)
 {
-    assert(p->ast->token_type == TT_SEQUENCE);
-
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new_sized(p->arena, 4);
-
-    HParsedToken **digits = p->ast->seq->elements;
-    uint32_t x = digits[0]->uint;
-    x <<= 6; x |= digits[1]->uint;
-    x <<= 6; x |= digits[2]->uint;
-    x <<= 6; x |= digits[3]->uint;
-
-    carray_append_uint(res->seq, (x >> 16) & 0xFF);
-    carray_append_uint(res->seq, (x >> 8) & 0xFF);
-    carray_append_uint(res->seq, x & 0xFF);
-
-    return res;
+    return act_base64_n(3, p);
 }
 
 const HParsedToken *act_base64_2(const HParseResult *p)
 {
-    assert(p->ast->token_type == TT_SEQUENCE);
-
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new_sized(p->arena, 4);
-
-    HParsedToken **digits = p->ast->seq->elements;
-    uint32_t x = digits[0]->uint;
-    x <<= 6; x |= digits[1]->uint;
-    x <<= 6; x |= digits[2]->uint;
-
-    carray_append_uint(res->seq, (x >> 10) & 0xFF);
-    carray_append_uint(res->seq, (x >> 2) & 0xFF);
-
-    return res;
+    return act_base64_n(2, p);
 }
 
 const HParsedToken *act_base64_1(const HParseResult *p)
 {
-    assert(p->ast->token_type == TT_SEQUENCE);
-
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new_sized(p->arena, 4);
-
-    HParsedToken **digits = p->ast->seq->elements;
-    uint32_t x = digits[0]->uint;
-    x <<= 6; x |= digits[1]->uint;
-
-    carray_append_uint(res->seq, (x >> 4) & 0xFF);
-
-    return res;
+    return act_base64_n(1, p);
 }
 
 #if 0

From c62079516d8441f937599d036231c8cfabd4e89a Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 11 Jan 2013 00:57:34 +0100
Subject: [PATCH 017/104] add semantic action for base64 rule

---
 examples/base64_sem1.c | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 6dbb2fd..5e1673a 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -125,14 +125,37 @@ const HParsedToken *act_base64_1(const HParseResult *p)
     return act_base64_n(1, p);
 }
 
-#if 0
+// Helper to concatenate two arrays.
+void carray_concat(HCountedArray *a, const HCountedArray *b)
+{
+    for(size_t i=0; i<b->used; i++)
+        h_carray_append(a, b->elements[i]);
+}
+
 const HParsedToken *act_base64(const HParseResult *p)
 {
-    // XXX
+    assert(p->ast->token_type == TT_SEQUENCE);
+    assert(p->ast->seq->used == 2);
+    assert(p->ast->seq->elements[0]->token_type == TT_SEQUENCE);
+
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new(p->arena);
+
     // concatenate base64_3 blocks
-    // append trailing base64_2 or _1 block
+    HCountedArray *seq = p->ast->seq->elements[0]->seq;
+    for(size_t i=0; i<seq->used; i++) {
+        assert(seq->elements[i]->token_type == TT_SEQUENCE);
+        carray_concat(res->seq, seq->elements[i]->seq);
+    }
+
+    // append one trailing base64_2 or _1 block
+    const HParsedToken *tok = p->ast->seq->elements[1];
+    if(tok->token_type == TT_SEQUENCE)
+        carray_concat(res->seq, tok->seq);
+
+    return res;
 }
-#endif
 
 
 ///
@@ -157,7 +180,7 @@ const HParser *init_parser(void)
     H_ARULE(base64_3,     h_repeat_n(bsfdig, 4));
     H_ARULE(base64_2,     h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL));
     H_ARULE(base64_1,     h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL));
-    H_RULE(base64,       h_sequence(h_many(base64_3),
+    H_ARULE(base64,       h_sequence(h_many(base64_3),
                                      h_optional(h_choice(base64_2,
                                                          base64_1, NULL)),
                                      NULL));

From ad3c2032dc86c4929f72154cd2121e8259397946 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 11 Jan 2013 01:10:16 +0100
Subject: [PATCH 018/104] add a note about weird bug

---
 examples/base64_sem1.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 5e1673a..8638bb3 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -188,6 +188,10 @@ const HParser *init_parser(void)
     H_ARULE(ws,           h_many(space));
     H_ARULE(document,     h_sequence(ws, base64, ws, h_end_p(), NULL));
 
+    // BUG sometimes inputs that should just don't parse.
+    // It *seemed* to happen mostly with things like "bbbbaaaaBA==".
+    // Using less actions seemed to make it less likely.
+
     return document;
 }
 

From 32dfae0af3db9eedfae6d97b02677cbd3e9e97db Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 13 Jan 2013 18:19:59 +0100
Subject: [PATCH 019/104] add a variant of the base64 example with
 coarse-grained semantic actions

---
 examples/Makefile      |   6 +-
 examples/base64_sem2.c | 207 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 211 insertions(+), 2 deletions(-)
 create mode 100644 examples/base64_sem2.c

diff --git a/examples/Makefile b/examples/Makefile
index a3be0ce..98797f3 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -4,7 +4,9 @@ OUTPUTS := dns.o \
 	   base64.o \
 	   base64 \
 	   base64_sem1.o \
-	   base64_sem1
+	   base64_sem1 \
+	   base64_sem2.o \
+	   base64_sem2
 
 TOPLEVEL := ../
 
@@ -14,7 +16,7 @@ LDFLAGS += $(pkg-config --libs glib-2.0)
 
 
 
-all: dns base64 base64_sem1
+all: dns base64 base64_sem1 base64_sem2
 
 dns: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
 dns: dns.o rr.o dns_common.o
diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
new file mode 100644
index 0000000..957ac48
--- /dev/null
+++ b/examples/base64_sem2.c
@@ -0,0 +1,207 @@
+#include "../src/hammer.h"
+#include "../src/internal.h"    // for h_carray functions (XXX ?!)
+#include <assert.h>
+
+
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+
+
+///
+// Semantic action helpers.
+// These might be candidates for inclusion in the library.
+///
+
+// The action equivalent of h_ignore.
+const HParsedToken *act_ignore(const HParseResult *p)
+{
+    return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+const HParsedToken *act_index0(const HParseResult *p)
+{
+    return act_index(0, p);
+}
+
+
+///
+// Semantic actions for the grammar below, each corresponds to an "ARULE".
+// They must be named act_<rulename>.
+///
+
+// helper: return the numeric value of a parsed base64 digit
+uint8_t bsfdig_value(const HParsedToken *p)
+{
+    uint8_t value = 0;
+
+    if(p && p->token_type == TT_UINT) {
+        uint8_t c = p->uint;
+        if(c >= 0x40 && c <= 0x5A) // A-Z
+            value = c - 0x41;
+        else if(c >= 0x60 && c <= 0x7A) // a-z
+            value = c - 0x61 + 26;
+        else if(c >= 0x30 && c <= 0x39) // 0-9
+            value = c - 0x30 + 52;
+        else if(c == '+')
+            value = 62;
+        else if(c == '/')
+            value = 63;
+    }
+
+    return value;
+}
+
+// helper: append a byte value to a sequence
+void seq_append_byte(HCountedArray *a, uint8_t b)
+{
+    HParsedToken *item = h_arena_malloc(a->arena, sizeof(HParsedToken));
+    item->token_type = TT_UINT;
+    item->uint = b;
+    h_carray_append(a, item);
+}
+
+const HParsedToken *act_base64(const HParseResult *p)
+{
+    assert(p->ast->token_type == TT_SEQUENCE);
+    assert(p->ast->seq->used == 2);
+    assert(p->ast->seq->elements[0]->token_type == TT_SEQUENCE);
+
+    // grab b64_3 block sequence
+    // grab and analyze b64 end block (_2 or _1)
+    const HParsedToken *b64_3 = p->ast->seq->elements[0];
+    const HParsedToken *b64_2 = p->ast->seq->elements[1];
+    const HParsedToken *b64_1 = p->ast->seq->elements[1];
+
+    if(b64_2->token_type != TT_SEQUENCE)
+        b64_1 = b64_2 = NULL;
+    else if(b64_2->seq->elements[2]->uint == '=')
+        b64_2 = NULL;
+    else
+        b64_1 = NULL;
+
+    // allocate result sequence
+    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    res->token_type = TT_SEQUENCE;
+    res->seq = h_carray_new(p->arena);
+
+    // concatenate base64_3 blocks
+    for(size_t i=0; i<b64_3->seq->used; i++) {
+        assert(b64_3->seq->elements[i]->token_type == TT_SEQUENCE);
+        HParsedToken **digits = b64_3->seq->elements[i]->seq->elements;
+
+        uint32_t x = bsfdig_value(digits[0]);
+        x <<= 6; x |= bsfdig_value(digits[1]);
+        x <<= 6; x |= bsfdig_value(digits[2]);
+        x <<= 6; x |= bsfdig_value(digits[3]);
+        seq_append_byte(res->seq, (x >> 16) & 0xFF);
+        seq_append_byte(res->seq, (x >> 8) & 0xFF);
+        seq_append_byte(res->seq, x & 0xFF);
+    }
+
+    // append one trailing base64_2 or _1 block
+    if(b64_2) {
+        HParsedToken **digits = b64_2->seq->elements;
+        uint32_t x = bsfdig_value(digits[0]);
+        x <<= 6; x |= bsfdig_value(digits[1]);
+        x <<= 6; x |= bsfdig_value(digits[2]);
+        seq_append_byte(res->seq, (x >> 10) & 0xFF);
+        seq_append_byte(res->seq, (x >> 2) & 0xFF);
+    } else if(b64_1) {
+        HParsedToken **digits = b64_1->seq->elements;
+        uint32_t x = bsfdig_value(digits[0]);
+        x <<= 6; x |= bsfdig_value(digits[1]);
+        seq_append_byte(res->seq, (x >> 4) & 0xFF);
+    }
+
+    return res;
+}
+
+#define act_ws           act_ignore
+#define act_document     act_index0
+
+
+///
+// Set up the parser with the grammar to be recognized.
+///
+
+const HParser *init_parser(void)
+{
+    // CORE
+    H_RULE (digit,   h_ch_range(0x30, 0x39));
+    H_RULE (alpha,   h_choice(h_ch_range(0x41, 0x5a), h_ch_range(0x61, 0x7a), NULL));
+    H_RULE (space,   h_in((uint8_t *)" \t\n\r\f\v", 6));
+
+    // AUX.
+    H_RULE (plus,    h_ch('+'));
+    H_RULE (slash,   h_ch('/'));
+    H_RULE (equals,  h_ch('='));
+
+    H_RULE (bsfdig,       h_choice(alpha, digit, plus, slash, NULL));
+    H_RULE (bsfdig_4bit,  h_in((uint8_t *)"AEIMQUYcgkosw048", 16));
+    H_RULE (bsfdig_2bit,  h_in((uint8_t *)"AQgw", 4));
+    H_RULE (base64_3,     h_repeat_n(bsfdig, 4));
+    H_RULE (base64_2,     h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL));
+    H_RULE (base64_1,     h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL));
+    H_ARULE(base64,       h_sequence(h_many(base64_3),
+                                     h_optional(h_choice(base64_2,
+                                                         base64_1, NULL)),
+                                     NULL));
+
+    H_ARULE(ws,           h_many(space));
+    H_ARULE(document,     h_sequence(ws, base64, ws, h_end_p(), NULL));
+
+    // BUG sometimes inputs that should just don't parse.
+    // It *seemed* to happen mostly with things like "bbbbaaaaBA==".
+    // Using less actions seemed to make it less likely.
+
+    return document;
+}
+
+
+///
+// Main routine: print input, parse, print result, return success/failure.
+///
+
+#include <stdio.h>
+
+int main(int argc, char **argv)
+{
+    uint8_t input[102400];
+    size_t inputsize;
+    const HParser *parser;
+    const HParseResult *result;
+
+    parser = init_parser();
+
+    inputsize = fread(input, 1, sizeof(input), stdin);
+    fprintf(stderr, "inputsize=%lu\ninput=", inputsize);
+    fwrite(input, 1, inputsize, stderr);
+    result = h_parse(parser, input, inputsize);
+
+    if(result) {
+        fprintf(stderr, "parsed=%lld bytes\n", result->bit_length/8);
+        h_pprint(stdout, result->ast, 0, 0);
+        return 0;
+    } else {
+        return 1;
+    }
+}

From 1998ae243aab922aa3d520d98e607ac853b3bc53 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 13 Jan 2013 18:47:03 +0100
Subject: [PATCH 020/104] add introductory comments to base64 examples

---
 examples/base64.c      | 10 ++++++++++
 examples/base64_sem1.c | 14 ++++++++++++++
 examples/base64_sem2.c | 15 +++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/examples/base64.c b/examples/base64.c
index cdb088e..ee142e3 100644
--- a/examples/base64.c
+++ b/examples/base64.c
@@ -1,3 +1,13 @@
+// Example parser: Base64, syntax only.
+//
+// Demonstrates how to construct a Hammer parser that recognizes valid Base64
+// sequences.
+//
+// Note that no semantic evaluation of the sequence is performed, i.e. the
+// byte sequence being represented is not returned, or determined. See
+// base64_sem1.c and base64_sem2.c for examples how to attach appropriate
+// semantic actions to the grammar.
+
 #include "../src/hammer.h"
 
 const HParser* document = NULL;
diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 8638bb3..92f0b3f 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -1,3 +1,17 @@
+// Example parser: Base64, with fine-grained semantic actions
+//
+// Demonstrates how to attach semantic actions to grammar rules and piece by
+// piece transform the parse tree into the desired semantic representation,
+// in this case a sequence of 8-bit values.
+//
+// Note how the grammar is defined by using the macros H_RULE and H_ARULE.
+// Those rules using ARULE get an attached action which must be declared (as
+// (a function of type HAction) with a standard name based on the rule name.
+//
+// This variant of the example uses fine-grained semantic actions that
+// transform the parse tree in small steps in a bottom-up fashion. Compare
+// base64_sem2.c for an alternative approach using a single top-level action.
+
 #include "../src/hammer.h"
 #include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
index 957ac48..c57555e 100644
--- a/examples/base64_sem2.c
+++ b/examples/base64_sem2.c
@@ -1,3 +1,18 @@
+// Example parser: Base64, with fine-grained semantic actions
+//
+// Demonstrates how to attach semantic actions to a grammar and transform the
+// parse tree into the desired semantic representation, in this case a sequence
+// of 8-bit values.
+//
+// Note how the grammar is defined by using the macros H_RULE and H_ARULE.
+// Those rules using ARULE get an attached action which must be declared (as
+// (a function of type HAction) with a standard name based on the rule name.
+//
+// This variant of the example uses coarse-grained semantic actions,
+// transforming the entire parse tree in one big step. Compare base64_sem1.c
+// for an alternative approach using a fine-grained piece-by-piece
+// transformation.
+
 #include "../src/hammer.h"
 #include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>

From 619725e5bb2e3962235ccaa0f543e710b5f88f55 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 13 Jan 2013 18:56:16 +0100
Subject: [PATCH 021/104] typo

---
 examples/base64_sem2.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
index c57555e..11b0660 100644
--- a/examples/base64_sem2.c
+++ b/examples/base64_sem2.c
@@ -6,7 +6,7 @@
 //
 // Note how the grammar is defined by using the macros H_RULE and H_ARULE.
 // Those rules using ARULE get an attached action which must be declared (as
-// (a function of type HAction) with a standard name based on the rule name.
+// a function of type HAction) with a standard name based on the rule name.
 //
 // This variant of the example uses coarse-grained semantic actions,
 // transforming the entire parse tree in one big step. Compare base64_sem1.c

From fab8705828b5e13ac15ba87c8a7ff995cf7063be Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 13 Jan 2013 18:56:45 +0100
Subject: [PATCH 022/104] typo

---
 examples/base64_sem1.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 92f0b3f..8de31db 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -6,7 +6,7 @@
 //
 // Note how the grammar is defined by using the macros H_RULE and H_ARULE.
 // Those rules using ARULE get an attached action which must be declared (as
-// (a function of type HAction) with a standard name based on the rule name.
+// a function of type HAction) with a standard name based on the rule name.
 //
 // This variant of the example uses fine-grained semantic actions that
 // transform the parse tree in small steps in a bottom-up fashion. Compare

From 40ab87b4fbb5c08696c13cc178573cd8774c08eb Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 14 Jan 2013 20:35:20 +0100
Subject: [PATCH 023/104] add user-defined token types for dns parts

---
 examples/dns.c |  2 +-
 examples/dns.h | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/examples/dns.c b/examples/dns.c
index 54d9c7e..676855e 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -286,7 +286,7 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
 const HParsedToken* pack_dns_struct(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_USER;
+  ret->token_type = TT_DNS_MESSAGE;
 
   dns_message_t *msg = h_arena_malloc(p->arena, sizeof(dns_message_t));
 
diff --git a/examples/dns.h b/examples/dns.h
index 151c46e..a26374e 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -1,5 +1,13 @@
 #include "../src/hammer.h"
 
+enum DNSTokenType_ {
+  TT_DNS_MESSAGE = TT_USER,
+  TT_DNS_HEADER,
+  TT_DNS_QNAME,
+  TT_DNS_QUESTION,
+  TT_DNS_RR
+};
+
 struct dns_header {
   uint16_t id;
   bool qr, aa, tc, rd, ra;
@@ -9,6 +17,7 @@ struct dns_header {
   size_t authority_count;
   size_t additional_count;
 };
+
 struct dns_qname {
   size_t qlen;
   struct {
@@ -16,11 +25,13 @@ struct dns_qname {
     uint8_t *label;
   } *labels;
 };
+
 struct dns_question {
   struct dns_qname qname;
   uint16_t qtype;
   uint16_t qclass;
 };
+
 struct dns_rr {
   char* name;
   uint16_t type;

From 2ff51ab9c36dfc0fc723c38d26009048413e9e37 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 14 Jan 2013 21:25:46 +0100
Subject: [PATCH 024/104] rework dns example syntax into H_RULE style

---
 examples/dns.c | 119 +++++++++++++++++++++++++------------------------
 1 file changed, 61 insertions(+), 58 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 676855e..010c462 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -10,6 +10,10 @@
 #define false 0
 #define true 1
 
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+
+
 bool is_zero(HParseResult *p) {
   if (TT_UINT != p->ast->token_type)
     return false;
@@ -283,7 +287,7 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
   }
 }
 
-const HParsedToken* pack_dns_struct(const HParseResult *p) {
+const HParsedToken* act_dns_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
   ret->token_type = TT_DNS_MESSAGE;
@@ -360,66 +364,65 @@ const HParsedToken* pack_dns_struct(const HParseResult *p) {
   return ret;
 }
 
+// The action equivalent of h_ignore.
+const HParsedToken *act_ignore(const HParseResult *p)
+{
+  return NULL;
+}
+
+#define act_dns_hdzero act_ignore
+
 const HParser* init_parser() {
-  static HParser *dns_message = NULL;
-  if (dns_message)
-    return dns_message;
+  static const HParser *ret = NULL;
+  if (ret)
+    return ret;
 
-  const HParser *domain = init_domain();
+  H_RULE (domain,     init_domain());
+  H_ARULE(dns_hdzero, h_attr_bool(h_bits(3, false), is_zero));
+  H_RULE (dns_header, h_sequence(h_bits(16, false), // ID
+				 h_bits(1, false),  // QR
+				 h_bits(4, false),  // opcode
+				 h_bits(1, false),  // AA
+				 h_bits(1, false),  // TC
+				 h_bits(1, false),  // RD
+				 h_bits(1, false),  // RA
+				 dns_hdzero,        // Z
+				 h_bits(4, false),  // RCODE
+				 h_uint16(), // QDCOUNT
+				 h_uint16(), // ANCOUNT
+				 h_uint16(), // NSCOUNT
+				 h_uint16(), // ARCOUNT
+				 NULL));
+  H_RULE (type,       h_int_range(h_uint16(), 1, 16));
+  H_RULE (qtype,      h_choice(type, 
+			       h_int_range(h_uint16(), 252, 255),
+			       NULL));
+  H_RULE (class,      h_int_range(h_uint16(), 1, 4));
+  H_RULE (qclass,     h_choice(class,
+			       h_int_range(h_uint16(), 255, 255),
+			       NULL));
+  H_RULE (dns_question, h_sequence(h_sequence(h_many1(h_length_value(h_int_range(h_uint8(), 1, 255), 
+								     h_uint8())), 
+					      h_ch('\x00'),
+					      NULL),  // QNAME
+				   qtype,             // QTYPE
+				   qclass,            // QCLASS
+				   NULL));
+  H_RULE (dns_rr,     h_sequence(domain,              // NAME
+				 type,                // TYPE
+				 class,               // CLASS
+				 h_uint32(),          // TTL
+				 h_length_value(h_uint16(), h_uint8()), // RDLENGTH+RDATA
+				 NULL));
+  H_ARULE(dns_message, h_attr_bool(h_sequence(dns_header,
+					      h_many(dns_question),
+					      h_many(dns_rr),
+					      h_end_p(),
+					      NULL),
+				   validate_dns));
 
-  const HParser *dns_header = h_sequence(h_bits(16, false), // ID
-					 h_bits(1, false),  // QR
-					 h_bits(4, false),  // opcode
-					 h_bits(1, false),  // AA
-					 h_bits(1, false),  // TC
-					 h_bits(1, false),  // RD
-					 h_bits(1, false),  // RA
-					 h_ignore(h_attr_bool(h_bits(3, false), is_zero)), // Z
-					 h_bits(4, false),  // RCODE
-					 h_uint16(), // QDCOUNT
-					 h_uint16(), // ANCOUNT
-					 h_uint16(), // NSCOUNT
-					 h_uint16(), // ARCOUNT
-					 NULL);
-
-  const HParser *type = h_int_range(h_uint16(), 1, 16);
-
-  const HParser *qtype = h_choice(type, 
-				  h_int_range(h_uint16(), 252, 255),
-				  NULL);
-
-  const HParser *class = h_int_range(h_uint16(), 1, 4);
-  
-  const HParser *qclass = h_choice(class,
-				   h_int_range(h_uint16(), 255, 255),
-				   NULL);
-
-  const HParser *dns_question = h_sequence(h_sequence(h_many1(h_length_value(h_int_range(h_uint8(), 1, 255), 
-									     h_uint8())), 
-						      h_ch('\x00'),
-						      NULL),  // QNAME
-					   qtype,           // QTYPE
-					   qclass,          // QCLASS
-					   NULL);
- 
-
-  const HParser *dns_rr = h_sequence(domain,                          // NAME
-				     type,                            // TYPE
-				     class,                           // CLASS
-				     h_uint32(),                        // TTL
-				     h_length_value(h_uint16(), h_uint8()), // RDLENGTH+RDATA
-				     NULL);
-
-
-  dns_message = (HParser*)h_action(h_attr_bool(h_sequence(dns_header,
-							  h_many(dns_question),
-							  h_many(dns_rr),
-							  h_end_p(),
-							  NULL),
-					       validate_dns),
-				   pack_dns_struct);
-
-  return dns_message;
+  ret = dns_message;
+  return ret;
 }
 
 int start_listening() {

From 2db9cd23c81e8c24a771675b27a63ee71848acbb Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 00:19:43 +0100
Subject: [PATCH 025/104] grammar beautification

---
 examples/dns.c | 88 +++++++++++++++++++++++++-------------------------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 010c462..08e4288 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -287,7 +287,7 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
   }
 }
 
-const HParsedToken* act_dns_message(const HParseResult *p) {
+const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
   ret->token_type = TT_DNS_MESSAGE;
@@ -370,58 +370,58 @@ const HParsedToken *act_ignore(const HParseResult *p)
   return NULL;
 }
 
-#define act_dns_hdzero act_ignore
+#define act_hdzero act_ignore
 
 const HParser* init_parser() {
   static const HParser *ret = NULL;
   if (ret)
     return ret;
 
-  H_RULE (domain,     init_domain());
-  H_ARULE(dns_hdzero, h_attr_bool(h_bits(3, false), is_zero));
-  H_RULE (dns_header, h_sequence(h_bits(16, false), // ID
-				 h_bits(1, false),  // QR
-				 h_bits(4, false),  // opcode
-				 h_bits(1, false),  // AA
-				 h_bits(1, false),  // TC
-				 h_bits(1, false),  // RD
-				 h_bits(1, false),  // RA
-				 dns_hdzero,        // Z
-				 h_bits(4, false),  // RCODE
-				 h_uint16(), // QDCOUNT
-				 h_uint16(), // ANCOUNT
-				 h_uint16(), // NSCOUNT
-				 h_uint16(), // ARCOUNT
-				 NULL));
-  H_RULE (type,       h_int_range(h_uint16(), 1, 16));
-  H_RULE (qtype,      h_choice(type, 
-			       h_int_range(h_uint16(), 252, 255),
+  H_RULE (domain,   init_domain());
+  H_ARULE(hdzero,   h_attr_bool(h_bits(3, false), is_zero));
+  H_RULE (header,   h_sequence(h_bits(16, false), // ID
+			       h_bits(1, false),  // QR
+			       h_bits(4, false),  // opcode
+			       h_bits(1, false),  // AA
+			       h_bits(1, false),  // TC
+			       h_bits(1, false),  // RD
+			       h_bits(1, false),  // RA
+			       hdzero,            // Z
+			       h_bits(4, false),  // RCODE
+			       h_uint16(),        // QDCOUNT
+			       h_uint16(),        // ANCOUNT
+			       h_uint16(),        // NSCOUNT
+			       h_uint16(),        // ARCOUNT
 			       NULL));
-  H_RULE (class,      h_int_range(h_uint16(), 1, 4));
-  H_RULE (qclass,     h_choice(class,
-			       h_int_range(h_uint16(), 255, 255),
+  H_RULE (type,     h_int_range(h_uint16(), 1, 16));
+  H_RULE (qtype,    h_choice(type, 
+			     h_int_range(h_uint16(), 252, 255),
+			     NULL));
+  H_RULE (class,    h_int_range(h_uint16(), 1, 4));
+  H_RULE (qclass,   h_choice(class,
+			     h_int_range(h_uint16(), 255, 255),
+			     NULL));
+  H_RULE (len,      h_int_range(h_uint8(), 1, 255));
+  H_RULE (label,    h_length_value(len, h_uint8())); 
+  H_RULE (qname,    h_sequence(h_many1(label),
+			       h_ch('\x00'),
 			       NULL));
-  H_RULE (dns_question, h_sequence(h_sequence(h_many1(h_length_value(h_int_range(h_uint8(), 1, 255), 
-								     h_uint8())), 
-					      h_ch('\x00'),
-					      NULL),  // QNAME
-				   qtype,             // QTYPE
-				   qclass,            // QCLASS
-				   NULL));
-  H_RULE (dns_rr,     h_sequence(domain,              // NAME
-				 type,                // TYPE
-				 class,               // CLASS
-				 h_uint32(),          // TTL
-				 h_length_value(h_uint16(), h_uint8()), // RDLENGTH+RDATA
-				 NULL));
-  H_ARULE(dns_message, h_attr_bool(h_sequence(dns_header,
-					      h_many(dns_question),
-					      h_many(dns_rr),
-					      h_end_p(),
-					      NULL),
-				   validate_dns));
+  H_RULE (question, h_sequence(qname, qtype, qclass, NULL));
+  H_RULE (rdata,    h_length_value(h_uint16(), h_uint8()));
+  H_RULE (rr,       h_sequence(domain,            // NAME
+			       type,              // TYPE
+			       class,             // CLASS
+			       h_uint32(),        // TTL
+			       rdata,             // RDLENGTH+RDATA
+			       NULL));
+  H_ARULE(message,  h_attr_bool(h_sequence(header,
+					   h_many(question),
+					   h_many(rr),
+					   h_end_p(),
+					   NULL),
+			        validate_dns));
 
-  ret = dns_message;
+  ret = message;
   return ret;
 }
 

From 21df49cc151b8cd17db866e9d4cf843ef194f316 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 01:24:47 +0100
Subject: [PATCH 026/104] split out act_header

---
 examples/dns.c | 77 +++++++++++++++++++++++++++++---------------------
 examples/dns.h | 14 ++++-----
 src/pprint.c   |  6 +++-
 3 files changed, 57 insertions(+), 40 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 08e4288..5c59626 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -2,6 +2,7 @@
 #include <netinet/in.h>
 #include <err.h>
 #include <string.h>
+#include <assert.h>
 #include "../src/hammer.h"
 #include "dns_common.h"
 #include "dns.h"
@@ -27,12 +28,12 @@ bool is_zero(HParseResult *p) {
 bool validate_dns(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
-  // The header holds the counts as its last 4 elements.
-  HParsedToken **elems = p->ast->seq->elements[0]->seq->elements;
-  size_t qd = elems[8]->uint;
-  size_t an = elems[9]->uint;
-  size_t ns = elems[10]->uint;
-  size_t ar = elems[11]->uint;
+  assert(p->ast->seq->elements[0]->token_type == (HTokenType)TT_dns_header);
+  dns_header_t *header = (dns_header_t *)p->ast->seq->elements[0]->user;
+  size_t qd = header->question_count;
+  size_t an = header->answer_count;
+  size_t ns = header->authority_count;
+  size_t ar = header->additional_count;
   HParsedToken *questions = p->ast->seq->elements[1];
   if (questions->seq->used != qd)
     return false;
@@ -287,34 +288,46 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
   }
 }
 
+const HParsedToken* act_header(const HParseResult *p) {
+  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
+  ret->token_type = TT_dns_header;
+  ret->user = h_arena_malloc(p->arena, sizeof(dns_header_t));
+
+  HParsedToken **fields = p->ast->seq->elements;
+  dns_header_t header_ = {
+    .id     = fields[0]->uint,
+    .qr     = fields[1]->uint,
+    .opcode = fields[2]->uint,
+    .aa     = fields[3]->uint,
+    .tc     = fields[4]->uint,
+    .rd     = fields[5]->uint,
+    .ra     = fields[6]->uint,
+    .rcode  = fields[7]->uint,
+    .question_count   = fields[8]->uint,
+    .answer_count     = fields[9]->uint,
+    .authority_count  = fields[10]->uint,
+    .additional_count = fields[11]->uint
+  };
+  *(dns_header_t *)ret->user = header_;
+
+  return ret;
+}
+
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_DNS_MESSAGE;
+  ret->token_type = TT_dns_message;
 
   dns_message_t *msg = h_arena_malloc(p->arena, sizeof(dns_message_t));
 
-  HParsedToken *hdr = p->ast->seq->elements[0];
-  struct dns_header header = {
-    .id = hdr->seq->elements[0]->uint,
-    .qr = hdr->seq->elements[1]->uint,
-    .opcode = hdr->seq->elements[2]->uint,
-    .aa = hdr->seq->elements[3]->uint,
-    .tc = hdr->seq->elements[4]->uint,
-    .rd = hdr->seq->elements[5]->uint,
-    .ra = hdr->seq->elements[6]->uint,
-    .rcode = hdr->seq->elements[7]->uint,
-    .question_count = hdr->seq->elements[8]->uint,
-    .answer_count = hdr->seq->elements[9]->uint,
-    .authority_count = hdr->seq->elements[10]->uint,
-    .additional_count = hdr->seq->elements[11]->uint
-  };
-  msg->header = header;
+  assert(p->ast->seq->elements[0]->token_type == (HTokenType)TT_dns_header);
+  dns_header_t *header = (dns_header_t *)p->ast->seq->elements[0]->user;
+  msg->header = *header;
 
   HParsedToken *qs = p->ast->seq->elements[1];
   struct dns_question *questions = h_arena_malloc(p->arena,
-						sizeof(struct dns_question)*(header.question_count));
-  for (size_t i=0; i<header.question_count; ++i) {
+						sizeof(struct dns_question)*(header->question_count));
+  for (size_t i=0; i<header->question_count; ++i) {
     // QNAME is a sequence of labels. In the parser, it's defined as
     // sequence(many1(length_value(...)), ch('\x00'), NULL).
     questions[i].qname = get_qname(qs->seq->elements[i]->seq->elements[0]);
@@ -325,8 +338,8 @@ const HParsedToken* act_message(const HParseResult *p) {
 
   HParsedToken *rrs = p->ast->seq->elements[2];
   struct dns_rr *answers = h_arena_malloc(p->arena,
-					  sizeof(struct dns_rr)*(header.answer_count));
-  for (size_t i=0; i<header.answer_count; ++i) {
+					  sizeof(struct dns_rr)*(header->answer_count));
+  for (size_t i=0; i<header->answer_count; ++i) {
     answers[i].name = get_domain(rrs[i].seq->elements[0]);
     answers[i].type = rrs[i].seq->elements[1]->uint;
     answers[i].class = rrs[i].seq->elements[2]->uint;
@@ -337,8 +350,8 @@ const HParsedToken* act_message(const HParseResult *p) {
   msg->answers = answers;
 
   struct dns_rr *authority = h_arena_malloc(p->arena,
-					  sizeof(struct dns_rr)*(header.authority_count));
-  for (size_t i=0, j=header.answer_count; i<header.authority_count; ++i, ++j) {
+					  sizeof(struct dns_rr)*(header->authority_count));
+  for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
     authority[i].name = get_domain(rrs[j].seq->elements[0]);
     authority[i].type = rrs[j].seq->elements[1]->uint;
     authority[i].class = rrs[j].seq->elements[2]->uint;
@@ -349,8 +362,8 @@ const HParsedToken* act_message(const HParseResult *p) {
   msg->authority = authority;
 
   struct dns_rr *additional = h_arena_malloc(p->arena,
-					     sizeof(struct dns_rr)*(header.additional_count));
-  for (size_t i=0, j=header.answer_count+header.authority_count; i<header.additional_count; ++i, ++j) {
+					     sizeof(struct dns_rr)*(header->additional_count));
+  for (size_t i=0, j=header->answer_count+header->authority_count; i<header->additional_count; ++i, ++j) {
     additional[i].name = get_domain(rrs[j].seq->elements[0]);
     additional[i].type = rrs[j].seq->elements[1]->uint;
     additional[i].class = rrs[j].seq->elements[2]->uint;
@@ -379,7 +392,7 @@ const HParser* init_parser() {
 
   H_RULE (domain,   init_domain());
   H_ARULE(hdzero,   h_attr_bool(h_bits(3, false), is_zero));
-  H_RULE (header,   h_sequence(h_bits(16, false), // ID
+  H_ARULE(header,   h_sequence(h_bits(16, false), // ID
 			       h_bits(1, false),  // QR
 			       h_bits(4, false),  // opcode
 			       h_bits(1, false),  // AA
diff --git a/examples/dns.h b/examples/dns.h
index a26374e..6f9fb13 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -1,14 +1,14 @@
 #include "../src/hammer.h"
 
 enum DNSTokenType_ {
-  TT_DNS_MESSAGE = TT_USER,
-  TT_DNS_HEADER,
-  TT_DNS_QNAME,
-  TT_DNS_QUESTION,
-  TT_DNS_RR
+  TT_dns_message = TT_USER,
+  TT_dns_header,
+  TT_dns_qname,
+  TT_dns_question,
+  TT_dns_rr
 };
 
-struct dns_header {
+typedef struct dns_header {
   uint16_t id;
   bool qr, aa, tc, rd, ra;
   char opcode, rcode;
@@ -16,7 +16,7 @@ struct dns_header {
   size_t answer_count;
   size_t authority_count;
   size_t additional_count;
-};
+} dns_header_t;
 
 struct dns_qname {
   size_t qlen;
diff --git a/src/pprint.c b/src/pprint.c
index 1ffe764..d8b22e2 100644
--- a/src/pprint.c
+++ b/src/pprint.c
@@ -69,7 +69,11 @@ void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta) {
     fprintf(stream, "%*sUSER\n", indent, "");
     break;
   default:
-    assert_message(0, "Should not reach here.");
+    if(tok->token_type > TT_USER) {
+      fprintf(stream, "%*sUSER %d\n", indent, "", tok->token_type-TT_USER);
+    } else {
+      assert_message(0, "Should not reach here.");
+    }
   }
 }
 

From acfc903a153b878b4b682be6c98be701b5a151c4 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 13:54:30 +0100
Subject: [PATCH 027/104] start pulling pack_dns_struct apart

---
 examples/dns.c | 98 ++++++++++++++++++++++++++++++++++----------------
 examples/dns.h | 37 ++++++++++---------
 2 files changed, 88 insertions(+), 47 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 5c59626..d25e825 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -43,27 +43,6 @@ bool validate_dns(HParseResult *p) {
   return true;
 }
 
-struct dns_qname get_qname(const HParsedToken *t) {
-  // The qname parser parses at least 1 length-value pair, then a NULL.
-  // So, t->seq->elements[0] is a sequence of at least 1 such pair,
-  // and t->seq->elements[1] is the null.
-  const HParsedToken *labels = t->seq->elements[0];
-  struct dns_qname ret = {
-    .qlen = labels->seq->used,
-    .labels = h_arena_malloc(t->seq->arena, sizeof(*ret.labels)*labels->seq->used)
-  };
-  // i is which label we're on
-  for (size_t i=0; i<labels->seq->used; ++i) {
-    ret.labels[i].len = labels->seq->elements[i]->seq->used;
-    ret.labels[i].label = h_arena_malloc(t->seq->arena, ret.labels[i].len + 1);
-    // j is which char of the label we're on
-    for (size_t j=0; j<ret.labels[i].len; ++j)
-      ret.labels[i].label[j] = labels->seq->elements[i]->seq->elements[j]->uint;
-    ret.labels[i].label[ret.labels[i].len] = 0;
-  }
-  return ret;
-}
-
 char* get_domain(const HParsedToken *t) {
   switch(t->token_type) {
   case TT_UINT:
@@ -313,6 +292,43 @@ const HParsedToken* act_header(const HParseResult *p) {
   return ret;
 }
 
+const HParsedToken* act_label(const HParseResult *p) {
+  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
+  ret->token_type = TT_dns_label;
+  ret->user = h_arena_malloc(p->arena, sizeof(dns_label_t));
+  dns_label_t *r = (dns_label_t *)ret->user;
+
+  r->len = p->ast->seq->used;
+  r->label = h_arena_malloc(p->arena, r->len + 1);
+  for (size_t i=0; i<r->len; ++i)
+    r->label[i] = p->ast->seq->elements[i]->uint;
+  r->label[r->len] = 0;
+
+  return ret;
+}
+
+const HParsedToken* act_question(const HParseResult *p) {
+  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
+  ret->token_type = TT_dns_question;
+  ret->user = h_arena_malloc(p->arena, sizeof(dns_question_t));
+
+  dns_question_t *q = (dns_question_t *)ret->user;
+  HParsedToken **fields = p->ast->seq->elements;
+
+  // QNAME is a sequence of labels. Pack them into an array.
+  q->qname.qlen   = fields[0]->seq->used;
+  q->qname.labels = h_arena_malloc(p->arena, sizeof(dns_label_t)*q->qname.qlen);
+  for(size_t i=0; i<fields[0]->seq->used; i++) {
+    assert(fields[0]->seq->elements[i]->token_type == (HTokenType)TT_dns_label);
+    q->qname.labels[i] = *(dns_label_t *)fields[0]->seq->elements[i]->user;
+  }
+
+  q->qtype  = fields[1]->uint;
+  q->qclass = fields[2]->uint;
+
+  return ret;
+}
+
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
@@ -326,13 +342,10 @@ const HParsedToken* act_message(const HParseResult *p) {
 
   HParsedToken *qs = p->ast->seq->elements[1];
   struct dns_question *questions = h_arena_malloc(p->arena,
-						sizeof(struct dns_question)*(header->question_count));
+						  sizeof(struct dns_question)*(header->question_count));
   for (size_t i=0; i<header->question_count; ++i) {
-    // QNAME is a sequence of labels. In the parser, it's defined as
-    // sequence(many1(length_value(...)), ch('\x00'), NULL).
-    questions[i].qname = get_qname(qs->seq->elements[i]->seq->elements[0]);
-    questions[i].qtype = qs->seq->elements[i]->seq->elements[1]->uint;
-    questions[i].qclass = qs->seq->elements[i]->seq->elements[2]->uint;
+    assert(qs->seq->elements[i]->token_type == (HTokenType)TT_dns_question);
+    questions[i] = *(dns_question_t *)qs->seq->elements[i]->user;
   }
   msg->questions = questions;
 
@@ -383,7 +396,32 @@ const HParsedToken *act_ignore(const HParseResult *p)
   return NULL;
 }
 
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+const HParsedToken *act_index0(const HParseResult *p)
+{
+    return act_index(0, p);
+}
+
 #define act_hdzero act_ignore
+#define act_qname  act_index0
 
 const HParser* init_parser() {
   static const HParser *ret = NULL;
@@ -415,11 +453,11 @@ const HParser* init_parser() {
 			     h_int_range(h_uint16(), 255, 255),
 			     NULL));
   H_RULE (len,      h_int_range(h_uint8(), 1, 255));
-  H_RULE (label,    h_length_value(len, h_uint8())); 
-  H_RULE (qname,    h_sequence(h_many1(label),
+  H_ARULE(label,    h_length_value(len, h_uint8())); 
+  H_ARULE(qname,    h_sequence(h_many1(label),
 			       h_ch('\x00'),
 			       NULL));
-  H_RULE (question, h_sequence(qname, qtype, qclass, NULL));
+  H_ARULE(question, h_sequence(qname, qtype, qclass, NULL));
   H_RULE (rdata,    h_length_value(h_uint16(), h_uint8()));
   H_RULE (rr,       h_sequence(domain,            // NAME
 			       type,              // TYPE
diff --git a/examples/dns.h b/examples/dns.h
index 6f9fb13..81bed55 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -3,6 +3,7 @@
 enum DNSTokenType_ {
   TT_dns_message = TT_USER,
   TT_dns_header,
+  TT_dns_label,
   TT_dns_qname,
   TT_dns_question,
   TT_dns_rr
@@ -18,21 +19,23 @@ typedef struct dns_header {
   size_t additional_count;
 } dns_header_t;
 
-struct dns_qname {
-  size_t qlen;
-  struct {
-    size_t len;
-    uint8_t *label;
-  } *labels;
-};
+typedef struct dns_label {
+  size_t len;
+  uint8_t *label;
+} dns_label_t;
 
-struct dns_question {
-  struct dns_qname qname;
+typedef struct dns_qname {
+  size_t qlen;
+  dns_label_t *labels;
+} dns_qname_t;
+
+typedef struct dns_question {
+  dns_qname_t qname;
   uint16_t qtype;
   uint16_t qclass;
-};
+} dns_question_t;
 
-struct dns_rr {
+typedef struct dns_rr {
   char* name;
   uint16_t type;
   uint16_t class;
@@ -81,12 +84,12 @@ struct dns_rr {
       uint8_t* bit_map;
     } wks;
   };
-};
+} dns_rr_t;
 
 typedef struct dns_message {
-  struct dns_header header;
-  struct dns_question *questions;
-  struct dns_rr *answers;
-  struct dns_rr *authority;
-  struct dns_rr *additional;
+  dns_header_t header;
+  dns_question_t *questions;
+  dns_rr_t *answers;
+  dns_rr_t *authority;
+  dns_rr_t *additional;
 } dns_message_t;

From b5db3177db043e8baf2a76cf40f8ad6e31540981 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 17:50:18 +0100
Subject: [PATCH 028/104] normalize allocation of result in act_message wrt
 other actions

---
 examples/dns.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index d25e825..6db8bf8 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -331,10 +331,11 @@ const HParsedToken* act_question(const HParseResult *p) {
 
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
+
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
   ret->token_type = TT_dns_message;
-
-  dns_message_t *msg = h_arena_malloc(p->arena, sizeof(dns_message_t));
+  ret->user = h_arena_malloc(p->arena, sizeof(dns_message_t));
+  dns_message_t *msg = ret->user;
 
   assert(p->ast->seq->elements[0]->token_type == (HTokenType)TT_dns_header);
   dns_header_t *header = (dns_header_t *)p->ast->seq->elements[0]->user;
@@ -386,7 +387,6 @@ const HParsedToken* act_message(const HParseResult *p) {
   }
   msg->additional = additional;
 
-  ret->user = (void*)msg;
   return ret;
 }
 

From 342e39eb61bf583d582938ceaad3949c9b0665f5 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 20:02:53 +0100
Subject: [PATCH 029/104] add helpers for easier token construction and access
 to sequence elements

---
 examples/dns.c | 154 +++++++++++++++++++++++++++++++------------------
 1 file changed, 97 insertions(+), 57 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 6db8bf8..01442c8 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -11,9 +11,79 @@
 #define false 0
 #define true 1
 
+
+///
+// API Additions
+///
+
 #define H_RULE(rule, def) const HParser *rule = def
 #define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
 
+// The action equivalent of h_ignore.
+const HParsedToken *act_ignore(const HParseResult *p)
+{
+  return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+const HParsedToken *act_index0(const HParseResult *p)
+{
+    return act_index(0, p);
+}
+
+HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value) {
+  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
+  ret->token_type = type;
+  ret->user = value;
+  return ret;
+}
+
+#define H_MAKE(TYP) \
+  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
+
+#define H_MAKE_TOKEN(TYP, VAL) \
+  h_make_token(p->arena, TT_ ## TYP, VAL)
+
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i) {
+  assert(i < a->used);
+  return a->elements[i];
+}
+
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i) {
+  assert(p->token_type == TT_SEQUENCE);
+  return h_carray_index(p->seq, i);
+}
+
+void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i) {
+  HParsedToken *elem = h_seq_index(p, i);
+  assert(elem->token_type == (HTokenType)type);
+  return elem->user;
+}
+
+#define H_SEQ_INDEX(TYP, SEQ, IDX) \
+  ((TYP ## _t *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
+
+#define H_FIELD(TYP, IDX) \
+  H_SEQ_INDEX(TYP, p->ast, IDX)
+
 
 bool is_zero(HParseResult *p) {
   if (TT_UINT != p->ast->token_type)
@@ -21,6 +91,11 @@ bool is_zero(HParseResult *p) {
   return (0 == p->ast->uint);
 }
 
+
+///
+// Semantic Actions
+///
+
 /**
  * Every DNS message should have QDCOUNT entries in the question
  * section, and ANCOUNT+NSCOUNT+ARCOUNT resource records.
@@ -268,10 +343,6 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
 }
 
 const HParsedToken* act_header(const HParseResult *p) {
-  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_dns_header;
-  ret->user = h_arena_malloc(p->arena, sizeof(dns_header_t));
-
   HParsedToken **fields = p->ast->seq->elements;
   dns_header_t header_ = {
     .id     = fields[0]->uint,
@@ -287,16 +358,15 @@ const HParsedToken* act_header(const HParseResult *p) {
     .authority_count  = fields[10]->uint,
     .additional_count = fields[11]->uint
   };
-  *(dns_header_t *)ret->user = header_;
 
-  return ret;
+  dns_header_t *header = H_MAKE(dns_header);
+  *header = header_;
+
+  return H_MAKE_TOKEN(dns_header, header);
 }
 
 const HParsedToken* act_label(const HParseResult *p) {
-  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_dns_label;
-  ret->user = h_arena_malloc(p->arena, sizeof(dns_label_t));
-  dns_label_t *r = (dns_label_t *)ret->user;
+  dns_label_t *r = H_MAKE(dns_label);
 
   r->len = p->ast->seq->used;
   r->label = h_arena_malloc(p->arena, r->len + 1);
@@ -304,41 +374,31 @@ const HParsedToken* act_label(const HParseResult *p) {
     r->label[i] = p->ast->seq->elements[i]->uint;
   r->label[r->len] = 0;
 
-  return ret;
+  return H_MAKE_TOKEN(dns_label, r);
 }
 
 const HParsedToken* act_question(const HParseResult *p) {
-  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_dns_question;
-  ret->user = h_arena_malloc(p->arena, sizeof(dns_question_t));
-
-  dns_question_t *q = (dns_question_t *)ret->user;
+  dns_question_t *q = H_MAKE(dns_question);
   HParsedToken **fields = p->ast->seq->elements;
 
   // QNAME is a sequence of labels. Pack them into an array.
   q->qname.qlen   = fields[0]->seq->used;
   q->qname.labels = h_arena_malloc(p->arena, sizeof(dns_label_t)*q->qname.qlen);
   for(size_t i=0; i<fields[0]->seq->used; i++) {
-    assert(fields[0]->seq->elements[i]->token_type == (HTokenType)TT_dns_label);
-    q->qname.labels[i] = *(dns_label_t *)fields[0]->seq->elements[i]->user;
+    q->qname.labels[i] = *H_SEQ_INDEX(dns_label, fields[0], i);
   }
 
   q->qtype  = fields[1]->uint;
   q->qclass = fields[2]->uint;
 
-  return ret;
+  return H_MAKE_TOKEN(dns_question, q);
 }
 
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
+  dns_message_t *msg = H_MAKE(dns_message);
 
-  HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
-  ret->token_type = TT_dns_message;
-  ret->user = h_arena_malloc(p->arena, sizeof(dns_message_t));
-  dns_message_t *msg = ret->user;
-
-  assert(p->ast->seq->elements[0]->token_type == (HTokenType)TT_dns_header);
-  dns_header_t *header = (dns_header_t *)p->ast->seq->elements[0]->user;
+  dns_header_t *header = H_FIELD(dns_header, 0);
   msg->header = *header;
 
   HParsedToken *qs = p->ast->seq->elements[1];
@@ -387,42 +447,17 @@ const HParsedToken* act_message(const HParseResult *p) {
   }
   msg->additional = additional;
 
-  return ret;
-}
-
-// The action equivalent of h_ignore.
-const HParsedToken *act_ignore(const HParseResult *p)
-{
-  return NULL;
-}
-
-// Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *act_index(int i, const HParseResult *p)
-{
-    if(!p) return NULL;
-
-    const HParsedToken *tok = p->ast;
-
-    if(!tok || tok->token_type != TT_SEQUENCE)
-        return NULL;
-
-    const HCountedArray *seq = tok->seq;
-    size_t n = seq->used;
-
-    if(i<0 || (size_t)i>=n)
-        return NULL;
-    else
-        return tok->seq->elements[i];
-}
-
-const HParsedToken *act_index0(const HParseResult *p)
-{
-    return act_index(0, p);
+  return H_MAKE_TOKEN(dns_message, msg);
 }
 
 #define act_hdzero act_ignore
 #define act_qname  act_index0
 
+
+///
+// Parser / Grammar
+///
+
 const HParser* init_parser() {
   static const HParser *ret = NULL;
   if (ret)
@@ -476,6 +511,11 @@ const HParser* init_parser() {
   return ret;
 }
 
+
+///
+// Program Logic for a Dummy DNS Server
+///
+
 int start_listening() {
   // return: fd
   int sock;

From 83f494432133243fc8908a74af4fec6fa91a2721 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 22:12:13 +0100
Subject: [PATCH 030/104] separate parsing and packing of rdata

---
 examples/dns.c | 234 +++++++++++++++++--------------------------------
 1 file changed, 82 insertions(+), 152 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 01442c8..e38a2b4 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -167,176 +167,106 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
   for (size_t i=0; i<rdata->used; ++i)
     data[i] = rdata->elements[i]->uint;
 
+  // Mapping numeric RR types (as indices) to parsers
+  const HParser *parsers[] = {
+    NULL,          // there is no type 0
+    init_a(),      // 1
+    init_ns(),
+    init_md(),
+    init_mf(),
+    init_cname(),  // 5
+    init_soa(),
+    init_mb(),
+    init_mg(),
+    init_mr(),
+    init_null(),   // 10
+    init_wks(),
+    init_ptr(),
+    init_hinfo(),
+    init_minfo(),
+    init_mx(),     // 15
+    init_txt()
+  };
+
+  // Parse rdata if possible.
+  const HParseResult *r = NULL;
+  if (rr.type < sizeof(parsers)) {
+    const HParser *p = parsers[rr.type];
+    if (p)
+      r = h_parse(p, (const uint8_t*)data, rdata->used);
+  }
+	
   // If the RR doesn't parse, set its type to 0.
+  if (!r) 
+    rr.type = 0;
+
+  // Pack the parsed rdata into rr.
   switch(rr.type) {
   case 1: // A
-    {
-      const HParseResult *r = h_parse(init_a(), (const uint8_t*)data, rdata->used);
-      if (!r) 
-	rr.type = 0;
-      else 
-	rr.a = r->ast->seq->elements[0]->uint;
-      break;
-    }
+    rr.a = r->ast->seq->elements[0]->uint;
+    break;
   case 2: // NS
-    {
-      const HParseResult *r = h_parse(init_ns(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.ns = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.ns = get_domain(r->ast->seq->elements[0]);
+    break;
   case 3: // MD
-    {
-      const HParseResult *r = h_parse(init_md(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.md = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.md = get_domain(r->ast->seq->elements[0]);
+    break;
   case 4: // MF
-    {
-      const HParseResult *r = h_parse(init_mf(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.md = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.md = get_domain(r->ast->seq->elements[0]);
+    break;
   case 5: // CNAME
-    {
-      const HParseResult *r = h_parse(init_cname(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.cname = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.cname = get_domain(r->ast->seq->elements[0]);
+    break;
   case 6: // SOA
-    {
-      const HParseResult *r = h_parse(init_soa(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.soa.mname = get_domain(r->ast->seq->elements[0]);
-	rr.soa.rname = get_domain(r->ast->seq->elements[1]);
-	rr.soa.serial = r->ast->seq->elements[2]->uint;
-	rr.soa.refresh = r->ast->seq->elements[3]->uint;
-	rr.soa.retry = r->ast->seq->elements[4]->uint;
-	rr.soa.expire = r->ast->seq->elements[5]->uint;
-	rr.soa.minimum = r->ast->seq->elements[6]->uint;
-      }
-      break;
-    }
+    rr.soa.mname = get_domain(r->ast->seq->elements[0]);
+    rr.soa.rname = get_domain(r->ast->seq->elements[1]);
+    rr.soa.serial = r->ast->seq->elements[2]->uint;
+    rr.soa.refresh = r->ast->seq->elements[3]->uint;
+    rr.soa.retry = r->ast->seq->elements[4]->uint;
+    rr.soa.expire = r->ast->seq->elements[5]->uint;
+    rr.soa.minimum = r->ast->seq->elements[6]->uint;
+    break;
   case 7: // MB
-    {
-      const HParseResult *r = h_parse(init_mb(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.mb = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.mb = get_domain(r->ast->seq->elements[0]);
+    break;
   case 8: // MG
-    {
-      const HParseResult *r = h_parse(init_mg(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.mg = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.mg = get_domain(r->ast->seq->elements[0]);
+    break;
   case 9: // MR
-    {
-      const HParseResult *r = h_parse(init_mr(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.mr = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.mr = get_domain(r->ast->seq->elements[0]);
+    break;
   case 10: // NULL
-    {
-      const HParseResult *r = h_parse(init_null(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->used);
-	for (size_t i=0; i<r->ast->seq->used; ++i)
-	  rr.null[i] = r->ast->seq->elements[i]->uint;
-      }
-      break;
-    }
+    rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->used);
+    for (size_t i=0; i<r->ast->seq->used; ++i)
+      rr.null[i] = r->ast->seq->elements[i]->uint;
+    break;
   case 11: // WKS
-    {
-      const HParseResult *r = h_parse(init_wks(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.wks.address = r->ast->seq->elements[0]->uint;
-	rr.wks.protocol = r->ast->seq->elements[1]->uint;
-	rr.wks.len = r->ast->seq->elements[2]->seq->used;
-	rr.wks.bit_map = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->elements[2]->seq->used);
-	for (size_t i=0; i<rr.wks.len; ++i)
-	  rr.wks.bit_map[i] = r->ast->seq->elements[2]->seq->elements[i]->uint;
-      }
-      break;
-    }
+    rr.wks.address = r->ast->seq->elements[0]->uint;
+    rr.wks.protocol = r->ast->seq->elements[1]->uint;
+    rr.wks.len = r->ast->seq->elements[2]->seq->used;
+    rr.wks.bit_map = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->elements[2]->seq->used);
+    for (size_t i=0; i<rr.wks.len; ++i)
+      rr.wks.bit_map[i] = r->ast->seq->elements[2]->seq->elements[i]->uint;
+    break;
   case 12: // PTR
-    {
-      const HParseResult *r = h_parse(init_ptr(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else
-	rr.ptr = get_domain(r->ast->seq->elements[0]);
-      break;
-    }
+    rr.ptr = get_domain(r->ast->seq->elements[0]);
+    break;
   case 13: // HINFO
-    {
-      const HParseResult *r = h_parse(init_hinfo(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.hinfo.cpu = get_cs(r->ast->seq->elements[0]->seq);
-	rr.hinfo.os = get_cs(r->ast->seq->elements[1]->seq);
-      }
-      break;
-    }
+    rr.hinfo.cpu = get_cs(r->ast->seq->elements[0]->seq);
+    rr.hinfo.os = get_cs(r->ast->seq->elements[1]->seq);
+    break;
   case 14: // MINFO
-    {
-      const HParseResult *r = h_parse(init_minfo(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.minfo.rmailbx = get_domain(r->ast->seq->elements[0]);
-	rr.minfo.emailbx = get_domain(r->ast->seq->elements[1]);
-      }
-      break;
-    }
+    rr.minfo.rmailbx = get_domain(r->ast->seq->elements[0]);
+    rr.minfo.emailbx = get_domain(r->ast->seq->elements[1]);
+    break;
   case 15: // MX
-    {
-      const HParseResult *r = h_parse(init_mx(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.mx.preference = r->ast->seq->elements[0]->uint;
-	rr.mx.exchange = get_domain(r->ast->seq->elements[1]);
-      }
-      break;
-    }
+    rr.mx.preference = r->ast->seq->elements[0]->uint;
+    rr.mx.exchange = get_domain(r->ast->seq->elements[1]);
+    break;
   case 16: // TXT
-    {
-      const HParseResult *r = h_parse(init_txt(), (const uint8_t*)data, rdata->used);
-      if (!r)
-	rr.type = 0;
-      else {
-	rr.txt.count = r->ast->seq->elements[0]->seq->used;
-	rr.txt.txt_data = get_txt(r->ast->seq->elements[0]->seq);
-      }
-      break;
-    }
+    rr.txt.count = r->ast->seq->elements[0]->seq->used;
+    rr.txt.txt_data = get_txt(r->ast->seq->elements[0]->seq);
+    break;
   default:
     break;
   }

From 9c10a75f3d7e7da1f5908f09f3bbb498eb189ba6 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 22:18:33 +0100
Subject: [PATCH 031/104] use H_SEQ_INDEX to access a sequence

---
 examples/dns.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index e38a2b4..d786d59 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -335,8 +335,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_question *questions = h_arena_malloc(p->arena,
 						  sizeof(struct dns_question)*(header->question_count));
   for (size_t i=0; i<header->question_count; ++i) {
-    assert(qs->seq->elements[i]->token_type == (HTokenType)TT_dns_question);
-    questions[i] = *(dns_question_t *)qs->seq->elements[i]->user;
+    questions[i] = *H_SEQ_INDEX(dns_question, qs, i);
   }
   msg->questions = questions;
 

From 4b30ebdb772ec288ffaee89fb1269cc75c332b69 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 15 Jan 2013 22:34:15 +0100
Subject: [PATCH 032/104] pull out struct types of complex rdata fields

---
 examples/dns.h | 94 ++++++++++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 41 deletions(-)

diff --git a/examples/dns.h b/examples/dns.h
index 81bed55..672e3b7 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -35,6 +35,43 @@ typedef struct dns_question {
   uint16_t qclass;
 } dns_question_t;
 
+typedef struct {
+  uint8_t* cpu;
+  uint8_t* os;
+} dns_rr_hinfo_t;
+
+typedef struct {
+  char* rmailbx;
+  char* emailbx;
+} dns_rr_minfo_t;
+
+typedef struct {
+  uint16_t preference;
+  char* exchange;
+} dns_rr_mx_t;
+
+typedef struct {
+  char* mname;
+  char* rname;
+  uint32_t serial;
+  uint32_t refresh;
+  uint32_t retry;
+  uint32_t expire;
+  uint32_t minimum;
+} dns_rr_soa_t;
+
+typedef struct {
+  size_t count;
+  uint8_t** txt_data;
+} dns_rr_txt_t;
+
+typedef struct {
+  uint32_t address;
+  uint8_t protocol;
+  size_t len;
+  uint8_t* bit_map;
+} dns_rr_wks_t;
+
 typedef struct dns_rr {
   char* name;
   uint16_t type;
@@ -42,47 +79,22 @@ typedef struct dns_rr {
   uint32_t ttl; // cmos is also acceptable.
   uint16_t rdlength;
   union {
-    char* cname;
-    struct {
-      uint8_t* cpu;
-      uint8_t* os;
-    } hinfo;
-    char* mb;
-    char* md;
-    char* mf;
-    char* mg;
-    struct {
-      char* rmailbx;
-      char* emailbx;
-    } minfo;
-    char* mr;
-    struct {
-      uint16_t preference;
-      char* exchange;
-    } mx;
-    uint8_t* null;
-    char* ns;
-    char* ptr;
-    struct {
-      char* mname;
-      char* rname;
-      uint32_t serial;
-      uint32_t refresh;
-      uint32_t retry;
-      uint32_t expire;
-      uint32_t minimum;
-    } soa;
-    struct {
-      size_t count;
-      uint8_t** txt_data;
-    } txt;
-    uint32_t a;
-    struct {
-      uint32_t address;
-      uint8_t protocol;
-      size_t len;
-      uint8_t* bit_map;
-    } wks;
+    uint32_t       a;
+    char*          ns;
+    char*          md;
+    char*          mf;
+    char*          cname;
+    dns_rr_soa_t   soa;
+    char*          mb;
+    char*          mg;
+    char*          mr;
+    uint8_t*       null;
+    dns_rr_wks_t   wks;
+    char*          ptr;
+    dns_rr_hinfo_t hinfo;
+    dns_rr_minfo_t minfo;
+    dns_rr_mx_t    mx;
+    dns_rr_txt_t   txt;
   };
 } dns_rr_t;
 

From 341254137052b4a33e151bde3c2036d8e2a63c42 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 15:05:04 +0100
Subject: [PATCH 033/104] move api additions to glue.[ch]

---
 examples/Makefile     |  5 ++-
 examples/dns.c        | 76 +------------------------------------------
 examples/dns_common.c |  1 +
 examples/dns_common.h |  1 +
 examples/glue.c       | 65 ++++++++++++++++++++++++++++++++++++
 examples/glue.h       | 36 ++++++++++++++++++++
 6 files changed, 106 insertions(+), 78 deletions(-)
 create mode 100644 examples/glue.c
 create mode 100644 examples/glue.h

diff --git a/examples/Makefile b/examples/Makefile
index 6a054ca..786af44 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -15,14 +15,13 @@ LDFLAGS += $(pkg-config --libs glib-2.0)
 all: dns base64
 
 dns: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
-dns: dns.o rr.o dns_common.o
+dns: dns.o rr.o dns_common.o glue.o
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
 
 dns.o: ../src/hammer.h dns_common.h
-
 rr.o: ../src/hammer.h rr.h dns_common.h
-
 dns_common.o: ../src/hammer.h dns_common.h
+glue.o: ../src/hammer.h glue.h
 
 base64: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
 base64: base64.o
diff --git a/examples/dns.c b/examples/dns.c
index d786d59..b513a51 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -13,89 +13,15 @@
 
 
 ///
-// API Additions
+// Semantic Actions and Validations
 ///
 
-#define H_RULE(rule, def) const HParser *rule = def
-#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
-
-// The action equivalent of h_ignore.
-const HParsedToken *act_ignore(const HParseResult *p)
-{
-  return NULL;
-}
-
-// Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *act_index(int i, const HParseResult *p)
-{
-    if(!p) return NULL;
-
-    const HParsedToken *tok = p->ast;
-
-    if(!tok || tok->token_type != TT_SEQUENCE)
-        return NULL;
-
-    const HCountedArray *seq = tok->seq;
-    size_t n = seq->used;
-
-    if(i<0 || (size_t)i>=n)
-        return NULL;
-    else
-        return tok->seq->elements[i];
-}
-
-const HParsedToken *act_index0(const HParseResult *p)
-{
-    return act_index(0, p);
-}
-
-HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value) {
-  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
-  ret->token_type = type;
-  ret->user = value;
-  return ret;
-}
-
-#define H_MAKE(TYP) \
-  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
-
-#define H_MAKE_TOKEN(TYP, VAL) \
-  h_make_token(p->arena, TT_ ## TYP, VAL)
-
-HParsedToken *h_carray_index(const HCountedArray *a, size_t i) {
-  assert(i < a->used);
-  return a->elements[i];
-}
-
-HParsedToken *h_seq_index(const HParsedToken *p, size_t i) {
-  assert(p->token_type == TT_SEQUENCE);
-  return h_carray_index(p->seq, i);
-}
-
-void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i) {
-  HParsedToken *elem = h_seq_index(p, i);
-  assert(elem->token_type == (HTokenType)type);
-  return elem->user;
-}
-
-#define H_SEQ_INDEX(TYP, SEQ, IDX) \
-  ((TYP ## _t *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
-
-#define H_FIELD(TYP, IDX) \
-  H_SEQ_INDEX(TYP, p->ast, IDX)
-
-
 bool is_zero(HParseResult *p) {
   if (TT_UINT != p->ast->token_type)
     return false;
   return (0 == p->ast->uint);
 }
 
-
-///
-// Semantic Actions
-///
-
 /**
  * Every DNS message should have QDCOUNT entries in the question
  * section, and ANCOUNT+NSCOUNT+ARCOUNT resource records.
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 3d349f1..5bd2374 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -13,6 +13,7 @@ bool validate_label(HParseResult *p) {
   return (64 > p->ast->seq->used);
 }
 
+
 const HParser* init_domain() {
   static const HParser *domain = NULL;
   if (domain)
diff --git a/examples/dns_common.h b/examples/dns_common.h
index 41d73f0..2d796f8 100644
--- a/examples/dns_common.h
+++ b/examples/dns_common.h
@@ -2,6 +2,7 @@
 #define HAMMER_DNS_COMMON__H
 
 #include "../src/hammer.h"
+#include "glue.h"
 
 const HParser* init_domain();
 const HParser* init_character_string();
diff --git a/examples/glue.c b/examples/glue.c
new file mode 100644
index 0000000..a438b17
--- /dev/null
+++ b/examples/glue.c
@@ -0,0 +1,65 @@
+#include "glue.h"
+
+
+// The action equivalent of h_ignore.
+const HParsedToken *act_ignore(const HParseResult *p)
+{
+  return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+const HParsedToken *act_index0(const HParseResult *p)
+{
+    return act_index(0, p);
+}
+
+HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value)
+{
+  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
+  ret->token_type = type;
+  ret->user = value;
+  return ret;
+}
+
+#define H_MAKE(TYP) \
+  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
+
+#define H_MAKE_TOKEN(TYP, VAL) \
+  h_make_token(p->arena, TT_ ## TYP, VAL)
+
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
+{
+  assert(i < a->used);
+  return a->elements[i];
+}
+
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
+{
+  assert(p->token_type == TT_SEQUENCE);
+  return h_carray_index(p->seq, i);
+}
+
+void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
+{
+  HParsedToken *elem = h_seq_index(p, i);
+  assert(elem->token_type == (HTokenType)type);
+  return elem->user;
+}
diff --git a/examples/glue.h b/examples/glue.h
new file mode 100644
index 0000000..d8776b3
--- /dev/null
+++ b/examples/glue.h
@@ -0,0 +1,36 @@
+#ifndef HAMMER_EXAMPLES_GLUE__H
+#define HAMMER_EXAMPLES_GLUE__H
+
+#include <assert.h>
+#include "../src/hammer.h"
+
+///
+// API Additions
+///
+
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+
+const HParsedToken *act_ignore(const HParseResult *p);
+const HParsedToken *act_index(int i, const HParseResult *p);
+const HParsedToken *act_index0(const HParseResult *p);
+
+HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
+
+#define H_MAKE(TYP) \
+  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
+
+#define H_MAKE_TOKEN(TYP, VAL) \
+  h_make_token(p->arena, TT_ ## TYP, VAL)
+
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i);
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
+void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
+
+#define H_SEQ_INDEX(TYP, SEQ, IDX) \
+  ((TYP ## _t *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
+
+#define H_FIELD(TYP, IDX) \
+  H_SEQ_INDEX(TYP, p->ast, IDX)
+
+#endif

From a38d2e4ab0cc1fd3c274af80216412669d6f3101 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 15:06:51 +0100
Subject: [PATCH 034/104] replace another field access with a glue macro

---
 examples/dns.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index b513a51..fb2b148 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -2,7 +2,6 @@
 #include <netinet/in.h>
 #include <err.h>
 #include <string.h>
-#include <assert.h>
 #include "../src/hammer.h"
 #include "dns_common.h"
 #include "dns.h"
@@ -29,8 +28,7 @@ bool is_zero(HParseResult *p) {
 bool validate_dns(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
-  assert(p->ast->seq->elements[0]->token_type == (HTokenType)TT_dns_header);
-  dns_header_t *header = (dns_header_t *)p->ast->seq->elements[0]->user;
+  dns_header_t *header = H_FIELD(dns_header, 0);
   size_t qd = header->question_count;
   size_t an = header->answer_count;
   size_t ns = header->authority_count;

From aa295f1536d195f9a030026626c1c215ebf3b6a3 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 15:40:10 +0100
Subject: [PATCH 035/104] rewrite dns domain grammar using H_RULE

---
 examples/dns_common.c | 58 ++++++++++++-------------------------------
 1 file changed, 16 insertions(+), 42 deletions(-)

diff --git a/examples/dns_common.c b/examples/dns_common.c
index 5bd2374..0b07cf8 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -15,50 +15,24 @@ bool validate_label(HParseResult *p) {
 
 
 const HParser* init_domain() {
-  static const HParser *domain = NULL;
-  if (domain)
-    return domain;
+  static const HParser *ret = NULL;
+  if (ret)
+    return ret;
 
-  const HParser *letter = h_choice(h_ch_range('a', 'z'),
-				   h_ch_range('A', 'Z'),
-				   NULL);
+  H_RULE (letter,    h_choice(h_ch_range('a','z'), h_ch_range('A','Z'), NULL));
+  H_RULE (let_dig,   h_choice(letter, h_ch_range('0','9'), NULL));
+  H_RULE (ldh_str,   h_many1(h_choice(let_dig, h_ch('-'), NULL)));
+  H_RULE (label,     h_attr_bool(h_sequence(letter,
+					    h_optional(h_sequence(h_optional(ldh_str),
+								  let_dig,
+								  NULL)),
+					    NULL),
+			         validate_label));
+  H_RULE (subdomain, h_sepBy1(label, h_ch('.')));
+  H_RULE (domain,    h_choice(subdomain, h_ch(' '), NULL));
 
-  const HParser *let_dig = h_choice(letter,
-				    h_ch_range('0', '9'),
-				    NULL);
-
-  const HParser *ldh_str = h_many1(h_choice(let_dig,
-					    h_ch('-'),
-					    NULL));
-
-  const HParser *label = h_attr_bool(h_sequence(letter,
-						h_optional(h_sequence(h_optional(ldh_str),
-								      let_dig,
-								      NULL)),
-						NULL),
-				     validate_label);
-
-  /**
-   * You could write it like this ...
-   *   HParser *indirect_subdomain = h_indirect();
-   *   const HParser *subdomain = h_choice(label,
-   *				           h_sequence(indirect_subdomain,
-   *					              h_ch('.'),
-   *					              label,
-   *					              NULL),
-   *				           NULL);
-   *   h_bind_indirect(indirect_subdomain, subdomain);
-   *
-   * ... but this is easier and equivalent
-   */
-
-  const HParser *subdomain = h_sepBy1(label, h_ch('.'));
-
-  domain = h_choice(subdomain, 
-		    h_ch(' '), 
-		    NULL); 
-
-  return domain;
+  ret = domain;
+  return ret;
 }
 
 const HParser* init_character_string() {

From 16b1e02baaaa0e14f2bc048466e0870043ab21ef Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:03:20 +0100
Subject: [PATCH 036/104] add an assertion to h_seq_index

---
 examples/glue.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/glue.c b/examples/glue.c
index a438b17..dd6d1b2 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -53,6 +53,7 @@ HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
 {
+  assert(p != NULL);
   assert(p->token_type == TT_SEQUENCE);
   return h_carray_index(p->seq, i);
 }

From 0083031d6f3fcdf0a1ad84cdeb899c11dd02860b Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:08:30 +0100
Subject: [PATCH 037/104] move get_domain logic into an action on the domain
 parser

---
 examples/dns.c        | 27 +++------------------------
 examples/dns.h        |  5 ++++-
 examples/dns_common.c | 29 ++++++++++++++++++++++++++++-
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index fb2b148..5904616 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -43,30 +43,9 @@ bool validate_dns(HParseResult *p) {
 }
 
 char* get_domain(const HParsedToken *t) {
-  switch(t->token_type) {
-  case TT_UINT:
-    return " ";
-  case TT_SEQUENCE:
-    {
-      // Sequence of subdomains separated by "."
-      // Each subdomain is a label, which can be no more than 63 chars.
-      char *ret = h_arena_malloc(t->seq->arena, 64*t->seq->used);
-      size_t count = 0;
-      for (size_t i=0; i<t->seq->used; ++i) {
-	HParsedToken *tmp = t->seq->elements[i];
-	for (size_t j=0; j<tmp->seq->used; ++j) {
-	  ret[count] = tmp->seq->elements[i]->uint;
-	  ++count;
-	}
-	ret[count] = '.';
-	++count;
-      }
-      ret[count-1] = '\x00';
-      return ret;
-    }
-  default:
-    return NULL;
-  }
+  assert(t != NULL);
+  assert(t->token_type == (HTokenType)TT_dns_domain);
+  return t->user;
 }
 
 uint8_t* get_cs(const HCountedArray *arr) {
diff --git a/examples/dns.h b/examples/dns.h
index 672e3b7..913b186 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -6,7 +6,8 @@ enum DNSTokenType_ {
   TT_dns_label,
   TT_dns_qname,
   TT_dns_question,
-  TT_dns_rr
+  TT_dns_rr,
+  TT_dns_domain
 };
 
 typedef struct dns_header {
@@ -98,6 +99,8 @@ typedef struct dns_rr {
   };
 } dns_rr_t;
 
+typedef char *dns_domain_t;
+
 typedef struct dns_message {
   dns_header_t header;
   dns_question_t *questions;
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 0b07cf8..d25abe2 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -1,5 +1,6 @@
 #include "../src/hammer.h"
 #include "dns_common.h"
+#include "dns.h"
 
 #define false 0
 #define true 1
@@ -13,6 +14,32 @@ bool validate_label(HParseResult *p) {
   return (64 > p->ast->seq->used);
 }
 
+const HParsedToken* act_domain(const HParseResult *p) {
+  switch(p->ast->token_type) {
+  case TT_UINT:
+    return H_MAKE_TOKEN(dns_domain, " ");
+  case TT_SEQUENCE:
+    {
+      // Sequence of subdomains separated by "."
+      // Each subdomain is a label, which can be no more than 63 chars.
+      char *ret = h_arena_malloc(p->arena, 64*p->ast->seq->used);
+      size_t count = 0;
+      for (size_t i=0; i<p->ast->seq->used; ++i) {
+	HParsedToken *tmp = p->ast->seq->elements[i];
+	for (size_t j=0; j<tmp->seq->used; ++j) {
+	  ret[count] = tmp->seq->elements[i]->uint;
+	  ++count;
+	}
+	ret[count] = '.';
+	++count;
+      }
+      ret[count-1] = '\x00';
+      return H_MAKE_TOKEN(dns_domain, ret);
+    }
+  default:
+    return NULL;
+  }
+}
 
 const HParser* init_domain() {
   static const HParser *ret = NULL;
@@ -29,7 +56,7 @@ const HParser* init_domain() {
 					    NULL),
 			         validate_label));
   H_RULE (subdomain, h_sepBy1(label, h_ch('.')));
-  H_RULE (domain,    h_choice(subdomain, h_ch(' '), NULL));
+  H_ARULE(domain,    h_choice(subdomain, h_ch(' '), NULL));
 
   ret = domain;
   return ret;

From e54c5236ff6d5c34e19939d2844331a2a511adf4 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:31:46 +0100
Subject: [PATCH 038/104] replace get_domain calls in set_rr with H_FIELD

---
 examples/dns.c        | 74 +++++++++++++++++++++----------------------
 examples/dns_common.c | 43 +++++++++++++++----------
 2 files changed, 64 insertions(+), 53 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 5904616..6acf5af 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -92,83 +92,83 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
   };
 
   // Parse rdata if possible.
-  const HParseResult *r = NULL;
+  const HParseResult *p = NULL;
   if (rr.type < sizeof(parsers)) {
-    const HParser *p = parsers[rr.type];
-    if (p)
-      r = h_parse(p, (const uint8_t*)data, rdata->used);
+    const HParser *parser = parsers[rr.type];
+    if (parser)
+      p = h_parse(parser, (const uint8_t*)data, rdata->used);
   }
 	
   // If the RR doesn't parse, set its type to 0.
-  if (!r) 
+  if (!p) 
     rr.type = 0;
 
   // Pack the parsed rdata into rr.
   switch(rr.type) {
   case 1: // A
-    rr.a = r->ast->seq->elements[0]->uint;
+    rr.a = p->ast->seq->elements[0]->uint;
     break;
   case 2: // NS
-    rr.ns = get_domain(r->ast->seq->elements[0]);
+    rr.ns = *H_FIELD(dns_domain, 0);
     break;
   case 3: // MD
-    rr.md = get_domain(r->ast->seq->elements[0]);
+    rr.md = *H_FIELD(dns_domain, 0);
     break;
   case 4: // MF
-    rr.md = get_domain(r->ast->seq->elements[0]);
+    rr.md = *H_FIELD(dns_domain, 0);
     break;
   case 5: // CNAME
-    rr.cname = get_domain(r->ast->seq->elements[0]);
+    rr.cname = *H_FIELD(dns_domain, 0);
     break;
   case 6: // SOA
-    rr.soa.mname = get_domain(r->ast->seq->elements[0]);
-    rr.soa.rname = get_domain(r->ast->seq->elements[1]);
-    rr.soa.serial = r->ast->seq->elements[2]->uint;
-    rr.soa.refresh = r->ast->seq->elements[3]->uint;
-    rr.soa.retry = r->ast->seq->elements[4]->uint;
-    rr.soa.expire = r->ast->seq->elements[5]->uint;
-    rr.soa.minimum = r->ast->seq->elements[6]->uint;
+    rr.soa.mname = *H_FIELD(dns_domain, 0);
+    rr.soa.rname = *H_FIELD(dns_domain, 1);
+    rr.soa.serial = p->ast->seq->elements[2]->uint;
+    rr.soa.refresh = p->ast->seq->elements[3]->uint;
+    rr.soa.retry = p->ast->seq->elements[4]->uint;
+    rr.soa.expire = p->ast->seq->elements[5]->uint;
+    rr.soa.minimum = p->ast->seq->elements[6]->uint;
     break;
   case 7: // MB
-    rr.mb = get_domain(r->ast->seq->elements[0]);
+    rr.mb = *H_FIELD(dns_domain, 0);
     break;
   case 8: // MG
-    rr.mg = get_domain(r->ast->seq->elements[0]);
+    rr.mg = *H_FIELD(dns_domain, 0);
     break;
   case 9: // MR
-    rr.mr = get_domain(r->ast->seq->elements[0]);
+    rr.mr = *H_FIELD(dns_domain, 0);
     break;
   case 10: // NULL
-    rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->used);
-    for (size_t i=0; i<r->ast->seq->used; ++i)
-      rr.null[i] = r->ast->seq->elements[i]->uint;
+    rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->used);
+    for (size_t i=0; i<p->ast->seq->used; ++i)
+      rr.null[i] = p->ast->seq->elements[i]->uint;
     break;
   case 11: // WKS
-    rr.wks.address = r->ast->seq->elements[0]->uint;
-    rr.wks.protocol = r->ast->seq->elements[1]->uint;
-    rr.wks.len = r->ast->seq->elements[2]->seq->used;
-    rr.wks.bit_map = h_arena_malloc(rdata->arena, sizeof(uint8_t)*r->ast->seq->elements[2]->seq->used);
+    rr.wks.address = p->ast->seq->elements[0]->uint;
+    rr.wks.protocol = p->ast->seq->elements[1]->uint;
+    rr.wks.len = p->ast->seq->elements[2]->seq->used;
+    rr.wks.bit_map = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->elements[2]->seq->used);
     for (size_t i=0; i<rr.wks.len; ++i)
-      rr.wks.bit_map[i] = r->ast->seq->elements[2]->seq->elements[i]->uint;
+      rr.wks.bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
     break;
   case 12: // PTR
-    rr.ptr = get_domain(r->ast->seq->elements[0]);
+    rr.ptr = *H_FIELD(dns_domain, 0);
     break;
   case 13: // HINFO
-    rr.hinfo.cpu = get_cs(r->ast->seq->elements[0]->seq);
-    rr.hinfo.os = get_cs(r->ast->seq->elements[1]->seq);
+    rr.hinfo.cpu = get_cs(p->ast->seq->elements[0]->seq);
+    rr.hinfo.os = get_cs(p->ast->seq->elements[1]->seq);
     break;
   case 14: // MINFO
-    rr.minfo.rmailbx = get_domain(r->ast->seq->elements[0]);
-    rr.minfo.emailbx = get_domain(r->ast->seq->elements[1]);
+    rr.minfo.rmailbx = *H_FIELD(dns_domain, 0);
+    rr.minfo.emailbx = *H_FIELD(dns_domain, 1);
     break;
   case 15: // MX
-    rr.mx.preference = r->ast->seq->elements[0]->uint;
-    rr.mx.exchange = get_domain(r->ast->seq->elements[1]);
+    rr.mx.preference = p->ast->seq->elements[0]->uint;
+    rr.mx.exchange = *H_FIELD(dns_domain, 1);
     break;
   case 16: // TXT
-    rr.txt.count = r->ast->seq->elements[0]->seq->used;
-    rr.txt.txt_data = get_txt(r->ast->seq->elements[0]->seq);
+    rr.txt.count = p->ast->seq->elements[0]->seq->used;
+    rr.txt.txt_data = get_txt(p->ast->seq->elements[0]->seq);
     break;
   default:
     break;
diff --git a/examples/dns_common.c b/examples/dns_common.c
index d25abe2..f05a3e6 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -15,30 +15,41 @@ bool validate_label(HParseResult *p) {
 }
 
 const HParsedToken* act_domain(const HParseResult *p) {
+  const HParsedToken *ret = NULL;
+  char *arr = NULL;
+
   switch(p->ast->token_type) {
   case TT_UINT:
-    return H_MAKE_TOKEN(dns_domain, " ");
+    arr = " ";
+    break;
   case TT_SEQUENCE:
-    {
-      // Sequence of subdomains separated by "."
-      // Each subdomain is a label, which can be no more than 63 chars.
-      char *ret = h_arena_malloc(p->arena, 64*p->ast->seq->used);
-      size_t count = 0;
-      for (size_t i=0; i<p->ast->seq->used; ++i) {
-	HParsedToken *tmp = p->ast->seq->elements[i];
-	for (size_t j=0; j<tmp->seq->used; ++j) {
-	  ret[count] = tmp->seq->elements[i]->uint;
-	  ++count;
-	}
-	ret[count] = '.';
+    // Sequence of subdomains separated by "."
+    // Each subdomain is a label, which can be no more than 63 chars.
+    arr = h_arena_malloc(p->arena, 64*p->ast->seq->used);
+    size_t count = 0;
+    for (size_t i=0; i<p->ast->seq->used; ++i) {
+      HParsedToken *tmp = p->ast->seq->elements[i];
+      for (size_t j=0; j<tmp->seq->used; ++j) {
+	arr[count] = tmp->seq->elements[i]->uint;
 	++count;
       }
-      ret[count-1] = '\x00';
-      return H_MAKE_TOKEN(dns_domain, ret);
+      arr[count] = '.';
+      ++count;
     }
+    arr[count-1] = '\x00';
+    break;
   default:
-    return NULL;
+    arr = NULL;
+    ret = NULL;
   }
+
+  if(arr) {
+    dns_domain_t *val = H_MAKE(dns_domain);  // dns_domain_t is char*
+    *val = arr;
+    ret = H_MAKE_TOKEN(dns_domain, val);
+  }
+
+  return ret;
 }
 
 const HParser* init_domain() {

From ccd304ddd7f8996db57e411d05c524c9c66fa3d6 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:36:23 +0100
Subject: [PATCH 039/104] replace remaining get_domain calls and remove
 get_domain

---
 examples/dns.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 6acf5af..dc10832 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -42,12 +42,6 @@ bool validate_dns(HParseResult *p) {
   return true;
 }
 
-char* get_domain(const HParsedToken *t) {
-  assert(t != NULL);
-  assert(t->token_type == (HTokenType)TT_dns_domain);
-  return t->user;
-}
-
 uint8_t* get_cs(const HCountedArray *arr) {
   uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
   for (size_t i=0; i<arr->used; ++i)
@@ -246,7 +240,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *answers = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->answer_count));
   for (size_t i=0; i<header->answer_count; ++i) {
-    answers[i].name = get_domain(rrs[i].seq->elements[0]);
+    answers[i].name = *H_SEQ_INDEX(dns_domain, rrs+i, 0);
     answers[i].type = rrs[i].seq->elements[1]->uint;
     answers[i].class = rrs[i].seq->elements[2]->uint;
     answers[i].ttl = rrs[i].seq->elements[3]->uint;
@@ -258,7 +252,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *authority = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->authority_count));
   for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
-    authority[i].name = get_domain(rrs[j].seq->elements[0]);
+    authority[i].name = *H_SEQ_INDEX(dns_domain, rrs+j, 0);
     authority[i].type = rrs[j].seq->elements[1]->uint;
     authority[i].class = rrs[j].seq->elements[2]->uint;
     authority[i].ttl = rrs[j].seq->elements[3]->uint;
@@ -270,7 +264,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *additional = h_arena_malloc(p->arena,
 					     sizeof(struct dns_rr)*(header->additional_count));
   for (size_t i=0, j=header->answer_count+header->authority_count; i<header->additional_count; ++i, ++j) {
-    additional[i].name = get_domain(rrs[j].seq->elements[0]);
+    additional[i].name = *H_SEQ_INDEX(dns_domain, rrs+j, 0);
     additional[i].type = rrs[j].seq->elements[1]->uint;
     additional[i].class = rrs[j].seq->elements[2]->uint;
     additional[i].ttl = rrs[j].seq->elements[3]->uint;

From ac3ad6d690febd5ee35e4d17b0c81dd60e5e239c Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:40:49 +0100
Subject: [PATCH 040/104] remove left-over definitions that have moved to
 glue.h

---
 examples/glue.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index dd6d1b2..767f853 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -39,12 +39,6 @@ HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value)
   return ret;
 }
 
-#define H_MAKE(TYP) \
-  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
-
-#define H_MAKE_TOKEN(TYP, VAL) \
-  h_make_token(p->arena, TT_ ## TYP, VAL)
-
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 {
   assert(i < a->used);

From 4f1e9ad22d0d5144ecd3cc9bde6d4aca66512538 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 16:41:23 +0100
Subject: [PATCH 041/104] don't add _t type suffix inside H_ macros

---
 examples/dns.c  | 56 ++++++++++++++++++++++++-------------------------
 examples/dns.h  | 14 ++++++-------
 examples/glue.h |  4 ++--
 3 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index dc10832..026287c 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -28,7 +28,7 @@ bool is_zero(HParseResult *p) {
 bool validate_dns(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
-  dns_header_t *header = H_FIELD(dns_header, 0);
+  dns_header_t *header = H_FIELD(dns_header_t, 0);
   size_t qd = header->question_count;
   size_t an = header->answer_count;
   size_t ns = header->authority_count;
@@ -103,20 +103,20 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
     rr.a = p->ast->seq->elements[0]->uint;
     break;
   case 2: // NS
-    rr.ns = *H_FIELD(dns_domain, 0);
+    rr.ns = *H_FIELD(dns_domain_t, 0);
     break;
   case 3: // MD
-    rr.md = *H_FIELD(dns_domain, 0);
+    rr.md = *H_FIELD(dns_domain_t, 0);
     break;
   case 4: // MF
-    rr.md = *H_FIELD(dns_domain, 0);
+    rr.md = *H_FIELD(dns_domain_t, 0);
     break;
   case 5: // CNAME
-    rr.cname = *H_FIELD(dns_domain, 0);
+    rr.cname = *H_FIELD(dns_domain_t, 0);
     break;
   case 6: // SOA
-    rr.soa.mname = *H_FIELD(dns_domain, 0);
-    rr.soa.rname = *H_FIELD(dns_domain, 1);
+    rr.soa.mname = *H_FIELD(dns_domain_t, 0);
+    rr.soa.rname = *H_FIELD(dns_domain_t, 1);
     rr.soa.serial = p->ast->seq->elements[2]->uint;
     rr.soa.refresh = p->ast->seq->elements[3]->uint;
     rr.soa.retry = p->ast->seq->elements[4]->uint;
@@ -124,13 +124,13 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
     rr.soa.minimum = p->ast->seq->elements[6]->uint;
     break;
   case 7: // MB
-    rr.mb = *H_FIELD(dns_domain, 0);
+    rr.mb = *H_FIELD(dns_domain_t, 0);
     break;
   case 8: // MG
-    rr.mg = *H_FIELD(dns_domain, 0);
+    rr.mg = *H_FIELD(dns_domain_t, 0);
     break;
   case 9: // MR
-    rr.mr = *H_FIELD(dns_domain, 0);
+    rr.mr = *H_FIELD(dns_domain_t, 0);
     break;
   case 10: // NULL
     rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->used);
@@ -146,19 +146,19 @@ void set_rr(struct dns_rr rr, HCountedArray *rdata) {
       rr.wks.bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
     break;
   case 12: // PTR
-    rr.ptr = *H_FIELD(dns_domain, 0);
+    rr.ptr = *H_FIELD(dns_domain_t, 0);
     break;
   case 13: // HINFO
     rr.hinfo.cpu = get_cs(p->ast->seq->elements[0]->seq);
     rr.hinfo.os = get_cs(p->ast->seq->elements[1]->seq);
     break;
   case 14: // MINFO
-    rr.minfo.rmailbx = *H_FIELD(dns_domain, 0);
-    rr.minfo.emailbx = *H_FIELD(dns_domain, 1);
+    rr.minfo.rmailbx = *H_FIELD(dns_domain_t, 0);
+    rr.minfo.emailbx = *H_FIELD(dns_domain_t, 1);
     break;
   case 15: // MX
     rr.mx.preference = p->ast->seq->elements[0]->uint;
-    rr.mx.exchange = *H_FIELD(dns_domain, 1);
+    rr.mx.exchange = *H_FIELD(dns_domain_t, 1);
     break;
   case 16: // TXT
     rr.txt.count = p->ast->seq->elements[0]->seq->used;
@@ -186,14 +186,14 @@ const HParsedToken* act_header(const HParseResult *p) {
     .additional_count = fields[11]->uint
   };
 
-  dns_header_t *header = H_MAKE(dns_header);
+  dns_header_t *header = H_MAKE(dns_header_t);
   *header = header_;
 
-  return H_MAKE_TOKEN(dns_header, header);
+  return H_MAKE_TOKEN(dns_header_t, header);
 }
 
 const HParsedToken* act_label(const HParseResult *p) {
-  dns_label_t *r = H_MAKE(dns_label);
+  dns_label_t *r = H_MAKE(dns_label_t);
 
   r->len = p->ast->seq->used;
   r->label = h_arena_malloc(p->arena, r->len + 1);
@@ -201,38 +201,38 @@ const HParsedToken* act_label(const HParseResult *p) {
     r->label[i] = p->ast->seq->elements[i]->uint;
   r->label[r->len] = 0;
 
-  return H_MAKE_TOKEN(dns_label, r);
+  return H_MAKE_TOKEN(dns_label_t, r);
 }
 
 const HParsedToken* act_question(const HParseResult *p) {
-  dns_question_t *q = H_MAKE(dns_question);
+  dns_question_t *q = H_MAKE(dns_question_t);
   HParsedToken **fields = p->ast->seq->elements;
 
   // QNAME is a sequence of labels. Pack them into an array.
   q->qname.qlen   = fields[0]->seq->used;
   q->qname.labels = h_arena_malloc(p->arena, sizeof(dns_label_t)*q->qname.qlen);
   for(size_t i=0; i<fields[0]->seq->used; i++) {
-    q->qname.labels[i] = *H_SEQ_INDEX(dns_label, fields[0], i);
+    q->qname.labels[i] = *H_SEQ_INDEX(dns_label_t, fields[0], i);
   }
 
   q->qtype  = fields[1]->uint;
   q->qclass = fields[2]->uint;
 
-  return H_MAKE_TOKEN(dns_question, q);
+  return H_MAKE_TOKEN(dns_question_t, q);
 }
 
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
-  dns_message_t *msg = H_MAKE(dns_message);
+  dns_message_t *msg = H_MAKE(dns_message_t);
 
-  dns_header_t *header = H_FIELD(dns_header, 0);
+  dns_header_t *header = H_FIELD(dns_header_t, 0);
   msg->header = *header;
 
   HParsedToken *qs = p->ast->seq->elements[1];
   struct dns_question *questions = h_arena_malloc(p->arena,
 						  sizeof(struct dns_question)*(header->question_count));
   for (size_t i=0; i<header->question_count; ++i) {
-    questions[i] = *H_SEQ_INDEX(dns_question, qs, i);
+    questions[i] = *H_SEQ_INDEX(dns_question_t, qs, i);
   }
   msg->questions = questions;
 
@@ -240,7 +240,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *answers = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->answer_count));
   for (size_t i=0; i<header->answer_count; ++i) {
-    answers[i].name = *H_SEQ_INDEX(dns_domain, rrs+i, 0);
+    answers[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+i, 0);
     answers[i].type = rrs[i].seq->elements[1]->uint;
     answers[i].class = rrs[i].seq->elements[2]->uint;
     answers[i].ttl = rrs[i].seq->elements[3]->uint;
@@ -252,7 +252,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *authority = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->authority_count));
   for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
-    authority[i].name = *H_SEQ_INDEX(dns_domain, rrs+j, 0);
+    authority[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+j, 0);
     authority[i].type = rrs[j].seq->elements[1]->uint;
     authority[i].class = rrs[j].seq->elements[2]->uint;
     authority[i].ttl = rrs[j].seq->elements[3]->uint;
@@ -264,7 +264,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *additional = h_arena_malloc(p->arena,
 					     sizeof(struct dns_rr)*(header->additional_count));
   for (size_t i=0, j=header->answer_count+header->authority_count; i<header->additional_count; ++i, ++j) {
-    additional[i].name = *H_SEQ_INDEX(dns_domain, rrs+j, 0);
+    additional[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+j, 0);
     additional[i].type = rrs[j].seq->elements[1]->uint;
     additional[i].class = rrs[j].seq->elements[2]->uint;
     additional[i].ttl = rrs[j].seq->elements[3]->uint;
@@ -273,7 +273,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   }
   msg->additional = additional;
 
-  return H_MAKE_TOKEN(dns_message, msg);
+  return H_MAKE_TOKEN(dns_message_t, msg);
 }
 
 #define act_hdzero act_ignore
diff --git a/examples/dns.h b/examples/dns.h
index 913b186..57602b0 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -1,13 +1,13 @@
 #include "../src/hammer.h"
 
 enum DNSTokenType_ {
-  TT_dns_message = TT_USER,
-  TT_dns_header,
-  TT_dns_label,
-  TT_dns_qname,
-  TT_dns_question,
-  TT_dns_rr,
-  TT_dns_domain
+  TT_dns_message_t = TT_USER,
+  TT_dns_header_t,
+  TT_dns_label_t,
+  TT_dns_qname_t,
+  TT_dns_question_t,
+  TT_dns_rr_t,
+  TT_dns_domain_t
 };
 
 typedef struct dns_header {
diff --git a/examples/glue.h b/examples/glue.h
index d8776b3..88b788d 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -18,7 +18,7 @@ const HParsedToken *act_index0(const HParseResult *p);
 HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
 
 #define H_MAKE(TYP) \
-  ((TYP ## _t *) h_arena_malloc(p->arena, sizeof(TYP ## _t)))
+  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
 
 #define H_MAKE_TOKEN(TYP, VAL) \
   h_make_token(p->arena, TT_ ## TYP, VAL)
@@ -28,7 +28,7 @@ HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
 
 #define H_SEQ_INDEX(TYP, SEQ, IDX) \
-  ((TYP ## _t *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
+  ((TYP *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
 
 #define H_FIELD(TYP, IDX) \
   H_SEQ_INDEX(TYP, p->ast, IDX)

From 587143eec190b3afdfb458bd4ee1007d04259819 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 18:20:23 +0100
Subject: [PATCH 042/104] cleanup and bugfixing on domain parser

---
 examples/dns_common.c | 18 +++++++------
 examples/glue.c       | 61 ++++++++++++++++++++++++++++++++++++++++++-
 examples/glue.h       |  7 +++++
 3 files changed, 77 insertions(+), 9 deletions(-)

diff --git a/examples/dns_common.c b/examples/dns_common.c
index f05a3e6..9f02fc6 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -44,14 +44,16 @@ const HParsedToken* act_domain(const HParseResult *p) {
   }
 
   if(arr) {
-    dns_domain_t *val = H_MAKE(dns_domain);  // dns_domain_t is char*
+    dns_domain_t *val = H_MAKE(dns_domain_t);  // dns_domain_t is char*
     *val = arr;
-    ret = H_MAKE_TOKEN(dns_domain, val);
+    ret = H_MAKE_TOKEN(dns_domain_t, val);
   }
 
   return ret;
 }
 
+#define act_label_ act_flatten
+
 const HParser* init_domain() {
   static const HParser *ret = NULL;
   if (ret)
@@ -60,12 +62,12 @@ const HParser* init_domain() {
   H_RULE (letter,    h_choice(h_ch_range('a','z'), h_ch_range('A','Z'), NULL));
   H_RULE (let_dig,   h_choice(letter, h_ch_range('0','9'), NULL));
   H_RULE (ldh_str,   h_many1(h_choice(let_dig, h_ch('-'), NULL)));
-  H_RULE (label,     h_attr_bool(h_sequence(letter,
-					    h_optional(h_sequence(h_optional(ldh_str),
-								  let_dig,
-								  NULL)),
-					    NULL),
-			         validate_label));
+  H_ARULE(label_,    h_sequence(letter,
+				h_optional(h_sequence(h_optional(ldh_str),
+						      let_dig,
+						      NULL)),
+				NULL));
+  H_RULE (label,     h_attr_bool(label_, validate_label));
   H_RULE (subdomain, h_sepBy1(label, h_ch('.')));
   H_ARULE(domain,    h_choice(subdomain, h_ch(' '), NULL));
 
diff --git a/examples/glue.c b/examples/glue.c
index 767f853..b635d7c 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -1,4 +1,5 @@
 #include "glue.h"
+#include "../src/internal.h"  // for h_carray_*
 
 
 // The action equivalent of h_ignore.
@@ -31,10 +32,68 @@ const HParsedToken *act_index0(const HParseResult *p)
     return act_index(0, p);
 }
 
-HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value)
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+
+  h_carray_append(xs->seq, (void *)x);
+}
+
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+  assert(ys != NULL);
+  assert(ys->token_type == TT_SEQUENCE);
+
+  for(size_t i; i<ys->seq->used; i++)
+	h_carray_append(xs->seq, ys->seq->elements[i]);
+}
+
+// Flatten nested sequences. Always returns a sequence.
+// If input element is not a sequence, returns it as a singleton sequence.
+const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p)
+{
+  assert(p != NULL);
+
+  HParsedToken *ret = h_make_token_seq(arena);
+  switch(p->token_type) {
+  case TT_SEQUENCE:
+    // Flatten and append all.
+    for(size_t i; i<p->seq->used; i++) {
+	  h_seq_append(ret, h_token_flatten(arena, h_seq_index(p, i)));
+	}
+	break;
+  default:
+    // Make singleton sequence.
+    h_seq_snoc(ret, p);
+	break;
+  }
+
+  return ret;
+}
+
+// Action version of h_token_flatten.
+const HParsedToken *act_flatten(const HParseResult *p) {
+  return h_token_flatten(p->arena, p->ast);
+}
+
+HParsedToken *h_make_token_(HArena *arena, HTokenType type)
 {
   HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
   ret->token_type = type;
+  return ret;
+}
+
+HParsedToken *h_make_token_seq(HArena *arena)
+{
+  return h_make_token_(arena, TT_SEQUENCE);
+}
+
+HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value)
+{
+  HParsedToken *ret = h_make_token_(arena, type);
   ret->user = value;
   return ret;
 }
diff --git a/examples/glue.h b/examples/glue.h
index 88b788d..c65f7b4 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -14,8 +14,15 @@
 const HParsedToken *act_ignore(const HParseResult *p);
 const HParsedToken *act_index(int i, const HParseResult *p);
 const HParsedToken *act_index0(const HParseResult *p);
+const HParsedToken *act_flatten(const HParseResult *p);
+
+const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p);
+
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
 
 HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
+HParsedToken *h_make_token_seq(HArena *arena);
 
 #define H_MAKE(TYP) \
   ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))

From f0155d18d670c2397235fd3566b03994352a599b Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 16 Jan 2013 22:42:47 +0100
Subject: [PATCH 043/104] pull packing of RRs out of act_message into act_rr

---
 examples/dns.c | 43 ++++++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 026287c..afeb7b1 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -204,6 +204,21 @@ const HParsedToken* act_label(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_label_t, r);
 }
 
+const HParsedToken* act_rr(const HParseResult *p) {
+  dns_rr_t *rr = H_MAKE(dns_rr_t);
+
+  rr->name     = *H_FIELD(dns_domain_t, 0);
+  rr->type     = p->ast->seq->elements[1]->uint;
+  rr->class    = p->ast->seq->elements[2]->uint;
+  rr->ttl      = p->ast->seq->elements[3]->uint;
+  rr->rdlength = p->ast->seq->elements[4]->seq->used;
+
+  // Parse and pack RDATA.
+  set_rr(*rr, p->ast->seq->elements[4]->seq);	   
+
+  return H_MAKE_TOKEN(dns_rr_t, rr);
+}
+
 const HParsedToken* act_question(const HParseResult *p) {
   dns_question_t *q = H_MAKE(dns_question_t);
   HParsedToken **fields = p->ast->seq->elements;
@@ -225,9 +240,11 @@ const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
   dns_message_t *msg = H_MAKE(dns_message_t);
 
+  // Copy header into message struct.
   dns_header_t *header = H_FIELD(dns_header_t, 0);
   msg->header = *header;
 
+  // Copy questions into message struct.
   HParsedToken *qs = p->ast->seq->elements[1];
   struct dns_question *questions = h_arena_malloc(p->arena,
 						  sizeof(struct dns_question)*(header->question_count));
@@ -236,40 +253,28 @@ const HParsedToken* act_message(const HParseResult *p) {
   }
   msg->questions = questions;
 
+  // Copy answer RRs into message struct.
   HParsedToken *rrs = p->ast->seq->elements[2];
   struct dns_rr *answers = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->answer_count));
   for (size_t i=0; i<header->answer_count; ++i) {
-    answers[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+i, 0);
-    answers[i].type = rrs[i].seq->elements[1]->uint;
-    answers[i].class = rrs[i].seq->elements[2]->uint;
-    answers[i].ttl = rrs[i].seq->elements[3]->uint;
-    answers[i].rdlength = rrs[i].seq->elements[4]->seq->used;
-    set_rr(answers[i], rrs[i].seq->elements[4]->seq);	   
+    answers[i] = *H_SEQ_INDEX(dns_rr_t, rrs, i);
   }
   msg->answers = answers;
 
+  // Copy authority RRs into message struct.
   struct dns_rr *authority = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->authority_count));
   for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
-    authority[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+j, 0);
-    authority[i].type = rrs[j].seq->elements[1]->uint;
-    authority[i].class = rrs[j].seq->elements[2]->uint;
-    authority[i].ttl = rrs[j].seq->elements[3]->uint;
-    authority[i].rdlength = rrs[j].seq->elements[4]->seq->used;
-    set_rr(authority[i], rrs[j].seq->elements[4]->seq);
+    authority[i] = *H_SEQ_INDEX(dns_rr_t, rrs, j);
   }
   msg->authority = authority;
 
+  // Copy additional RRs into message struct.
   struct dns_rr *additional = h_arena_malloc(p->arena,
 					     sizeof(struct dns_rr)*(header->additional_count));
   for (size_t i=0, j=header->answer_count+header->authority_count; i<header->additional_count; ++i, ++j) {
-    additional[i].name = *H_SEQ_INDEX(dns_domain_t, rrs+j, 0);
-    additional[i].type = rrs[j].seq->elements[1]->uint;
-    additional[i].class = rrs[j].seq->elements[2]->uint;
-    additional[i].ttl = rrs[j].seq->elements[3]->uint;
-    additional[i].rdlength = rrs[j].seq->elements[4]->seq->used;
-    set_rr(additional[i], rrs[j].seq->elements[4]->seq);
+    additional[i] = *H_SEQ_INDEX(dns_rr_t, rrs, j);
   }
   msg->additional = additional;
 
@@ -320,7 +325,7 @@ const HParser* init_parser() {
 			       NULL));
   H_ARULE(question, h_sequence(qname, qtype, qclass, NULL));
   H_RULE (rdata,    h_length_value(h_uint16(), h_uint8()));
-  H_RULE (rr,       h_sequence(domain,            // NAME
+  H_ARULE (rr,      h_sequence(domain,            // NAME
 			       type,              // TYPE
 			       class,             // CLASS
 			       h_uint32(),        // TTL

From 588d534f3249e2576efe31e0794b91cce6ef3bf5 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 00:50:15 +0100
Subject: [PATCH 044/104] pull selection and initialization of RDATA parser out
 of set_rdata

---
 examples/dns.c | 35 ++++++-----------------------------
 examples/rr.c  | 33 +++++++++++++++++++++++++++++++++
 examples/rr.h  |  2 ++
 3 files changed, 41 insertions(+), 29 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index afeb7b1..4174206 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -59,39 +59,16 @@ uint8_t** get_txt(const HCountedArray *arr) {
   return ret;
 }
 
-void set_rr(struct dns_rr rr, HCountedArray *rdata) {
+void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
     data[i] = rdata->elements[i]->uint;
 
-  // Mapping numeric RR types (as indices) to parsers
-  const HParser *parsers[] = {
-    NULL,          // there is no type 0
-    init_a(),      // 1
-    init_ns(),
-    init_md(),
-    init_mf(),
-    init_cname(),  // 5
-    init_soa(),
-    init_mb(),
-    init_mg(),
-    init_mr(),
-    init_null(),   // 10
-    init_wks(),
-    init_ptr(),
-    init_hinfo(),
-    init_minfo(),
-    init_mx(),     // 15
-    init_txt()
-  };
-
-  // Parse rdata if possible.
+  // Parse RDATA if possible.
   const HParseResult *p = NULL;
-  if (rr.type < sizeof(parsers)) {
-    const HParser *parser = parsers[rr.type];
-    if (parser)
-      p = h_parse(parser, (const uint8_t*)data, rdata->used);
-  }
+  const HParser *parser = init_rdata(rr.type);
+  if (parser)
+    p = h_parse(parser, (const uint8_t*)data, rdata->used);
 	
   // If the RR doesn't parse, set its type to 0.
   if (!p) 
@@ -214,7 +191,7 @@ const HParsedToken* act_rr(const HParseResult *p) {
   rr->rdlength = p->ast->seq->elements[4]->seq->used;
 
   // Parse and pack RDATA.
-  set_rr(*rr, p->ast->seq->elements[4]->seq);	   
+  set_rdata(*rr, p->ast->seq->elements[4]->seq);	   
 
   return H_MAKE_TOKEN(dns_rr_t, rr);
 }
diff --git a/examples/rr.c b/examples/rr.c
index 8dae885..5feaf37 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -5,6 +5,39 @@
 #define false 0
 #define true 1
 
+#define RDATA_TYPE_MAX 16
+const HParser* init_rdata(uint16_t type) {
+  static const HParser *parsers[RDATA_TYPE_MAX+1];
+  static int inited = 0;
+
+  if (type > RDATA_TYPE_MAX)
+    return NULL;
+  
+  if (inited)
+    return parsers[type];
+
+  parsers[ 0] = NULL;            // there is no type 0
+  parsers[ 1] = init_a();
+  parsers[ 2] = init_ns();
+  parsers[ 3] = init_md();
+  parsers[ 4] = init_mf();
+  parsers[ 5] = init_cname();
+  parsers[ 6] = init_soa();
+  parsers[ 7] = init_mb();
+  parsers[ 8] = init_mg();
+  parsers[ 9] = init_mr();
+  parsers[10] = init_null();
+  parsers[11] = init_wks();
+  parsers[12] = init_ptr();
+  parsers[13] = init_hinfo();
+  parsers[14] = init_minfo();
+  parsers[15] = init_mx();
+  parsers[16] = init_txt();
+
+  inited = 1;
+  return parsers[type];
+}
+  
 const HParser* init_cname() {
   static const HParser *cname = NULL;
   if (cname)
diff --git a/examples/rr.h b/examples/rr.h
index 54172f8..ccac1a6 100644
--- a/examples/rr.h
+++ b/examples/rr.h
@@ -3,6 +3,8 @@
 
 #include "../src/hammer.h"
 
+const HParser* init_rdata(uint16_t type);
+
 const HParser* init_cname();
 const HParser* init_hinfo();
 const HParser* init_mb();

From 6515a80c3aade81714cebd892f825d279e16ed9f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 01:17:37 +0100
Subject: [PATCH 045/104] condense rr.c

---
 examples/dns.c |   2 +-
 examples/rr.c  | 301 +++++++++++--------------------------------------
 examples/rr.h  |  17 ---
 3 files changed, 67 insertions(+), 253 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 4174206..81c2488 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -69,7 +69,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   const HParser *parser = init_rdata(rr.type);
   if (parser)
     p = h_parse(parser, (const uint8_t*)data, rdata->used);
-	
+
   // If the RR doesn't parse, set its type to 0.
   if (!p) 
     rr.type = 0;
diff --git a/examples/rr.c b/examples/rr.c
index 5feaf37..84022ab 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -5,248 +5,79 @@
 #define false 0
 #define true 1
 
-#define RDATA_TYPE_MAX 16
-const HParser* init_rdata(uint16_t type) {
-  static const HParser *parsers[RDATA_TYPE_MAX+1];
-  static int inited = 0;
-
-  if (type > RDATA_TYPE_MAX)
-    return NULL;
-  
-  if (inited)
-    return parsers[type];
-
-  parsers[ 0] = NULL;            // there is no type 0
-  parsers[ 1] = init_a();
-  parsers[ 2] = init_ns();
-  parsers[ 3] = init_md();
-  parsers[ 4] = init_mf();
-  parsers[ 5] = init_cname();
-  parsers[ 6] = init_soa();
-  parsers[ 7] = init_mb();
-  parsers[ 8] = init_mg();
-  parsers[ 9] = init_mr();
-  parsers[10] = init_null();
-  parsers[11] = init_wks();
-  parsers[12] = init_ptr();
-  parsers[13] = init_hinfo();
-  parsers[14] = init_minfo();
-  parsers[15] = init_mx();
-  parsers[16] = init_txt();
-
-  inited = 1;
-  return parsers[type];
-}
-  
-const HParser* init_cname() {
-  static const HParser *cname = NULL;
-  if (cname)
-    return cname;
-  
-  cname = h_sequence(init_domain(),
-		     h_end_p(),
-		     NULL);
-  
-  return cname;
-}
-
-const HParser* init_hinfo() {
-  static const HParser *hinfo = NULL;
-  if (hinfo)
-    return hinfo;
-
-  const HParser* cstr = init_character_string();
-  
-  hinfo = h_sequence(cstr,
-		     cstr,
-		     h_end_p(),
-		     NULL);
-
-  return hinfo;
-}
-
-const HParser* init_mb() {
-  static const HParser *mb = NULL;
-  if (mb)
-    return mb;
-  
-  mb = h_sequence(init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return mb;
-}
-
-const HParser* init_md() {
-  static const HParser *md = NULL;
-  if (md)
-    return md;
-  
-  md = h_sequence(init_domain(),
-		  h_end_p,
-		  NULL);
-
-  return md;
-}
-
-const HParser* init_mf() {
-  static const HParser *mf = NULL;
-  if (mf)
-    return mf;
-  
-  mf = h_sequence(init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return mf;
-}
-
-const HParser* init_mg() {
-  static const HParser *mg = NULL;
-  if (mg)
-    return mg;
-  
-  mg = h_sequence(init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return mg;
-}
-
-const HParser* init_minfo() {
-  static const HParser *minfo = NULL;
-  if (minfo)
-    return minfo;
-
-  const HParser* domain = init_domain();
-
-  minfo = h_sequence(domain,
-		     domain,
-		     h_end_p(),
-		     NULL);
-
-  return minfo;
-}
-
-const HParser* init_mr() {
-  static const HParser *mr = NULL;
-  if (mr)
-    return mr;
-  
-  mr = h_sequence(init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return mr;
-}
-
-const HParser* init_mx() {
-  static const HParser *mx = NULL;
-  if (mx)
-    return mx;
-  
-  mx = h_sequence(h_uint16(),
-		  init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return mx;
-}
-
 bool validate_null(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
   return (65536 > p->ast->seq->used);
 }
 
-const HParser* init_null() {
-  static const HParser *null_ = NULL;
-  if (null_)
-    return null_;
+#define RDATA_TYPE_MAX 16
+const HParser* init_rdata(uint16_t type) {
+  static const HParser *parsers[RDATA_TYPE_MAX+1];
+  static int inited = 0;
 
-  null_ = h_attr_bool(h_many(h_uint8()), validate_null);
-
-  return null_;
-}
-
-const HParser* init_ns() {
-  static const HParser *ns = NULL;
-  if (ns)
-    return ns;
-
-  ns = h_sequence(init_domain(),
-		  h_end_p(),
-		  NULL);
-
-  return ns;
-}
-
-const HParser* init_ptr() {
-  static const HParser *ptr = NULL;
-  if (ptr)
-    return ptr;
+  if (type >= sizeof(parsers))
+    return NULL;
   
-  ptr = h_sequence(init_domain(),
-		   h_end_p(),
-		   NULL);
+  if (inited)
+    return parsers[type];
 
-  return ptr;
-}
-
-const HParser* init_soa() {
-  static const HParser *soa = NULL;
-  if (soa)
-    return soa;
-
-  const HParser *domain = init_domain();
-
-  soa = h_sequence(domain,   // MNAME
-		   domain,   // RNAME
-		   h_uint32(), // SERIAL
-		   h_uint32(), // REFRESH
-		   h_uint32(), // RETRY
-		   h_uint32(), // EXPIRE
-		   h_uint32(), // MINIMUM
-		   h_end_p(),
-		   NULL);
-
-  return soa;
-}
-
-const HParser* init_txt() {
-  static const HParser *txt = NULL;
-  if (txt)
-    return txt;
-
-  txt = h_sequence(h_many1(init_character_string()),
-		   h_end_p(),
-		   NULL);
-
-  return txt;
-}
-
-const HParser* init_a() {
-  static const HParser *a = NULL;
-  if (a)
-    return a;
-
-  a = h_sequence(h_uint32(),
-		 h_end_p(),
-		 NULL);
-
-  return a;
-}
-
-const HParser* init_wks() {
-  static const HParser *wks = NULL;
-  if (wks)
-    return wks;
-
-  wks = h_sequence(h_uint32(),
-		   h_uint8(),
-		   h_many(h_uint8()),
-		   h_end_p(),
-		   NULL);
-
-  return wks;
+
+  H_RULE (domain, init_domain());
+  H_RULE (cstr,   init_character_string());
+
+  H_RULE (a,      h_uint32());
+  H_RULE (ns,     domain);
+  H_RULE (md,     domain);
+  H_RULE (mf,     domain);
+  H_RULE (cname,  domain);
+  H_RULE (soa,    h_sequence(domain,     // MNAME
+			     domain,     // RNAME
+			     h_uint32(), // SERIAL
+			     h_uint32(), // REFRESH
+			     h_uint32(), // RETRY
+			     h_uint32(), // EXPIRE
+			     h_uint32(), // MINIMUM
+			     NULL));
+  H_RULE (mb,     domain);
+  H_RULE (mg,     domain);
+  H_RULE (mr,     domain);
+  H_RULE (null_,  h_attr_bool(h_many(h_uint8()), validate_null));
+  H_RULE (wks,    h_sequence(h_uint32(),
+			     h_uint8(),
+			     h_many(h_uint8()),
+			     NULL));
+  H_RULE (ptr,    domain);
+  H_RULE (hinfo,  h_sequence(cstr, cstr, NULL));
+  H_RULE (minfo,  h_sequence(domain, domain, NULL));
+  H_RULE (mx,     h_sequence(h_uint16(), domain, NULL));
+  H_RULE (txt,    h_many1(cstr));
+
+
+  parsers[ 0] = NULL;            // there is no type 0
+  parsers[ 1] = a;
+  parsers[ 2] = ns;
+  parsers[ 3] = md;
+  parsers[ 4] = mf;
+  parsers[ 5] = cname;
+  parsers[ 6] = soa;
+  parsers[ 7] = mb;
+  parsers[ 8] = mg;
+  parsers[ 9] = mr;
+  parsers[10] = null_;
+  parsers[11] = wks;
+  parsers[12] = ptr;
+  parsers[13] = hinfo;
+  parsers[14] = minfo;
+  parsers[15] = mx;
+  parsers[16] = txt;
+
+  // All parsers must consume their input exactly.
+  for(uint16_t i; i<sizeof(parsers); i++) {
+    parsers[i] = h_action(h_sequence(parsers[i], h_end_p(), NULL),
+			  act_index0);
+  }
+
+  inited = 1;
+  return parsers[type];
 }
diff --git a/examples/rr.h b/examples/rr.h
index ccac1a6..fce4578 100644
--- a/examples/rr.h
+++ b/examples/rr.h
@@ -5,21 +5,4 @@
 
 const HParser* init_rdata(uint16_t type);
 
-const HParser* init_cname();
-const HParser* init_hinfo();
-const HParser* init_mb();
-const HParser* init_md();
-const HParser* init_mf();
-const HParser* init_mg();
-const HParser* init_minfo();
-const HParser* init_mr();
-const HParser* init_mx();
-const HParser* init_null();
-const HParser* init_ns();
-const HParser* init_ptr();
-const HParser* init_soa();
-const HParser* init_txt();
-const HParser* init_a();
-const HParser* init_wks();
-
 #endif

From e875dcd19a8ecde7f980672d3d8f06a94d22eb33 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 17:44:41 +0100
Subject: [PATCH 046/104] add "VRULE" family of macros to attach validations
 like actions

---
 examples/dns.c  | 95 +++++++++++++++++++++++++------------------------
 examples/glue.h |  6 ++++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 81c2488..b02a4ba 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -12,10 +12,10 @@
 
 
 ///
-// Semantic Actions and Validations
+// Validations
 ///
 
-bool is_zero(HParseResult *p) {
+bool validate_hdzero(HParseResult *p) {
   if (TT_UINT != p->ast->token_type)
     return false;
   return (0 == p->ast->uint);
@@ -25,7 +25,7 @@ bool is_zero(HParseResult *p) {
  * Every DNS message should have QDCOUNT entries in the question
  * section, and ANCOUNT+NSCOUNT+ARCOUNT resource records.
  */
-bool validate_dns(HParseResult *p) {
+bool validate_message(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
   dns_header_t *header = H_FIELD(dns_header_t, 0);
@@ -42,6 +42,10 @@ bool validate_dns(HParseResult *p) {
   return true;
 }
 
+///
+// Semantic Actions
+///
+
 uint8_t* get_cs(const HCountedArray *arr) {
   uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
   for (size_t i=0; i<arr->used; ++i)
@@ -271,49 +275,48 @@ const HParser* init_parser() {
   if (ret)
     return ret;
 
-  H_RULE (domain,   init_domain());
-  H_ARULE(hdzero,   h_attr_bool(h_bits(3, false), is_zero));
-  H_ARULE(header,   h_sequence(h_bits(16, false), // ID
-			       h_bits(1, false),  // QR
-			       h_bits(4, false),  // opcode
-			       h_bits(1, false),  // AA
-			       h_bits(1, false),  // TC
-			       h_bits(1, false),  // RD
-			       h_bits(1, false),  // RA
-			       hdzero,            // Z
-			       h_bits(4, false),  // RCODE
-			       h_uint16(),        // QDCOUNT
-			       h_uint16(),        // ANCOUNT
-			       h_uint16(),        // NSCOUNT
-			       h_uint16(),        // ARCOUNT
-			       NULL));
-  H_RULE (type,     h_int_range(h_uint16(), 1, 16));
-  H_RULE (qtype,    h_choice(type, 
-			     h_int_range(h_uint16(), 252, 255),
-			     NULL));
-  H_RULE (class,    h_int_range(h_uint16(), 1, 4));
-  H_RULE (qclass,   h_choice(class,
-			     h_int_range(h_uint16(), 255, 255),
-			     NULL));
-  H_RULE (len,      h_int_range(h_uint8(), 1, 255));
-  H_ARULE(label,    h_length_value(len, h_uint8())); 
-  H_ARULE(qname,    h_sequence(h_many1(label),
-			       h_ch('\x00'),
-			       NULL));
-  H_ARULE(question, h_sequence(qname, qtype, qclass, NULL));
-  H_RULE (rdata,    h_length_value(h_uint16(), h_uint8()));
-  H_ARULE (rr,      h_sequence(domain,            // NAME
-			       type,              // TYPE
-			       class,             // CLASS
-			       h_uint32(),        // TTL
-			       rdata,             // RDLENGTH+RDATA
-			       NULL));
-  H_ARULE(message,  h_attr_bool(h_sequence(header,
-					   h_many(question),
-					   h_many(rr),
-					   h_end_p(),
-					   NULL),
-			        validate_dns));
+  H_RULE  (domain,   init_domain());
+  H_AVRULE(hdzero,   h_bits(3, false));
+  H_ARULE (header,   h_sequence(h_bits(16, false), // ID
+				h_bits(1, false),  // QR
+				h_bits(4, false),  // opcode
+				h_bits(1, false),  // AA
+				h_bits(1, false),  // TC
+				h_bits(1, false),  // RD
+				h_bits(1, false),  // RA
+				hdzero,            // Z
+				h_bits(4, false),  // RCODE
+				h_uint16(),        // QDCOUNT
+				h_uint16(),        // ANCOUNT
+				h_uint16(),        // NSCOUNT
+				h_uint16(),        // ARCOUNT
+				NULL));
+  H_RULE  (type,     h_int_range(h_uint16(), 1, 16));
+  H_RULE  (qtype,    h_choice(type, 
+			      h_int_range(h_uint16(), 252, 255),
+			      NULL));
+  H_RULE  (class,    h_int_range(h_uint16(), 1, 4));
+  H_RULE  (qclass,   h_choice(class,
+			      h_int_range(h_uint16(), 255, 255),
+			      NULL));
+  H_RULE  (len,      h_int_range(h_uint8(), 1, 255));
+  H_ARULE (label,    h_length_value(len, h_uint8())); 
+  H_ARULE (qname,    h_sequence(h_many1(label),
+				h_ch('\x00'),
+				NULL));
+  H_ARULE (question, h_sequence(qname, qtype, qclass, NULL));
+  H_RULE  (rdata,    h_length_value(h_uint16(), h_uint8()));
+  H_ARULE (rr,       h_sequence(domain,            // NAME
+				type,              // TYPE
+				class,             // CLASS
+				h_uint32(),        // TTL
+				rdata,             // RDLENGTH+RDATA
+				NULL));
+  H_AVRULE(message,  h_sequence(header,
+				h_many(question),
+				h_many(rr),
+				h_end_p(),
+				NULL));
 
   ret = message;
   return ret;
diff --git a/examples/glue.h b/examples/glue.h
index c65f7b4..7939a48 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -10,6 +10,12 @@
 
 #define H_RULE(rule, def) const HParser *rule = def
 #define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+#define H_VRULE(rule, def) const HParser *rule = \
+  h_attr_bool(def, validate_ ## rule)
+#define H_VARULE(rule, def) const HParser *rule = \
+  h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule)
+#define H_AVRULE(rule, def) const HParser *rule = \
+  h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
 
 const HParsedToken *act_ignore(const HParseResult *p);
 const HParsedToken *act_index(int i, const HParseResult *p);

From dd574bd735ba1813e0fe1a2a3b0a80967481eed8 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 18:31:13 +0100
Subject: [PATCH 047/104] give glue actions an h_ prefix and add H_ACT_APPLY
 macro

---
 examples/dns.c        |  3 ++-
 examples/dns_common.c |  4 +++-
 examples/dns_common.h |  2 ++
 examples/glue.c       | 11 +++--------
 examples/glue.h       | 14 ++++++++++----
 5 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index b02a4ba..ba1f592 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -262,10 +262,11 @@ const HParsedToken* act_message(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_message_t, msg);
 }
 
-#define act_hdzero act_ignore
+#define act_hdzero h_act_ignore
 #define act_qname  act_index0
 
 
+
 ///
 // Parser / Grammar
 ///
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 9f02fc6..57f81aa 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -5,6 +5,8 @@
 #define false 0
 #define true 1
 
+H_ACT_APPLY(act_index0, h_act_index, 0)
+
 /**
  * A label can't be more than 63 characters.
  */
@@ -52,7 +54,7 @@ const HParsedToken* act_domain(const HParseResult *p) {
   return ret;
 }
 
-#define act_label_ act_flatten
+#define act_label_ h_act_flatten
 
 const HParser* init_domain() {
   static const HParser *ret = NULL;
diff --git a/examples/dns_common.h b/examples/dns_common.h
index 2d796f8..6b04519 100644
--- a/examples/dns_common.h
+++ b/examples/dns_common.h
@@ -7,4 +7,6 @@
 const HParser* init_domain();
 const HParser* init_character_string();
 
+const HParsedToken* act_index0(const HParseResult *p);
+
 #endif
diff --git a/examples/glue.c b/examples/glue.c
index b635d7c..80864fc 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -3,13 +3,13 @@
 
 
 // The action equivalent of h_ignore.
-const HParsedToken *act_ignore(const HParseResult *p)
+const HParsedToken *h_act_ignore(const HParseResult *p)
 {
   return NULL;
 }
 
 // Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *act_index(int i, const HParseResult *p)
+const HParsedToken *h_act_index(int i, const HParseResult *p)
 {
     if(!p) return NULL;
 
@@ -27,11 +27,6 @@ const HParsedToken *act_index(int i, const HParseResult *p)
         return tok->seq->elements[i];
 }
 
-const HParsedToken *act_index0(const HParseResult *p)
-{
-    return act_index(0, p);
-}
-
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
 {
   assert(xs != NULL);
@@ -75,7 +70,7 @@ const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p)
 }
 
 // Action version of h_token_flatten.
-const HParsedToken *act_flatten(const HParseResult *p) {
+const HParsedToken *h_act_flatten(const HParseResult *p) {
   return h_token_flatten(p->arena, p->ast);
 }
 
diff --git a/examples/glue.h b/examples/glue.h
index 7939a48..2b60646 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -17,10 +17,16 @@
 #define H_AVRULE(rule, def) const HParser *rule = \
   h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
 
-const HParsedToken *act_ignore(const HParseResult *p);
-const HParsedToken *act_index(int i, const HParseResult *p);
-const HParsedToken *act_index0(const HParseResult *p);
-const HParsedToken *act_flatten(const HParseResult *p);
+const HParsedToken *h_act_ignore(const HParseResult *p);
+const HParsedToken *h_act_index(int i, const HParseResult *p);
+const HParsedToken *h_act_flatten(const HParseResult *p);
+
+// Define 'myaction' as a specialization of 'paction' by supplying the leading
+// parameters.
+#define H_ACT_APPLY(myaction, paction, ...) \
+  const HParsedToken *myaction(const HParseResult *p) { \
+    return paction(__VA_ARGS__, p); \
+  }
 
 const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p);
 

From 954a762fc0c184627cfe3273d73739b7ee8a9855 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 18:34:11 +0100
Subject: [PATCH 048/104] replace remaining h_attr_bool calls with VRULEs

---
 examples/dns_common.c | 25 ++++++++++++-------------
 examples/rr.c         |  4 ++--
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/examples/dns_common.c b/examples/dns_common.c
index 57f81aa..3ea5594 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -16,6 +16,8 @@ bool validate_label(HParseResult *p) {
   return (64 > p->ast->seq->used);
 }
 
+#define act_label h_act_flatten
+
 const HParsedToken* act_domain(const HParseResult *p) {
   const HParsedToken *ret = NULL;
   char *arr = NULL;
@@ -54,24 +56,21 @@ const HParsedToken* act_domain(const HParseResult *p) {
   return ret;
 }
 
-#define act_label_ h_act_flatten
-
 const HParser* init_domain() {
   static const HParser *ret = NULL;
   if (ret)
     return ret;
 
-  H_RULE (letter,    h_choice(h_ch_range('a','z'), h_ch_range('A','Z'), NULL));
-  H_RULE (let_dig,   h_choice(letter, h_ch_range('0','9'), NULL));
-  H_RULE (ldh_str,   h_many1(h_choice(let_dig, h_ch('-'), NULL)));
-  H_ARULE(label_,    h_sequence(letter,
-				h_optional(h_sequence(h_optional(ldh_str),
-						      let_dig,
-						      NULL)),
-				NULL));
-  H_RULE (label,     h_attr_bool(label_, validate_label));
-  H_RULE (subdomain, h_sepBy1(label, h_ch('.')));
-  H_ARULE(domain,    h_choice(subdomain, h_ch(' '), NULL));
+  H_RULE  (letter,    h_choice(h_ch_range('a','z'), h_ch_range('A','Z'), NULL));
+  H_RULE  (let_dig,   h_choice(letter, h_ch_range('0','9'), NULL));
+  H_RULE  (ldh_str,   h_many1(h_choice(let_dig, h_ch('-'), NULL)));
+  H_VARULE(label,     h_sequence(letter,
+				 h_optional(h_sequence(h_optional(ldh_str),
+						       let_dig,
+						       NULL)),
+				 NULL));
+  H_RULE  (subdomain, h_sepBy1(label, h_ch('.')));
+  H_ARULE (domain,    h_choice(subdomain, h_ch(' '), NULL));
 
   ret = domain;
   return ret;
diff --git a/examples/rr.c b/examples/rr.c
index 84022ab..f252953 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -42,7 +42,7 @@ const HParser* init_rdata(uint16_t type) {
   H_RULE (mb,     domain);
   H_RULE (mg,     domain);
   H_RULE (mr,     domain);
-  H_RULE (null_,  h_attr_bool(h_many(h_uint8()), validate_null));
+  H_VRULE(null,   h_many(h_uint8()));
   H_RULE (wks,    h_sequence(h_uint32(),
 			     h_uint8(),
 			     h_many(h_uint8()),
@@ -64,7 +64,7 @@ const HParser* init_rdata(uint16_t type) {
   parsers[ 7] = mb;
   parsers[ 8] = mg;
   parsers[ 9] = mr;
-  parsers[10] = null_;
+  parsers[10] = null;
   parsers[11] = wks;
   parsers[12] = ptr;
   parsers[13] = hinfo;

From b953b9a763a7357f602cea85067804e23f7962eb Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 19:13:06 +0100
Subject: [PATCH 049/104] move get_txt into an action building the whole
 dns_rr_txt structure

---
 examples/dns.c | 15 ++-------------
 examples/dns.h |  6 ++++++
 examples/rr.c  | 21 ++++++++++++++++++++-
 3 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index ba1f592..637b2a1 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -53,16 +53,6 @@ uint8_t* get_cs(const HCountedArray *arr) {
   return ret;
 }
 
-uint8_t** get_txt(const HCountedArray *arr) {
-  uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
-  for (size_t i=0; i<arr->used; ++i) {
-    uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->elements[i]->seq->used);
-    for (size_t j=0; j<arr->elements[i]->seq->used; ++j)
-      tmp[j] = arr->elements[i]->seq->elements[j]->uint;
-  }
-  return ret;
-}
-
 void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
@@ -142,8 +132,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.mx.exchange = *H_FIELD(dns_domain_t, 1);
     break;
   case 16: // TXT
-    rr.txt.count = p->ast->seq->elements[0]->seq->used;
-    rr.txt.txt_data = get_txt(p->ast->seq->elements[0]->seq);
+    rr.txt = *(dns_rr_txt_t *)p->ast;
     break;
   default:
     break;
@@ -348,7 +337,7 @@ int start_listening() {
 
 const int TYPE_MAX = 16;
 typedef const char* cstr;
-const char* TYPE_STR[17] = {
+static const char* TYPE_STR[17] = {
   "nil", "A", "NS", "MD",
   "MF", "CNAME", "SOA", "MB",
   "MG", "MR", "NULL", "WKS",
diff --git a/examples/dns.h b/examples/dns.h
index 57602b0..f3e023c 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -7,6 +7,12 @@ enum DNSTokenType_ {
   TT_dns_qname_t,
   TT_dns_question_t,
   TT_dns_rr_t,
+  TT_dns_rr_txt_t,
+  TT_dns_rr_hinfo_t,
+  TT_dns_rr_minfo_t,
+  TT_dns_rr_mx_t,
+  TT_dns_rr_soa_t,
+  TT_dns_rr_wks_t,
   TT_dns_domain_t
 };
 
diff --git a/examples/rr.c b/examples/rr.c
index f252953..3619ca1 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -1,5 +1,6 @@
 #include "../src/hammer.h"
 #include "dns_common.h"
+#include "dns.h"
 #include "rr.h"
 
 #define false 0
@@ -11,6 +12,24 @@ bool validate_null(HParseResult *p) {
   return (65536 > p->ast->seq->used);
 }
 
+const HParsedToken *act_txt(const HParseResult *p) {
+  dns_rr_txt_t *txt = H_MAKE(dns_rr_txt_t);
+
+  const HCountedArray *arr = p->ast->seq->elements[0]->seq;
+  uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
+  for (size_t i=0; i<arr->used; ++i) {
+    uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->elements[i]->seq->used);
+    for (size_t j=0; j<arr->elements[i]->seq->used; ++j)
+      tmp[j] = arr->elements[i]->seq->elements[j]->uint;
+    ret[i] = tmp;
+  }
+
+  txt->count = p->ast->seq->elements[0]->seq->used;
+  txt->txt_data = ret;
+
+  return H_MAKE_TOKEN(dns_rr_txt_t, txt);
+}
+
 #define RDATA_TYPE_MAX 16
 const HParser* init_rdata(uint16_t type) {
   static const HParser *parsers[RDATA_TYPE_MAX+1];
@@ -51,7 +70,7 @@ const HParser* init_rdata(uint16_t type) {
   H_RULE (hinfo,  h_sequence(cstr, cstr, NULL));
   H_RULE (minfo,  h_sequence(domain, domain, NULL));
   H_RULE (mx,     h_sequence(h_uint16(), domain, NULL));
-  H_RULE (txt,    h_many1(cstr));
+  H_ARULE(txt,    h_many1(cstr));
 
 
   parsers[ 0] = NULL;            // there is no type 0

From 0304c75bba2a94e2ffb24fe29420c58e580decc7 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 19:15:33 +0100
Subject: [PATCH 050/104] correct improper uses of H_FIELD on domain-only
 RDATAs

---
 examples/dns.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 637b2a1..e3c8bd9 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -74,16 +74,16 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.a = p->ast->seq->elements[0]->uint;
     break;
   case 2: // NS
-    rr.ns = *H_FIELD(dns_domain_t, 0);
+    rr.ns = *(dns_domain_t *)p->ast->user;
     break;
   case 3: // MD
-    rr.md = *H_FIELD(dns_domain_t, 0);
+    rr.md = *(dns_domain_t *)p->ast->user;
     break;
   case 4: // MF
-    rr.md = *H_FIELD(dns_domain_t, 0);
+    rr.md = *(dns_domain_t *)p->ast->user;
     break;
   case 5: // CNAME
-    rr.cname = *H_FIELD(dns_domain_t, 0);
+    rr.cname = *(dns_domain_t *)p->ast->user;
     break;
   case 6: // SOA
     rr.soa.mname = *H_FIELD(dns_domain_t, 0);
@@ -95,13 +95,13 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.soa.minimum = p->ast->seq->elements[6]->uint;
     break;
   case 7: // MB
-    rr.mb = *H_FIELD(dns_domain_t, 0);
+    rr.mb = *(dns_domain_t *)p->ast->user;
     break;
   case 8: // MG
-    rr.mg = *H_FIELD(dns_domain_t, 0);
+    rr.mg = *(dns_domain_t *)p->ast->user;
     break;
   case 9: // MR
-    rr.mr = *H_FIELD(dns_domain_t, 0);
+    rr.mr = *(dns_domain_t *)p->ast->user;
     break;
   case 10: // NULL
     rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->used);
@@ -117,7 +117,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
       rr.wks.bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
     break;
   case 12: // PTR
-    rr.ptr = *H_FIELD(dns_domain_t, 0);
+    rr.ptr = *(dns_domain_t *)p->ast->user;
     break;
   case 13: // HINFO
     rr.hinfo.cpu = get_cs(p->ast->seq->elements[0]->seq);
@@ -132,7 +132,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.mx.exchange = *H_FIELD(dns_domain_t, 1);
     break;
   case 16: // TXT
-    rr.txt = *(dns_rr_txt_t *)p->ast;
+    rr.txt = *(dns_rr_txt_t *)p->ast->user;
     break;
   default:
     break;

From 8fbc137223fbd6d16dd3f922f948af67a283fd11 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 20:43:29 +0100
Subject: [PATCH 051/104] move get_cs into an action

---
 examples/dns.c | 11 ++---------
 examples/dns.h | 12 +++++++-----
 examples/rr.c  | 17 +++++++++++++++--
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index e3c8bd9..3f4ed9b 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -46,13 +46,6 @@ bool validate_message(HParseResult *p) {
 // Semantic Actions
 ///
 
-uint8_t* get_cs(const HCountedArray *arr) {
-  uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
-  for (size_t i=0; i<arr->used; ++i)
-    ret[i] = arr->elements[i]->uint;
-  return ret;
-}
-
 void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
@@ -120,8 +113,8 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.ptr = *(dns_domain_t *)p->ast->user;
     break;
   case 13: // HINFO
-    rr.hinfo.cpu = get_cs(p->ast->seq->elements[0]->seq);
-    rr.hinfo.os = get_cs(p->ast->seq->elements[1]->seq);
+    rr.hinfo.cpu = *H_FIELD(dns_cstr_t, 0);
+    rr.hinfo.os  = *H_FIELD(dns_cstr_t, 1);
     break;
   case 14: // MINFO
     rr.minfo.rmailbx = *H_FIELD(dns_domain_t, 0);
diff --git a/examples/dns.h b/examples/dns.h
index f3e023c..2cff916 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -13,9 +13,13 @@ enum DNSTokenType_ {
   TT_dns_rr_mx_t,
   TT_dns_rr_soa_t,
   TT_dns_rr_wks_t,
-  TT_dns_domain_t
+  TT_dns_domain_t,
+  TT_dns_cstr_t
 };
 
+typedef char *dns_domain_t;
+typedef uint8_t *dns_cstr_t;
+
 typedef struct dns_header {
   uint16_t id;
   bool qr, aa, tc, rd, ra;
@@ -43,8 +47,8 @@ typedef struct dns_question {
 } dns_question_t;
 
 typedef struct {
-  uint8_t* cpu;
-  uint8_t* os;
+  dns_cstr_t cpu;
+  dns_cstr_t os;
 } dns_rr_hinfo_t;
 
 typedef struct {
@@ -105,8 +109,6 @@ typedef struct dns_rr {
   };
 } dns_rr_t;
 
-typedef char *dns_domain_t;
-
 typedef struct dns_message {
   dns_header_t header;
   dns_question_t *questions;
diff --git a/examples/rr.c b/examples/rr.c
index 3619ca1..42b4648 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -15,7 +15,7 @@ bool validate_null(HParseResult *p) {
 const HParsedToken *act_txt(const HParseResult *p) {
   dns_rr_txt_t *txt = H_MAKE(dns_rr_txt_t);
 
-  const HCountedArray *arr = p->ast->seq->elements[0]->seq;
+  const HCountedArray *arr = p->ast->seq;
   uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
   for (size_t i=0; i<arr->used; ++i) {
     uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->elements[i]->seq->used);
@@ -30,6 +30,19 @@ const HParsedToken *act_txt(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_rr_txt_t, txt);
 }
 
+const HParsedToken* act_cstr(const HParseResult *p) {
+  dns_cstr_t *cs = H_MAKE(dns_cstr_t);
+
+  const HCountedArray *arr = p->ast->seq;
+  uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
+  for (size_t i=0; i<arr->used; ++i)
+    ret[i] = arr->elements[i]->uint;
+  assert(ret[arr->used-1] == '\0'); // XXX Is this right?! If so, shouldn't it be a validation?
+  *cs = ret;
+
+  return H_MAKE_TOKEN(dns_cstr_t, cs);
+}
+
 #define RDATA_TYPE_MAX 16
 const HParser* init_rdata(uint16_t type) {
   static const HParser *parsers[RDATA_TYPE_MAX+1];
@@ -43,7 +56,7 @@ const HParser* init_rdata(uint16_t type) {
 
 
   H_RULE (domain, init_domain());
-  H_RULE (cstr,   init_character_string());
+  H_ARULE(cstr,   init_character_string());
 
   H_RULE (a,      h_uint32());
   H_RULE (ns,     domain);

From b88d6ca34a3ee82d6c6fb57e3094b90fbebfae5f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 21:35:33 +0100
Subject: [PATCH 052/104] move SOA processing into an action

---
 examples/dns.c |  9 ++-------
 examples/rr.c  | 16 +++++++++++++++-
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 3f4ed9b..0456223 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -79,13 +79,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.cname = *(dns_domain_t *)p->ast->user;
     break;
   case 6: // SOA
-    rr.soa.mname = *H_FIELD(dns_domain_t, 0);
-    rr.soa.rname = *H_FIELD(dns_domain_t, 1);
-    rr.soa.serial = p->ast->seq->elements[2]->uint;
-    rr.soa.refresh = p->ast->seq->elements[3]->uint;
-    rr.soa.retry = p->ast->seq->elements[4]->uint;
-    rr.soa.expire = p->ast->seq->elements[5]->uint;
-    rr.soa.minimum = p->ast->seq->elements[6]->uint;
+    rr.soa = *(dns_rr_soa_t *)p->ast->user;
     break;
   case 7: // MB
     rr.mb = *(dns_domain_t *)p->ast->user;
@@ -100,6 +94,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->used);
     for (size_t i=0; i<p->ast->seq->used; ++i)
       rr.null[i] = p->ast->seq->elements[i]->uint;
+    // XXX Where is the length stored!?
     break;
   case 11: // WKS
     rr.wks.address = p->ast->seq->elements[0]->uint;
diff --git a/examples/rr.c b/examples/rr.c
index 42b4648..5752fe9 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -43,6 +43,20 @@ const HParsedToken* act_cstr(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_cstr_t, cs);
 }
 
+const HParsedToken* act_soa(const HParseResult *p) {
+  dns_rr_soa_t *soa = H_MAKE(dns_rr_soa_t);
+
+  soa->mname   = *H_FIELD(dns_domain_t, 0);
+  soa->rname   = *H_FIELD(dns_domain_t, 1);
+  soa->serial  = p->ast->seq->elements[2]->uint;
+  soa->refresh = p->ast->seq->elements[3]->uint;
+  soa->retry   = p->ast->seq->elements[4]->uint;
+  soa->expire  = p->ast->seq->elements[5]->uint;
+  soa->minimum = p->ast->seq->elements[6]->uint;
+
+  return H_MAKE_TOKEN(dns_rr_soa_t, soa);
+}
+
 #define RDATA_TYPE_MAX 16
 const HParser* init_rdata(uint16_t type) {
   static const HParser *parsers[RDATA_TYPE_MAX+1];
@@ -63,7 +77,7 @@ const HParser* init_rdata(uint16_t type) {
   H_RULE (md,     domain);
   H_RULE (mf,     domain);
   H_RULE (cname,  domain);
-  H_RULE (soa,    h_sequence(domain,     // MNAME
+  H_ARULE(soa,    h_sequence(domain,     // MNAME
 			     domain,     // RNAME
 			     h_uint32(), // SERIAL
 			     h_uint32(), // REFRESH

From 4d4094049320ed938408c7c46aef7c8092bcaa15 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 21:39:32 +0100
Subject: [PATCH 053/104] move WKS into action

---
 examples/dns.c |  7 +------
 examples/rr.c  | 13 +++++++++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 0456223..f591591 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -97,12 +97,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
     // XXX Where is the length stored!?
     break;
   case 11: // WKS
-    rr.wks.address = p->ast->seq->elements[0]->uint;
-    rr.wks.protocol = p->ast->seq->elements[1]->uint;
-    rr.wks.len = p->ast->seq->elements[2]->seq->used;
-    rr.wks.bit_map = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->elements[2]->seq->used);
-    for (size_t i=0; i<rr.wks.len; ++i)
-      rr.wks.bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
+    rr.wks = *(dns_rr_wks_t *)p->ast->user;
     break;
   case 12: // PTR
     rr.ptr = *(dns_domain_t *)p->ast->user;
diff --git a/examples/rr.c b/examples/rr.c
index 5752fe9..83241d8 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -57,6 +57,19 @@ const HParsedToken* act_soa(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_rr_soa_t, soa);
 }
 
+const HParsedToken* act_wks(const HParseResult *p) {
+  dns_rr_wks_t *wks = H_MAKE(dns_rr_wks_t);
+
+  wks->address  = p->ast->seq->elements[0]->uint;
+  wks->protocol = p->ast->seq->elements[1]->uint;
+  wks->len      = p->ast->seq->elements[2]->seq->used;
+  wks->bit_map  = h_arena_malloc(p->arena, sizeof(uint8_t)*wks->len);
+  for (size_t i=0; i<wks->len; ++i)
+    wks->bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
+
+  return H_MAKE_TOKEN(dns_rr_wks_t, wks);
+}
+
 #define RDATA_TYPE_MAX 16
 const HParser* init_rdata(uint16_t type) {
   static const HParser *parsers[RDATA_TYPE_MAX+1];

From e6a546b95b59e8cd6314d042727b97b400ff9f73 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 21:58:17 +0100
Subject: [PATCH 054/104] make actions for the remaining RDATA types

---
 examples/dns.c | 36 +++++++++++++++---------------------
 examples/dns.h |  5 ++++-
 examples/rr.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 22 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index f591591..8afd29c 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -64,58 +64,52 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   // Pack the parsed rdata into rr.
   switch(rr.type) {
   case 1: // A
-    rr.a = p->ast->seq->elements[0]->uint;
+    rr.a     = p->ast->seq->elements[0]->uint;
     break;
   case 2: // NS
-    rr.ns = *(dns_domain_t *)p->ast->user;
+    rr.ns    = *(dns_domain_t *)p->ast->user;
     break;
   case 3: // MD
-    rr.md = *(dns_domain_t *)p->ast->user;
+    rr.md    = *(dns_domain_t *)p->ast->user;
     break;
   case 4: // MF
-    rr.md = *(dns_domain_t *)p->ast->user;
+    rr.md    = *(dns_domain_t *)p->ast->user;
     break;
   case 5: // CNAME
     rr.cname = *(dns_domain_t *)p->ast->user;
     break;
   case 6: // SOA
-    rr.soa = *(dns_rr_soa_t *)p->ast->user;
+    rr.soa   = *(dns_rr_soa_t *)p->ast->user;
     break;
   case 7: // MB
-    rr.mb = *(dns_domain_t *)p->ast->user;
+    rr.mb    = *(dns_domain_t *)p->ast->user;
     break;
   case 8: // MG
-    rr.mg = *(dns_domain_t *)p->ast->user;
+    rr.mg    = *(dns_domain_t *)p->ast->user;
     break;
   case 9: // MR
-    rr.mr = *(dns_domain_t *)p->ast->user;
+    rr.mr    = *(dns_domain_t *)p->ast->user;
     break;
   case 10: // NULL
-    rr.null = h_arena_malloc(rdata->arena, sizeof(uint8_t)*p->ast->seq->used);
-    for (size_t i=0; i<p->ast->seq->used; ++i)
-      rr.null[i] = p->ast->seq->elements[i]->uint;
-    // XXX Where is the length stored!?
+    rr.null  = *(dns_rr_null_t *)p->ast->user;
     break;
   case 11: // WKS
-    rr.wks = *(dns_rr_wks_t *)p->ast->user;
+    rr.wks   = *(dns_rr_wks_t *)p->ast->user;
     break;
   case 12: // PTR
-    rr.ptr = *(dns_domain_t *)p->ast->user;
+    rr.ptr   = *(dns_domain_t *)p->ast->user;
     break;
   case 13: // HINFO
-    rr.hinfo.cpu = *H_FIELD(dns_cstr_t, 0);
-    rr.hinfo.os  = *H_FIELD(dns_cstr_t, 1);
+    rr.hinfo = *(dns_rr_hinfo_t *)p->ast->user;
     break;
   case 14: // MINFO
-    rr.minfo.rmailbx = *H_FIELD(dns_domain_t, 0);
-    rr.minfo.emailbx = *H_FIELD(dns_domain_t, 1);
+    rr.minfo = *(dns_rr_minfo_t *)p->ast->user;
     break;
   case 15: // MX
-    rr.mx.preference = p->ast->seq->elements[0]->uint;
-    rr.mx.exchange = *H_FIELD(dns_domain_t, 1);
+    rr.mx    = *(dns_rr_mx_t *)p->ast->user;
     break;
   case 16: // TXT
-    rr.txt = *(dns_rr_txt_t *)p->ast->user;
+    rr.txt   = *(dns_rr_txt_t *)p->ast->user;
     break;
   default:
     break;
diff --git a/examples/dns.h b/examples/dns.h
index 2cff916..ed2c26f 100644
--- a/examples/dns.h
+++ b/examples/dns.h
@@ -13,6 +13,7 @@ enum DNSTokenType_ {
   TT_dns_rr_mx_t,
   TT_dns_rr_soa_t,
   TT_dns_rr_wks_t,
+  TT_dns_rr_null_t,
   TT_dns_domain_t,
   TT_dns_cstr_t
 };
@@ -83,6 +84,8 @@ typedef struct {
   uint8_t* bit_map;
 } dns_rr_wks_t;
 
+typedef uint8_t *dns_rr_null_t;
+
 typedef struct dns_rr {
   char* name;
   uint16_t type;
@@ -99,7 +102,7 @@ typedef struct dns_rr {
     char*          mb;
     char*          mg;
     char*          mr;
-    uint8_t*       null;
+    dns_rr_null_t  null;
     dns_rr_wks_t   wks;
     char*          ptr;
     dns_rr_hinfo_t hinfo;
diff --git a/examples/rr.c b/examples/rr.c
index 83241d8..6bfb782 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -6,12 +6,28 @@
 #define false 0
 #define true 1
 
+
+///
+// Validations and Semantic Actions
+///
+
 bool validate_null(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
   return (65536 > p->ast->seq->used);
 }
 
+const HParsedToken *act_null(const HParseResult *p) {
+  dns_rr_null_t *null = H_MAKE(dns_rr_null_t);
+
+  size_t len = p->ast->seq->used;
+  uint8_t *buf = h_arena_malloc(p->arena, sizeof(uint8_t)*len);
+  for (size_t i=0; i<len; ++i)
+    buf[i] = p->ast->seq->elements[i]->uint;
+
+  return H_MAKE_TOKEN(dns_rr_null_t, null);
+}
+
 const HParsedToken *act_txt(const HParseResult *p) {
   dns_rr_txt_t *txt = H_MAKE(dns_rr_txt_t);
 
@@ -70,6 +86,38 @@ const HParsedToken* act_wks(const HParseResult *p) {
   return H_MAKE_TOKEN(dns_rr_wks_t, wks);
 }
 
+const HParsedToken* act_hinfo(const HParseResult *p) {
+  dns_rr_hinfo_t *hinfo = H_MAKE(dns_rr_hinfo_t);
+
+  hinfo->cpu = *H_FIELD(dns_cstr_t, 0);
+  hinfo->os  = *H_FIELD(dns_cstr_t, 1);
+
+  return H_MAKE_TOKEN(dns_rr_hinfo_t, hinfo);
+}
+
+const HParsedToken* act_minfo(const HParseResult *p) {
+  dns_rr_minfo_t *minfo = H_MAKE(dns_rr_minfo_t);
+
+  minfo->rmailbx = *H_FIELD(dns_domain_t, 0);
+  minfo->emailbx = *H_FIELD(dns_domain_t, 1);
+
+  return H_MAKE_TOKEN(dns_rr_minfo_t, minfo);
+}
+
+const HParsedToken* act_mx(const HParseResult *p) {
+  dns_rr_mx_t *mx = H_MAKE(dns_rr_mx_t);
+
+  mx->preference = p->ast->seq->elements[0]->uint;
+  mx->exchange   = *H_FIELD(dns_domain_t, 1);
+
+  return H_MAKE_TOKEN(dns_rr_mx_t, mx);
+}
+
+
+///
+// Parsers for all types of RDATA
+///
+
 #define RDATA_TYPE_MAX 16
 const HParser* init_rdata(uint16_t type) {
   static const HParser *parsers[RDATA_TYPE_MAX+1];

From c9d3101f4128bfbfa533f043e4f66d213804a59a Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 21:59:55 +0100
Subject: [PATCH 055/104] a comment

---
 examples/dns.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/dns.c b/examples/dns.c
index 8afd29c..d8b9bed 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -46,6 +46,7 @@ bool validate_message(HParseResult *p) {
 // Semantic Actions
 ///
 
+// Helper: Parse and pack the RDATA field of a Resource Record.
 void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)

From 8dea41c237a635e938240c3712892a066e6c782f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Jan 2013 22:20:58 +0100
Subject: [PATCH 056/104] condense the packing table in set_rdata a little

---
 examples/dns.c | 67 +++++++++++++-------------------------------------
 1 file changed, 17 insertions(+), 50 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index d8b9bed..012b7fb 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -64,56 +64,23 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
 
   // Pack the parsed rdata into rr.
   switch(rr.type) {
-  case 1: // A
-    rr.a     = p->ast->seq->elements[0]->uint;
-    break;
-  case 2: // NS
-    rr.ns    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 3: // MD
-    rr.md    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 4: // MF
-    rr.md    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 5: // CNAME
-    rr.cname = *(dns_domain_t *)p->ast->user;
-    break;
-  case 6: // SOA
-    rr.soa   = *(dns_rr_soa_t *)p->ast->user;
-    break;
-  case 7: // MB
-    rr.mb    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 8: // MG
-    rr.mg    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 9: // MR
-    rr.mr    = *(dns_domain_t *)p->ast->user;
-    break;
-  case 10: // NULL
-    rr.null  = *(dns_rr_null_t *)p->ast->user;
-    break;
-  case 11: // WKS
-    rr.wks   = *(dns_rr_wks_t *)p->ast->user;
-    break;
-  case 12: // PTR
-    rr.ptr   = *(dns_domain_t *)p->ast->user;
-    break;
-  case 13: // HINFO
-    rr.hinfo = *(dns_rr_hinfo_t *)p->ast->user;
-    break;
-  case 14: // MINFO
-    rr.minfo = *(dns_rr_minfo_t *)p->ast->user;
-    break;
-  case 15: // MX
-    rr.mx    = *(dns_rr_mx_t *)p->ast->user;
-    break;
-  case 16: // TXT
-    rr.txt   = *(dns_rr_txt_t *)p->ast->user;
-    break;
-  default:
-    break;
+  case 1:  rr.a     = p->ast->seq->elements[0]->uint;  break;
+  case 2:  rr.ns    = *(dns_domain_t   *)p->ast->user; break;
+  case 3:  rr.md    = *(dns_domain_t   *)p->ast->user; break;
+  case 4:  rr.md    = *(dns_domain_t   *)p->ast->user; break;
+  case 5:  rr.cname = *(dns_domain_t   *)p->ast->user; break;
+  case 6:  rr.soa   = *(dns_rr_soa_t   *)p->ast->user; break;
+  case 7:  rr.mb    = *(dns_domain_t   *)p->ast->user; break;
+  case 8:  rr.mg    = *(dns_domain_t   *)p->ast->user; break;
+  case 9:  rr.mr    = *(dns_domain_t   *)p->ast->user; break;
+  case 10: rr.null  = *(dns_rr_null_t  *)p->ast->user; break;
+  case 11: rr.wks   = *(dns_rr_wks_t   *)p->ast->user; break;
+  case 12: rr.ptr   = *(dns_domain_t   *)p->ast->user; break;
+  case 13: rr.hinfo = *(dns_rr_hinfo_t *)p->ast->user; break;
+  case 14: rr.minfo = *(dns_rr_minfo_t *)p->ast->user; break;
+  case 15: rr.mx    = *(dns_rr_mx_t    *)p->ast->user; break;
+  case 16: rr.txt   = *(dns_rr_txt_t   *)p->ast->user; break;
+  default:                                             break;
   }
 }
 

From d0d115f20617cc303ee027255df7d832bdadf3b3 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 16:55:26 +0100
Subject: [PATCH 057/104] wrap only non-null rdata parsers with h_end_p

---
 examples/rr.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/rr.c b/examples/rr.c
index 6bfb782..8aa48e6 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -181,8 +181,10 @@ const HParser* init_rdata(uint16_t type) {
 
   // All parsers must consume their input exactly.
   for(uint16_t i; i<sizeof(parsers); i++) {
-    parsers[i] = h_action(h_sequence(parsers[i], h_end_p(), NULL),
-			  act_index0);
+    if(parsers[i]) {
+      parsers[i] = h_action(h_sequence(parsers[i], h_end_p(), NULL),
+			    act_index0);
+    }
   }
 
   inited = 1;

From 1b0c61fc1555880919ba0e82d09079a8a2471fde Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 16:57:59 +0100
Subject: [PATCH 058/104] cosmetics

---
 examples/dns.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 012b7fb..622c62a 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -42,6 +42,7 @@ bool validate_message(HParseResult *p) {
   return true;
 }
 
+
 ///
 // Semantic Actions
 ///
@@ -200,9 +201,8 @@ const HParsedToken* act_message(const HParseResult *p) {
 #define act_qname  act_index0
 
 
-
 ///
-// Parser / Grammar
+// Grammar
 ///
 
 const HParser* init_parser() {
@@ -259,7 +259,7 @@ const HParser* init_parser() {
 
 
 ///
-// Program Logic for a Dummy DNS Server
+// Main Program for a Dummy DNS Server
 ///
 
 int start_listening() {

From dc037e6c2854ce9c529a192d9dc393c6b7773f43 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 18:16:14 +0100
Subject: [PATCH 059/104] organize definitions in glue.h a little

---
 examples/glue.h | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/examples/glue.h b/examples/glue.h
index 2b60646..7fb9f8a 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -1,12 +1,17 @@
+//
+// API additions for writing grammar and semantic actions more concisely
+//
+
 #ifndef HAMMER_EXAMPLES_GLUE__H
 #define HAMMER_EXAMPLES_GLUE__H
 
 #include <assert.h>
 #include "../src/hammer.h"
 
-///
-// API Additions
-///
+
+//
+// Grammar specification
+//
 
 #define H_RULE(rule, def) const HParser *rule = def
 #define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
@@ -17,6 +22,11 @@
 #define H_AVRULE(rule, def) const HParser *rule = \
   h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
 
+
+//
+// Pre-fab semantic actions
+//
+
 const HParsedToken *h_act_ignore(const HParseResult *p);
 const HParsedToken *h_act_index(int i, const HParseResult *p);
 const HParsedToken *h_act_flatten(const HParseResult *p);
@@ -28,10 +38,12 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
     return paction(__VA_ARGS__, p); \
   }
 
-const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p);
 
-void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
-void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
+//
+// Working with HParsedTokens
+//
+
+// Token constructors...
 
 HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_token_seq(HArena *arena);
@@ -42,6 +54,14 @@ HParsedToken *h_make_token_seq(HArena *arena);
 #define H_MAKE_TOKEN(TYP, VAL) \
   h_make_token(p->arena, TT_ ## TYP, VAL)
 
+// Sequences...
+
+// Flatten nested sequences into one.
+const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p);
+
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
+
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i);
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
@@ -52,4 +72,5 @@ void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
 #define H_FIELD(TYP, IDX) \
   H_SEQ_INDEX(TYP, p->ast, IDX)
 
+
 #endif

From ef17e42ec86c476fee42fab006ab6ff0128e0556 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 18:17:17 +0100
Subject: [PATCH 060/104] rename H_MAKE to H_ALLOC

---
 examples/dns.c        | 10 +++++-----
 examples/dns_common.c |  2 +-
 examples/glue.h       |  2 +-
 examples/rr.c         | 16 ++++++++--------
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 622c62a..e3f36a6 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -102,14 +102,14 @@ const HParsedToken* act_header(const HParseResult *p) {
     .additional_count = fields[11]->uint
   };
 
-  dns_header_t *header = H_MAKE(dns_header_t);
+  dns_header_t *header = H_ALLOC(dns_header_t);
   *header = header_;
 
   return H_MAKE_TOKEN(dns_header_t, header);
 }
 
 const HParsedToken* act_label(const HParseResult *p) {
-  dns_label_t *r = H_MAKE(dns_label_t);
+  dns_label_t *r = H_ALLOC(dns_label_t);
 
   r->len = p->ast->seq->used;
   r->label = h_arena_malloc(p->arena, r->len + 1);
@@ -121,7 +121,7 @@ const HParsedToken* act_label(const HParseResult *p) {
 }
 
 const HParsedToken* act_rr(const HParseResult *p) {
-  dns_rr_t *rr = H_MAKE(dns_rr_t);
+  dns_rr_t *rr = H_ALLOC(dns_rr_t);
 
   rr->name     = *H_FIELD(dns_domain_t, 0);
   rr->type     = p->ast->seq->elements[1]->uint;
@@ -136,7 +136,7 @@ const HParsedToken* act_rr(const HParseResult *p) {
 }
 
 const HParsedToken* act_question(const HParseResult *p) {
-  dns_question_t *q = H_MAKE(dns_question_t);
+  dns_question_t *q = H_ALLOC(dns_question_t);
   HParsedToken **fields = p->ast->seq->elements;
 
   // QNAME is a sequence of labels. Pack them into an array.
@@ -154,7 +154,7 @@ const HParsedToken* act_question(const HParseResult *p) {
 
 const HParsedToken* act_message(const HParseResult *p) {
   h_pprint(stdout, p->ast, 0, 2);
-  dns_message_t *msg = H_MAKE(dns_message_t);
+  dns_message_t *msg = H_ALLOC(dns_message_t);
 
   // Copy header into message struct.
   dns_header_t *header = H_FIELD(dns_header_t, 0);
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 3ea5594..0171a59 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -48,7 +48,7 @@ const HParsedToken* act_domain(const HParseResult *p) {
   }
 
   if(arr) {
-    dns_domain_t *val = H_MAKE(dns_domain_t);  // dns_domain_t is char*
+    dns_domain_t *val = H_ALLOC(dns_domain_t);  // dns_domain_t is char*
     *val = arr;
     ret = H_MAKE_TOKEN(dns_domain_t, val);
   }
diff --git a/examples/glue.h b/examples/glue.h
index 7fb9f8a..67f6907 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -48,7 +48,7 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_token_seq(HArena *arena);
 
-#define H_MAKE(TYP) \
+#define H_ALLOC(TYP) \
   ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
 
 #define H_MAKE_TOKEN(TYP, VAL) \
diff --git a/examples/rr.c b/examples/rr.c
index 8aa48e6..bd4c125 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -18,7 +18,7 @@ bool validate_null(HParseResult *p) {
 }
 
 const HParsedToken *act_null(const HParseResult *p) {
-  dns_rr_null_t *null = H_MAKE(dns_rr_null_t);
+  dns_rr_null_t *null = H_ALLOC(dns_rr_null_t);
 
   size_t len = p->ast->seq->used;
   uint8_t *buf = h_arena_malloc(p->arena, sizeof(uint8_t)*len);
@@ -29,7 +29,7 @@ const HParsedToken *act_null(const HParseResult *p) {
 }
 
 const HParsedToken *act_txt(const HParseResult *p) {
-  dns_rr_txt_t *txt = H_MAKE(dns_rr_txt_t);
+  dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t);
 
   const HCountedArray *arr = p->ast->seq;
   uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
@@ -47,7 +47,7 @@ const HParsedToken *act_txt(const HParseResult *p) {
 }
 
 const HParsedToken* act_cstr(const HParseResult *p) {
-  dns_cstr_t *cs = H_MAKE(dns_cstr_t);
+  dns_cstr_t *cs = H_ALLOC(dns_cstr_t);
 
   const HCountedArray *arr = p->ast->seq;
   uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
@@ -60,7 +60,7 @@ const HParsedToken* act_cstr(const HParseResult *p) {
 }
 
 const HParsedToken* act_soa(const HParseResult *p) {
-  dns_rr_soa_t *soa = H_MAKE(dns_rr_soa_t);
+  dns_rr_soa_t *soa = H_ALLOC(dns_rr_soa_t);
 
   soa->mname   = *H_FIELD(dns_domain_t, 0);
   soa->rname   = *H_FIELD(dns_domain_t, 1);
@@ -74,7 +74,7 @@ const HParsedToken* act_soa(const HParseResult *p) {
 }
 
 const HParsedToken* act_wks(const HParseResult *p) {
-  dns_rr_wks_t *wks = H_MAKE(dns_rr_wks_t);
+  dns_rr_wks_t *wks = H_ALLOC(dns_rr_wks_t);
 
   wks->address  = p->ast->seq->elements[0]->uint;
   wks->protocol = p->ast->seq->elements[1]->uint;
@@ -87,7 +87,7 @@ const HParsedToken* act_wks(const HParseResult *p) {
 }
 
 const HParsedToken* act_hinfo(const HParseResult *p) {
-  dns_rr_hinfo_t *hinfo = H_MAKE(dns_rr_hinfo_t);
+  dns_rr_hinfo_t *hinfo = H_ALLOC(dns_rr_hinfo_t);
 
   hinfo->cpu = *H_FIELD(dns_cstr_t, 0);
   hinfo->os  = *H_FIELD(dns_cstr_t, 1);
@@ -96,7 +96,7 @@ const HParsedToken* act_hinfo(const HParseResult *p) {
 }
 
 const HParsedToken* act_minfo(const HParseResult *p) {
-  dns_rr_minfo_t *minfo = H_MAKE(dns_rr_minfo_t);
+  dns_rr_minfo_t *minfo = H_ALLOC(dns_rr_minfo_t);
 
   minfo->rmailbx = *H_FIELD(dns_domain_t, 0);
   minfo->emailbx = *H_FIELD(dns_domain_t, 1);
@@ -105,7 +105,7 @@ const HParsedToken* act_minfo(const HParseResult *p) {
 }
 
 const HParsedToken* act_mx(const HParseResult *p) {
-  dns_rr_mx_t *mx = H_MAKE(dns_rr_mx_t);
+  dns_rr_mx_t *mx = H_ALLOC(dns_rr_mx_t);
 
   mx->preference = p->ast->seq->elements[0]->uint;
   mx->exchange   = *H_FIELD(dns_domain_t, 1);

From 4c8b0086d7771616f02e39cc1c512ea6b165a5e5 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 18:28:50 +0100
Subject: [PATCH 061/104] rename H_MAKE_TOKEN to H_MAKE and remove 'token' from
 other function names

---
 examples/dns.c        | 10 +++++-----
 examples/dns_common.c |  2 +-
 examples/glue.c       | 20 ++++++++++----------
 examples/glue.h       | 10 +++++-----
 examples/rr.c         | 16 ++++++++--------
 5 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index e3f36a6..c8e3faf 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -105,7 +105,7 @@ const HParsedToken* act_header(const HParseResult *p) {
   dns_header_t *header = H_ALLOC(dns_header_t);
   *header = header_;
 
-  return H_MAKE_TOKEN(dns_header_t, header);
+  return H_MAKE(dns_header_t, header);
 }
 
 const HParsedToken* act_label(const HParseResult *p) {
@@ -117,7 +117,7 @@ const HParsedToken* act_label(const HParseResult *p) {
     r->label[i] = p->ast->seq->elements[i]->uint;
   r->label[r->len] = 0;
 
-  return H_MAKE_TOKEN(dns_label_t, r);
+  return H_MAKE(dns_label_t, r);
 }
 
 const HParsedToken* act_rr(const HParseResult *p) {
@@ -132,7 +132,7 @@ const HParsedToken* act_rr(const HParseResult *p) {
   // Parse and pack RDATA.
   set_rdata(*rr, p->ast->seq->elements[4]->seq);	   
 
-  return H_MAKE_TOKEN(dns_rr_t, rr);
+  return H_MAKE(dns_rr_t, rr);
 }
 
 const HParsedToken* act_question(const HParseResult *p) {
@@ -149,7 +149,7 @@ const HParsedToken* act_question(const HParseResult *p) {
   q->qtype  = fields[1]->uint;
   q->qclass = fields[2]->uint;
 
-  return H_MAKE_TOKEN(dns_question_t, q);
+  return H_MAKE(dns_question_t, q);
 }
 
 const HParsedToken* act_message(const HParseResult *p) {
@@ -194,7 +194,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   }
   msg->additional = additional;
 
-  return H_MAKE_TOKEN(dns_message_t, msg);
+  return H_MAKE(dns_message_t, msg);
 }
 
 #define act_hdzero h_act_ignore
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 0171a59..76915b6 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -50,7 +50,7 @@ const HParsedToken* act_domain(const HParseResult *p) {
   if(arr) {
     dns_domain_t *val = H_ALLOC(dns_domain_t);  // dns_domain_t is char*
     *val = arr;
-    ret = H_MAKE_TOKEN(dns_domain_t, val);
+    ret = H_MAKE(dns_domain_t, val);
   }
 
   return ret;
diff --git a/examples/glue.c b/examples/glue.c
index 80864fc..1c86b15 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -48,16 +48,16 @@ void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
 
 // Flatten nested sequences. Always returns a sequence.
 // If input element is not a sequence, returns it as a singleton sequence.
-const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p)
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
 {
   assert(p != NULL);
 
-  HParsedToken *ret = h_make_token_seq(arena);
+  HParsedToken *ret = h_make_seq(arena);
   switch(p->token_type) {
   case TT_SEQUENCE:
     // Flatten and append all.
     for(size_t i; i<p->seq->used; i++) {
-	  h_seq_append(ret, h_token_flatten(arena, h_seq_index(p, i)));
+	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
 	}
 	break;
   default:
@@ -69,26 +69,26 @@ const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p)
   return ret;
 }
 
-// Action version of h_token_flatten.
+// Action version of h_seq_flatten.
 const HParsedToken *h_act_flatten(const HParseResult *p) {
-  return h_token_flatten(p->arena, p->ast);
+  return h_seq_flatten(p->arena, p->ast);
 }
 
-HParsedToken *h_make_token_(HArena *arena, HTokenType type)
+HParsedToken *h_make_(HArena *arena, HTokenType type)
 {
   HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
   ret->token_type = type;
   return ret;
 }
 
-HParsedToken *h_make_token_seq(HArena *arena)
+HParsedToken *h_make_seq(HArena *arena)
 {
-  return h_make_token_(arena, TT_SEQUENCE);
+  return h_make_(arena, TT_SEQUENCE);
 }
 
-HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value)
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
 {
-  HParsedToken *ret = h_make_token_(arena, type);
+  HParsedToken *ret = h_make_(arena, type);
   ret->user = value;
   return ret;
 }
diff --git a/examples/glue.h b/examples/glue.h
index 67f6907..dac8fbb 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -45,19 +45,19 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 
 // Token constructors...
 
-HParsedToken *h_make_token(HArena *arena, HTokenType type, void *value);
-HParsedToken *h_make_token_seq(HArena *arena);
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
+HParsedToken *h_make_seq(HArena *arena);
 
 #define H_ALLOC(TYP) \
   ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
 
-#define H_MAKE_TOKEN(TYP, VAL) \
-  h_make_token(p->arena, TT_ ## TYP, VAL)
+#define H_MAKE(TYP, VAL) \
+  h_make(p->arena, TT_ ## TYP, VAL)
 
 // Sequences...
 
 // Flatten nested sequences into one.
-const HParsedToken *h_token_flatten(HArena *arena, const HParsedToken *p);
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
 
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
 void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
diff --git a/examples/rr.c b/examples/rr.c
index bd4c125..94c4b7c 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -25,7 +25,7 @@ const HParsedToken *act_null(const HParseResult *p) {
   for (size_t i=0; i<len; ++i)
     buf[i] = p->ast->seq->elements[i]->uint;
 
-  return H_MAKE_TOKEN(dns_rr_null_t, null);
+  return H_MAKE(dns_rr_null_t, null);
 }
 
 const HParsedToken *act_txt(const HParseResult *p) {
@@ -43,7 +43,7 @@ const HParsedToken *act_txt(const HParseResult *p) {
   txt->count = p->ast->seq->elements[0]->seq->used;
   txt->txt_data = ret;
 
-  return H_MAKE_TOKEN(dns_rr_txt_t, txt);
+  return H_MAKE(dns_rr_txt_t, txt);
 }
 
 const HParsedToken* act_cstr(const HParseResult *p) {
@@ -56,7 +56,7 @@ const HParsedToken* act_cstr(const HParseResult *p) {
   assert(ret[arr->used-1] == '\0'); // XXX Is this right?! If so, shouldn't it be a validation?
   *cs = ret;
 
-  return H_MAKE_TOKEN(dns_cstr_t, cs);
+  return H_MAKE(dns_cstr_t, cs);
 }
 
 const HParsedToken* act_soa(const HParseResult *p) {
@@ -70,7 +70,7 @@ const HParsedToken* act_soa(const HParseResult *p) {
   soa->expire  = p->ast->seq->elements[5]->uint;
   soa->minimum = p->ast->seq->elements[6]->uint;
 
-  return H_MAKE_TOKEN(dns_rr_soa_t, soa);
+  return H_MAKE(dns_rr_soa_t, soa);
 }
 
 const HParsedToken* act_wks(const HParseResult *p) {
@@ -83,7 +83,7 @@ const HParsedToken* act_wks(const HParseResult *p) {
   for (size_t i=0; i<wks->len; ++i)
     wks->bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
 
-  return H_MAKE_TOKEN(dns_rr_wks_t, wks);
+  return H_MAKE(dns_rr_wks_t, wks);
 }
 
 const HParsedToken* act_hinfo(const HParseResult *p) {
@@ -92,7 +92,7 @@ const HParsedToken* act_hinfo(const HParseResult *p) {
   hinfo->cpu = *H_FIELD(dns_cstr_t, 0);
   hinfo->os  = *H_FIELD(dns_cstr_t, 1);
 
-  return H_MAKE_TOKEN(dns_rr_hinfo_t, hinfo);
+  return H_MAKE(dns_rr_hinfo_t, hinfo);
 }
 
 const HParsedToken* act_minfo(const HParseResult *p) {
@@ -101,7 +101,7 @@ const HParsedToken* act_minfo(const HParseResult *p) {
   minfo->rmailbx = *H_FIELD(dns_domain_t, 0);
   minfo->emailbx = *H_FIELD(dns_domain_t, 1);
 
-  return H_MAKE_TOKEN(dns_rr_minfo_t, minfo);
+  return H_MAKE(dns_rr_minfo_t, minfo);
 }
 
 const HParsedToken* act_mx(const HParseResult *p) {
@@ -110,7 +110,7 @@ const HParsedToken* act_mx(const HParseResult *p) {
   mx->preference = p->ast->seq->elements[0]->uint;
   mx->exchange   = *H_FIELD(dns_domain_t, 1);
 
-  return H_MAKE_TOKEN(dns_rr_mx_t, mx);
+  return H_MAKE(dns_rr_mx_t, mx);
 }
 
 

From 2d53a0f873d34c0edae0216312a46ef997effdaa Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 18:30:55 +0100
Subject: [PATCH 062/104] mark h_carray_index as meant to be internal

---
 examples/glue.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/glue.h b/examples/glue.h
index dac8fbb..c8b17f3 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -62,7 +62,7 @@ const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
 void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
 
-HParsedToken *h_carray_index(const HCountedArray *a, size_t i);
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
 

From 2c90bd3ef5d68d9c9073a8841fe5452ec3fffbb4 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 19:06:25 +0100
Subject: [PATCH 063/104] rename: h_seq_index_user -> h_seq_index ->
 h_seq_index_token

---
 examples/glue.c |  8 ++++----
 examples/glue.h | 38 ++++++++++++++++++++++----------------
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 1c86b15..778f356 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -57,7 +57,7 @@ const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
   case TT_SEQUENCE:
     // Flatten and append all.
     for(size_t i; i<p->seq->used; i++) {
-	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
+	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index_token(p, i)));
 	}
 	break;
   default:
@@ -99,16 +99,16 @@ HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
   return a->elements[i];
 }
 
-HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
+HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i)
 {
   assert(p != NULL);
   assert(p->token_type == TT_SEQUENCE);
   return h_carray_index(p->seq, i);
 }
 
-void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
+void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i)
 {
-  HParsedToken *elem = h_seq_index(p, i);
+  HParsedToken *elem = h_seq_index_token(p, i);
   assert(elem->token_type == (HTokenType)type);
   return elem->user;
 }
diff --git a/examples/glue.h b/examples/glue.h
index c8b17f3..5716197 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -43,34 +43,40 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 // Working with HParsedTokens
 //
 
+// Standard short-hand for arena-allocating a variable in a semantic action.
+#define H_ALLOC(TYP) \
+  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
+
 // Token constructors...
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_seq(HArena *arena);
 
-#define H_ALLOC(TYP) \
-  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
-
-#define H_MAKE(TYP, VAL) \
-  h_make(p->arena, TT_ ## TYP, VAL)
+// Standard short-hand to make a user-type token.
+#define H_MAKE(TYP, VAL) h_make(p->arena, TT_ ## TYP, VAL)
 
 // Sequences...
 
+// Access a sequence element by index.
+HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i);
+
+// Access a user-type element of a sequence by index.
+#define H_SEQ_INDEX(TYP, SEQ, IDX) \
+  ((TYP *) h_seq_index(TT_ ## TYP, SEQ, IDX))
+
+// Standard short-hand to access a user-type field on a sequence token.
+#define H_FIELD(TYP, IDX)  H_SEQ_INDEX(TYP, p->ast, IDX)
+
+// Append elements to a sequence.
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
+
 // Flatten nested sequences into one.
 const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
 
-void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);
-void h_seq_append(HParsedToken *xs, const HParsedToken *ys);
-
+// Helpers for implementing H_SEQ_INDEX.
+void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i); // XXX helper
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
-HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
-void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i);
-
-#define H_SEQ_INDEX(TYP, SEQ, IDX) \
-  ((TYP *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
-
-#define H_FIELD(TYP, IDX) \
-  H_SEQ_INDEX(TYP, p->ast, IDX)
 
 
 #endif

From 3df4030bf821ef33539594c4e3685aa0ce477472 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 19:10:36 +0100
Subject: [PATCH 064/104] cosmetics

---
 examples/glue.h | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/examples/glue.h b/examples/glue.h
index 5716197..d428e06 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -50,12 +50,12 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 // Token constructors...
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
-HParsedToken *h_make_seq(HArena *arena);
+HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
 
 // Standard short-hand to make a user-type token.
 #define H_MAKE(TYP, VAL) h_make(p->arena, TT_ ## TYP, VAL)
 
-// Sequences...
+// Sequence access...
 
 // Access a sequence element by index.
 HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i);
@@ -67,6 +67,12 @@ HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i);
 // Standard short-hand to access a user-type field on a sequence token.
 #define H_FIELD(TYP, IDX)  H_SEQ_INDEX(TYP, p->ast, IDX)
 
+// Lower-level helper for H_SEQ_INDEX.
+void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i);
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
+
+// Sequence modification...
+
 // Append elements to a sequence.
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
 void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
@@ -74,9 +80,5 @@ void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
 // Flatten nested sequences into one.
 const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
 
-// Helpers for implementing H_SEQ_INDEX.
-void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i); // XXX helper
-HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
-
 
 #endif

From 54046e25dae73783701602e6d6d101adb1d050c4 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 19:24:13 +0100
Subject: [PATCH 065/104] forgot to allocate the carray in h_make_seq (oops)

---
 examples/glue.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/glue.c b/examples/glue.c
index 778f356..29b8de7 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -83,7 +83,9 @@ HParsedToken *h_make_(HArena *arena, HTokenType type)
 
 HParsedToken *h_make_seq(HArena *arena)
 {
-  return h_make_(arena, TT_SEQUENCE);
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new(arena);
+  return ret;
 }
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value)

From 71cdd46cebac7862fb0e788ba7555df1a7401e89 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 23 Jan 2013 19:29:29 +0100
Subject: [PATCH 066/104] expand h_make family to include seq, bytes, sint, and
 uint tokens

---
 examples/glue.c | 31 ++++++++++++++++++++++++++++---
 examples/glue.h | 11 +++++++++--
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 29b8de7..4cc85d0 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -74,6 +74,7 @@ const HParsedToken *h_act_flatten(const HParseResult *p) {
   return h_seq_flatten(p->arena, p->ast);
 }
 
+// Low-level helper for the h_make family.
 HParsedToken *h_make_(HArena *arena, HTokenType type)
 {
   HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
@@ -81,6 +82,14 @@ HParsedToken *h_make_(HArena *arena, HTokenType type)
   return ret;
 }
 
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
+{
+  assert(type >= TT_USER);
+  HParsedToken *ret = h_make_(arena, type);
+  ret->user = value;
+  return ret;
+}
+
 HParsedToken *h_make_seq(HArena *arena)
 {
   HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
@@ -88,13 +97,29 @@ HParsedToken *h_make_seq(HArena *arena)
   return ret;
 }
 
-HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
+HParsedToken *h_make_bytes(HArena *arena, size_t len)
 {
-  HParsedToken *ret = h_make_(arena, type);
-  ret->user = value;
+  HParsedToken *ret = h_make_(arena, TT_BYTES);
+  ret->bytes.len = len;
+  ret->bytes.token = h_arena_malloc(arena, len);
   return ret;
 }
 
+HParsedToken *h_make_sint(HArena *arena, int64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_SINT);
+  ret->sint = val;
+  return ret;
+}
+
+HParsedToken *h_make_uint(HArena *arena, uint64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_UINT);
+  ret->uint = val;
+  return ret;
+}
+
+// XXX -> internal
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 {
   assert(i < a->used);
diff --git a/examples/glue.h b/examples/glue.h
index d428e06..9b11315 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -51,9 +51,16 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
+HParsedToken *h_make_bytes(HArena *arena, size_t len);
+HParsedToken *h_make_sint(HArena *arena, int64_t val);
+HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 
-// Standard short-hand to make a user-type token.
-#define H_MAKE(TYP, VAL) h_make(p->arena, TT_ ## TYP, VAL)
+// Standard short-hands to make tokens in an action.
+#define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
+#define H_MAKE_SEQ()      h_make_seq(p->arena)
+#define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
+#define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
+#define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
 
 // Sequence access...
 

From 70aaf1308cc6afb8e89a1464de40f8a2b2afcd26 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:12:00 +0100
Subject: [PATCH 067/104] massage glue api around some, add a bunch of dummy
 declarations

---
 examples/dns.c  | 10 +++++-----
 examples/glue.c |  8 ++++----
 examples/glue.h | 42 +++++++++++++++++++++++++++++++-----------
 src/hammer.h    | 10 ++++++----
 4 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index c8e3faf..c126812 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -143,7 +143,7 @@ const HParsedToken* act_question(const HParseResult *p) {
   q->qname.qlen   = fields[0]->seq->used;
   q->qname.labels = h_arena_malloc(p->arena, sizeof(dns_label_t)*q->qname.qlen);
   for(size_t i=0; i<fields[0]->seq->used; i++) {
-    q->qname.labels[i] = *H_SEQ_INDEX(dns_label_t, fields[0], i);
+    q->qname.labels[i] = *H_INDEX(dns_label_t, fields[0], i);
   }
 
   q->qtype  = fields[1]->uint;
@@ -165,7 +165,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_question *questions = h_arena_malloc(p->arena,
 						  sizeof(struct dns_question)*(header->question_count));
   for (size_t i=0; i<header->question_count; ++i) {
-    questions[i] = *H_SEQ_INDEX(dns_question_t, qs, i);
+    questions[i] = *H_INDEX(dns_question_t, qs, i);
   }
   msg->questions = questions;
 
@@ -174,7 +174,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *answers = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->answer_count));
   for (size_t i=0; i<header->answer_count; ++i) {
-    answers[i] = *H_SEQ_INDEX(dns_rr_t, rrs, i);
+    answers[i] = *H_INDEX(dns_rr_t, rrs, i);
   }
   msg->answers = answers;
 
@@ -182,7 +182,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *authority = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->authority_count));
   for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
-    authority[i] = *H_SEQ_INDEX(dns_rr_t, rrs, j);
+    authority[i] = *H_INDEX(dns_rr_t, rrs, j);
   }
   msg->authority = authority;
 
@@ -190,7 +190,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   struct dns_rr *additional = h_arena_malloc(p->arena,
 					     sizeof(struct dns_rr)*(header->additional_count));
   for (size_t i=0, j=header->answer_count+header->authority_count; i<header->additional_count; ++i, ++j) {
-    additional[i] = *H_SEQ_INDEX(dns_rr_t, rrs, j);
+    additional[i] = *H_INDEX(dns_rr_t, rrs, j);
   }
   msg->additional = additional;
 
diff --git a/examples/glue.c b/examples/glue.c
index 4cc85d0..6fe3392 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -57,7 +57,7 @@ const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
   case TT_SEQUENCE:
     // Flatten and append all.
     for(size_t i; i<p->seq->used; i++) {
-	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index_token(p, i)));
+	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
 	}
 	break;
   default:
@@ -126,16 +126,16 @@ HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
   return a->elements[i];
 }
 
-HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i)
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
 {
   assert(p != NULL);
   assert(p->token_type == TT_SEQUENCE);
   return h_carray_index(p->seq, i);
 }
 
-void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i)
+void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
 {
-  HParsedToken *elem = h_seq_index_token(p, i);
+  HParsedToken *elem = h_seq_index(p, i);
   assert(elem->token_type == (HTokenType)type);
   return elem->user;
 }
diff --git a/examples/glue.h b/examples/glue.h
index 9b11315..4712cfc 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -44,8 +44,7 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 //
 
 // Standard short-hand for arena-allocating a variable in a semantic action.
-#define H_ALLOC(TYP) \
-  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
+#define H_ALLOC(TYP)  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
 
 // Token constructors...
 
@@ -62,28 +61,49 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 #define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
 #define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
 
+// Extract type-specific value back from HParsedTokens...
+
+void *         h_cast      (HTokenType type, const HParsedToken *p);
+HCountedArray *h_cast_seq  (const HParsedToken *p);
+HBytes         h_cast_bytes(const HParsedToken *p);
+int64_t        h_cast_sint (const HParsedToken *p);
+uint64_t       h_cast_uint (const HParsedToken *p);
+
+// Standard short-hand to cast to a user type.
+#define H_CAST(TYP, TOK)  ((TYP *) h_cast(TT_ ## TYP, TOK))
+
 // Sequence access...
 
 // Access a sequence element by index.
-HParsedToken *h_seq_index_token(const HParsedToken *p, size_t i);
+HParsedToken * h_seq_index(const HParsedToken *p, size_t i);
 
-// Access a user-type element of a sequence by index.
-#define H_SEQ_INDEX(TYP, SEQ, IDX) \
-  ((TYP *) h_seq_index(TT_ ## TYP, SEQ, IDX))
+// Convenience functions combining index access and h_cast_*.
+HCountedArray *h_seq_index_seq  (const HParsedToken *p, size_t i);
+HBytes         h_seq_index_bytes(const HParsedToken *p, size_t i);
+int64_t        h_seq_index_sint (const HParsedToken *p, size_t i);
+uint64_t       h_seq_index_uint (const HParsedToken *p, size_t i);
+void *         h_seq_index_user (HTokenType type, const HParsedToken *p, size_t i);
 
-// Standard short-hand to access a user-type field on a sequence token.
-#define H_FIELD(TYP, IDX)  H_SEQ_INDEX(TYP, p->ast, IDX)
+// Standard short-hand to access and cast a user-type sequence element.
+#define H_INDEX(TYP, SEQ, IDX) \
+  ((TYP *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
 
-// Lower-level helper for H_SEQ_INDEX.
-void *h_seq_index(HTokenType type, const HParsedToken *p, size_t i);
+// Standard short-hand to access and cast a user-type field on a sequence token.
+#define H_FIELD(TYP, IDX)  H_INDEX(TYP, p->ast, IDX)
+
+// Lower-level helper for h_seq_index.
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
 
 // Sequence modification...
 
-// Append elements to a sequence.
+// Add elements to a sequence.
+void h_seq_cons(const HParsedToken *x, HParsedToken *xs);     // prepend one
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
+void h_seq_prepend(const HParsedToken *xs, HParsedToken *ys); // prepend many
 void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
 
+// XXX TODO: Remove elements from a sequence.
+
 // Flatten nested sequences into one.
 const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
 
diff --git a/src/hammer.h b/src/hammer.h
index 5a0c625..7ef2883 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -54,13 +54,15 @@ typedef struct HCountedArray_ {
   struct HParsedToken_ **elements;
 } HCountedArray;
 
+typedef struct HBytes_ {
+  const uint8_t *token;
+  size_t len;
+} HBytes;
+
 typedef struct HParsedToken_ {
   HTokenType token_type;
   union {
-    struct {
-      const uint8_t *token;
-      size_t len;
-    } bytes;
+    HBytes bytes;
     int64_t sint;
     uint64_t uint;
     double dbl;

From d6ef9ac526f417b9f6f0b60756bf8c19cc7a68c2 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:32:01 +0100
Subject: [PATCH 068/104] add definitions for h_cast family of functions

---
 examples/glue.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/examples/glue.c b/examples/glue.c
index 6fe3392..52ffd89 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -119,6 +119,36 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val)
   return ret;
 }
 
+void * h_cast(HTokenType type, const HParsedToken *p)
+{
+  assert(p->token_type == type);
+  return p->user;
+}
+
+HCountedArray *h_cast_seq (const HParsedToken *p)
+{
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq;
+}
+
+HBytes h_cast_bytes(const HParsedToken *p)
+{
+  assert(p->token_type == TT_BYTES);
+  return p->bytes;
+}
+
+int64_t h_cast_sint (const HParsedToken *p)
+{
+  assert(p->token_type == TT_SINT);
+  return p->sint;
+}
+
+uint64_t h_cast_uint (const HParsedToken *p)
+{
+  assert(p->token_type == TT_UINT);
+  return p->uint;
+}
+
 // XXX -> internal
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 {

From 0b4002c697102ce0ca1ca6ae413df1548fbd2307 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:37:04 +0100
Subject: [PATCH 069/104] add definitions for h_seq_index family of functions

---
 examples/glue.c | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 52ffd89..01eb08c 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -125,7 +125,7 @@ void * h_cast(HTokenType type, const HParsedToken *p)
   return p->user;
 }
 
-HCountedArray *h_cast_seq (const HParsedToken *p)
+HCountedArray *h_cast_seq(const HParsedToken *p)
 {
   assert(p->token_type == TT_SEQUENCE);
   return p->seq;
@@ -137,13 +137,13 @@ HBytes h_cast_bytes(const HParsedToken *p)
   return p->bytes;
 }
 
-int64_t h_cast_sint (const HParsedToken *p)
+int64_t h_cast_sint(const HParsedToken *p)
 {
   assert(p->token_type == TT_SINT);
   return p->sint;
 }
 
-uint64_t h_cast_uint (const HParsedToken *p)
+uint64_t h_cast_uint(const HParsedToken *p)
 {
   assert(p->token_type == TT_UINT);
   return p->uint;
@@ -163,9 +163,27 @@ HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
   return h_carray_index(p->seq, i);
 }
 
+HCountedArray *h_seq_index_seq(const HParsedToken *p, size_t i)
+{
+  return h_cast_seq(h_seq_index(p, i));
+}
+
+HBytes h_seq_index_bytes(const HParsedToken *p, size_t i)
+{
+  return h_cast_bytes(h_seq_index(p, i));
+}
+
+int64_t h_seq_index_sint(const HParsedToken *p, size_t i)
+{
+  return h_cast_sint(h_seq_index(p, i));
+}
+
+uint64_t h_seq_index_uint(const HParsedToken *p, size_t i)
+{
+  return h_cast_uint(h_seq_index(p, i));
+}
+
 void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
 {
-  HParsedToken *elem = h_seq_index(p, i);
-  assert(elem->token_type == (HTokenType)type);
-  return elem->user;
+  return h_cast(type, h_seq_index(p, i));
 }

From dbb77c0c551364c291ebb4a9908aab2011fc9654 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:42:45 +0100
Subject: [PATCH 070/104] ah, it's not an slist, forget about cons and prepend
 for now

---
 examples/glue.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/glue.h b/examples/glue.h
index 4712cfc..5fdabc3 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -97,9 +97,7 @@ HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> intern
 // Sequence modification...
 
 // Add elements to a sequence.
-void h_seq_cons(const HParsedToken *x, HParsedToken *xs);     // prepend one
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
-void h_seq_prepend(const HParsedToken *xs, HParsedToken *ys); // prepend many
 void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
 
 // XXX TODO: Remove elements from a sequence.

From 118c03c5346c33768dff6c07f9bad6a346b52291 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:43:40 +0100
Subject: [PATCH 071/104] move h_seq modification functions around to match the
 order in the header

---
 examples/glue.c | 84 ++++++++++++++++++++++++-------------------------
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 01eb08c..49bda56 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -27,48 +27,6 @@ const HParsedToken *h_act_index(int i, const HParseResult *p)
         return tok->seq->elements[i];
 }
 
-void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
-{
-  assert(xs != NULL);
-  assert(xs->token_type == TT_SEQUENCE);
-
-  h_carray_append(xs->seq, (void *)x);
-}
-
-void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
-{
-  assert(xs != NULL);
-  assert(xs->token_type == TT_SEQUENCE);
-  assert(ys != NULL);
-  assert(ys->token_type == TT_SEQUENCE);
-
-  for(size_t i; i<ys->seq->used; i++)
-	h_carray_append(xs->seq, ys->seq->elements[i]);
-}
-
-// Flatten nested sequences. Always returns a sequence.
-// If input element is not a sequence, returns it as a singleton sequence.
-const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
-{
-  assert(p != NULL);
-
-  HParsedToken *ret = h_make_seq(arena);
-  switch(p->token_type) {
-  case TT_SEQUENCE:
-    // Flatten and append all.
-    for(size_t i; i<p->seq->used; i++) {
-	  h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
-	}
-	break;
-  default:
-    // Make singleton sequence.
-    h_seq_snoc(ret, p);
-	break;
-  }
-
-  return ret;
-}
-
 // Action version of h_seq_flatten.
 const HParsedToken *h_act_flatten(const HParseResult *p) {
   return h_seq_flatten(p->arena, p->ast);
@@ -187,3 +145,45 @@ void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
 {
   return h_cast(type, h_seq_index(p, i));
 }
+
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+
+  h_carray_append(xs->seq, (HParsedToken *)x);
+}
+
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+  assert(ys != NULL);
+  assert(ys->token_type == TT_SEQUENCE);
+
+  for(size_t i; i<ys->seq->used; i++)
+    h_carray_append(xs->seq, ys->seq->elements[i]);
+}
+
+// Flatten nested sequences. Always returns a sequence.
+// If input element is not a sequence, returns it as a singleton sequence.
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
+{
+  assert(p != NULL);
+
+  HParsedToken *ret = h_make_seq(arena);
+  switch(p->token_type) {
+  case TT_SEQUENCE:
+    // Flatten and append all.
+    for(size_t i; i<p->seq->used; i++) {
+      h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
+    }
+    break;
+  default:
+    // Make singleton sequence.
+    h_seq_snoc(ret, p);
+    break;
+  }
+
+  return ret;
+}

From b58b82b4735d3045b9b7ec05fee01f123bd2c048 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 18:52:07 +0100
Subject: [PATCH 072/104] add h_seq_len

---
 examples/glue.c | 7 +++++++
 examples/glue.h | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/examples/glue.c b/examples/glue.c
index 49bda56..de5f1e7 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -114,6 +114,13 @@ HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
   return a->elements[i];
 }
 
+size_t h_seq_len(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->used;
+}
+
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
 {
   assert(p != NULL);
diff --git a/examples/glue.h b/examples/glue.h
index 5fdabc3..4f69f51 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -74,6 +74,9 @@ uint64_t       h_cast_uint (const HParsedToken *p);
 
 // Sequence access...
 
+// Return the length of a sequence.
+size_t h_seq_len(const HParsedToken *p);
+
 // Access a sequence element by index.
 HParsedToken * h_seq_index(const HParsedToken *p, size_t i);
 

From 6e166174d6c10a927c7c6edaadb410e6dd74e068 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:06:28 +0100
Subject: [PATCH 073/104] add H_FIELD variants for built-in types

---
 examples/glue.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/examples/glue.h b/examples/glue.h
index 4f69f51..568cbda 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -91,8 +91,12 @@ void *         h_seq_index_user (HTokenType type, const HParsedToken *p, size_t
 #define H_INDEX(TYP, SEQ, IDX) \
   ((TYP *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
 
-// Standard short-hand to access and cast a user-type field on a sequence token.
+// Standard short-hand to access and cast elements on a sequence token.
 #define H_FIELD(TYP, IDX)  H_INDEX(TYP, p->ast, IDX)
+#define H_FIELD_SEQ(IDX)   h_seq_index_seq(p->ast, IDX)
+#define H_FIELD_BYTES(IDX) h_seq_index_bytes(p->ast, IDX)
+#define H_FIELD_SINT(IDX)  h_seq_index_sint(p->ast, IDX)
+#define H_FIELD_UINT(IDX)  h_seq_index_uint(p->ast, IDX)
 
 // Lower-level helper for h_seq_index.
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal

From 0e37c6b8a16346dafe7d8d3eb4a3a0d86a57d41f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:11:02 +0100
Subject: [PATCH 074/104] apply some simplifications to dns example

---
 examples/dns.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index c126812..381be6a 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -28,17 +28,18 @@ bool validate_hdzero(HParseResult *p) {
 bool validate_message(HParseResult *p) {
   if (TT_SEQUENCE != p->ast->token_type)
     return false;
+
   dns_header_t *header = H_FIELD(dns_header_t, 0);
   size_t qd = header->question_count;
   size_t an = header->answer_count;
   size_t ns = header->authority_count;
   size_t ar = header->additional_count;
-  HParsedToken *questions = p->ast->seq->elements[1];
-  if (questions->seq->used != qd)
+
+  if (H_FIELD_SEQ(1)->used != qd)
     return false;
-  HParsedToken *rrs = p->ast->seq->elements[2];
-  if (an+ns+ar != rrs->seq->used)
+  if (an+ns+ar != H_FIELD_SEQ(2)->used)
     return false;
+
   return true;
 }
 
@@ -51,7 +52,7 @@ bool validate_message(HParseResult *p) {
 void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
-    data[i] = rdata->elements[i]->uint;
+    data[i] = h_cast_uint(rdata->elements[i]);
 
   // Parse RDATA if possible.
   const HParseResult *p = NULL;

From 690a17b1f6d7774848a68482ab0f79b74d6f297b Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:12:01 +0100
Subject: [PATCH 075/104] fix semantic action on A records

---
 examples/dns.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/dns.c b/examples/dns.c
index 381be6a..3125b39 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -66,7 +66,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
 
   // Pack the parsed rdata into rr.
   switch(rr.type) {
-  case 1:  rr.a     = p->ast->seq->elements[0]->uint;  break;
+  case 1:  rr.a     = h_cast_uint(p->ast);             break;
   case 2:  rr.ns    = *(dns_domain_t   *)p->ast->user; break;
   case 3:  rr.md    = *(dns_domain_t   *)p->ast->user; break;
   case 4:  rr.md    = *(dns_domain_t   *)p->ast->user; break;

From 2f312b8c1f94587392cc16f5d4f833754e7beb8f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:16:25 +0100
Subject: [PATCH 076/104] use H_CAST in set_rdata

---
 examples/dns.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 3125b39..fb33171 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -67,21 +67,21 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   // Pack the parsed rdata into rr.
   switch(rr.type) {
   case 1:  rr.a     = h_cast_uint(p->ast);             break;
-  case 2:  rr.ns    = *(dns_domain_t   *)p->ast->user; break;
-  case 3:  rr.md    = *(dns_domain_t   *)p->ast->user; break;
-  case 4:  rr.md    = *(dns_domain_t   *)p->ast->user; break;
-  case 5:  rr.cname = *(dns_domain_t   *)p->ast->user; break;
-  case 6:  rr.soa   = *(dns_rr_soa_t   *)p->ast->user; break;
-  case 7:  rr.mb    = *(dns_domain_t   *)p->ast->user; break;
-  case 8:  rr.mg    = *(dns_domain_t   *)p->ast->user; break;
-  case 9:  rr.mr    = *(dns_domain_t   *)p->ast->user; break;
-  case 10: rr.null  = *(dns_rr_null_t  *)p->ast->user; break;
-  case 11: rr.wks   = *(dns_rr_wks_t   *)p->ast->user; break;
-  case 12: rr.ptr   = *(dns_domain_t   *)p->ast->user; break;
-  case 13: rr.hinfo = *(dns_rr_hinfo_t *)p->ast->user; break;
-  case 14: rr.minfo = *(dns_rr_minfo_t *)p->ast->user; break;
-  case 15: rr.mx    = *(dns_rr_mx_t    *)p->ast->user; break;
-  case 16: rr.txt   = *(dns_rr_txt_t   *)p->ast->user; break;
+  case 2:  rr.ns    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 3:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 4:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 5:  rr.cname = *H_CAST(dns_domain_t,   p->ast); break;
+  case 6:  rr.soa   = *H_CAST(dns_rr_soa_t,   p->ast); break;
+  case 7:  rr.mb    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 8:  rr.mg    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 9:  rr.mr    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 10: rr.null  = *H_CAST(dns_rr_null_t,  p->ast); break;
+  case 11: rr.wks   = *H_CAST(dns_rr_wks_t,   p->ast); break;
+  case 12: rr.ptr   = *H_CAST(dns_domain_t,   p->ast); break;
+  case 13: rr.hinfo = *H_CAST(dns_rr_hinfo_t, p->ast); break;
+  case 14: rr.minfo = *H_CAST(dns_rr_minfo_t, p->ast); break;
+  case 15: rr.mx    = *H_CAST(dns_rr_mx_t,    p->ast); break;
+  case 16: rr.txt   = *H_CAST(dns_rr_txt_t,   p->ast); break;
   default:                                             break;
   }
 }

From 102d9e2c8655cf4f940c0d7c6628db455cb2a8ea Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:24:04 +0100
Subject: [PATCH 077/104] add h_seq_elements accessor

---
 examples/glue.c | 7 +++++++
 examples/glue.h | 5 ++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/examples/glue.c b/examples/glue.c
index de5f1e7..02087d4 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -121,6 +121,13 @@ size_t h_seq_len(const HParsedToken *p)
   return p->seq->used;
 }
 
+HParsedToken **h_seq_elements(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->elements;
+}
+
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
 {
   assert(p != NULL);
diff --git a/examples/glue.h b/examples/glue.h
index 568cbda..b386625 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -77,8 +77,11 @@ uint64_t       h_cast_uint (const HParsedToken *p);
 // Return the length of a sequence.
 size_t h_seq_len(const HParsedToken *p);
 
+// Access a sequence's element array.
+HParsedToken **h_seq_elements(const HParsedToken *p);
+
 // Access a sequence element by index.
-HParsedToken * h_seq_index(const HParsedToken *p, size_t i);
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 
 // Convenience functions combining index access and h_cast_*.
 HCountedArray *h_seq_index_seq  (const HParsedToken *p, size_t i);

From 53bc7af36aca2af6555f91e0e51da3ebaba0cfa6 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:31:10 +0100
Subject: [PATCH 078/104] use more glue in dns.c

---
 examples/dns.c | 54 +++++++++++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index fb33171..45c05ba 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -87,20 +87,20 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
 }
 
 const HParsedToken* act_header(const HParseResult *p) {
-  HParsedToken **fields = p->ast->seq->elements;
+  HParsedToken **fields = h_seq_elements(p->ast);
   dns_header_t header_ = {
-    .id     = fields[0]->uint,
-    .qr     = fields[1]->uint,
-    .opcode = fields[2]->uint,
-    .aa     = fields[3]->uint,
-    .tc     = fields[4]->uint,
-    .rd     = fields[5]->uint,
-    .ra     = fields[6]->uint,
-    .rcode  = fields[7]->uint,
-    .question_count   = fields[8]->uint,
-    .answer_count     = fields[9]->uint,
-    .authority_count  = fields[10]->uint,
-    .additional_count = fields[11]->uint
+    .id     = h_cast_uint(fields[0]),
+    .qr     = h_cast_uint(fields[1]),
+    .opcode = h_cast_uint(fields[2]),
+    .aa     = h_cast_uint(fields[3]),
+    .tc     = h_cast_uint(fields[4]),
+    .rd     = h_cast_uint(fields[5]),
+    .ra     = h_cast_uint(fields[6]),
+    .rcode  = h_cast_uint(fields[7]),
+    .question_count   = h_cast_uint(fields[8]),
+    .answer_count     = h_cast_uint(fields[9]),
+    .authority_count  = h_cast_uint(fields[10]),
+    .additional_count = h_cast_uint(fields[11])
   };
 
   dns_header_t *header = H_ALLOC(dns_header_t);
@@ -112,10 +112,10 @@ const HParsedToken* act_header(const HParseResult *p) {
 const HParsedToken* act_label(const HParseResult *p) {
   dns_label_t *r = H_ALLOC(dns_label_t);
 
-  r->len = p->ast->seq->used;
+  r->len = h_seq_len(p->ast);
   r->label = h_arena_malloc(p->arena, r->len + 1);
   for (size_t i=0; i<r->len; ++i)
-    r->label[i] = p->ast->seq->elements[i]->uint;
+    r->label[i] = H_FIELD_UINT(i);
   r->label[r->len] = 0;
 
   return H_MAKE(dns_label_t, r);
@@ -125,30 +125,30 @@ const HParsedToken* act_rr(const HParseResult *p) {
   dns_rr_t *rr = H_ALLOC(dns_rr_t);
 
   rr->name     = *H_FIELD(dns_domain_t, 0);
-  rr->type     = p->ast->seq->elements[1]->uint;
-  rr->class    = p->ast->seq->elements[2]->uint;
-  rr->ttl      = p->ast->seq->elements[3]->uint;
-  rr->rdlength = p->ast->seq->elements[4]->seq->used;
+  rr->type     = H_FIELD_UINT(1);
+  rr->class    = H_FIELD_UINT(2);
+  rr->ttl      = H_FIELD_UINT(3);
+  rr->rdlength = H_FIELD_SEQ(4)->used;
 
   // Parse and pack RDATA.
-  set_rdata(*rr, p->ast->seq->elements[4]->seq);	   
+  set_rdata(*rr, H_FIELD_SEQ(4));
 
   return H_MAKE(dns_rr_t, rr);
 }
 
 const HParsedToken* act_question(const HParseResult *p) {
   dns_question_t *q = H_ALLOC(dns_question_t);
-  HParsedToken **fields = p->ast->seq->elements;
+  HParsedToken **fields = h_seq_elements(p->ast);
 
   // QNAME is a sequence of labels. Pack them into an array.
-  q->qname.qlen   = fields[0]->seq->used;
+  q->qname.qlen   = h_seq_len(fields[0]);
   q->qname.labels = h_arena_malloc(p->arena, sizeof(dns_label_t)*q->qname.qlen);
-  for(size_t i=0; i<fields[0]->seq->used; i++) {
+  for(size_t i=0; i<q->qname.qlen; i++) {
     q->qname.labels[i] = *H_INDEX(dns_label_t, fields[0], i);
   }
 
-  q->qtype  = fields[1]->uint;
-  q->qclass = fields[2]->uint;
+  q->qtype  = h_cast_uint(fields[1]);
+  q->qclass = h_cast_uint(fields[2]);
 
   return H_MAKE(dns_question_t, q);
 }
@@ -162,7 +162,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   msg->header = *header;
 
   // Copy questions into message struct.
-  HParsedToken *qs = p->ast->seq->elements[1];
+  HParsedToken *qs = h_seq_index(p->ast, 1);
   struct dns_question *questions = h_arena_malloc(p->arena,
 						  sizeof(struct dns_question)*(header->question_count));
   for (size_t i=0; i<header->question_count; ++i) {
@@ -171,7 +171,7 @@ const HParsedToken* act_message(const HParseResult *p) {
   msg->questions = questions;
 
   // Copy answer RRs into message struct.
-  HParsedToken *rrs = p->ast->seq->elements[2];
+  HParsedToken *rrs = h_seq_index(p->ast, 2);
   struct dns_rr *answers = h_arena_malloc(p->arena,
 					  sizeof(struct dns_rr)*(header->answer_count));
   for (size_t i=0; i<header->answer_count; ++i) {

From b6856fdfbfb28b1ca113afa6b68e7a50236309fa Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 19:32:09 +0100
Subject: [PATCH 079/104] indentation fix

---
 examples/dns.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/dns.c b/examples/dns.c
index 45c05ba..c404f6c 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -181,7 +181,7 @@ const HParsedToken* act_message(const HParseResult *p) {
 
   // Copy authority RRs into message struct.
   struct dns_rr *authority = h_arena_malloc(p->arena,
-					  sizeof(struct dns_rr)*(header->authority_count));
+					    sizeof(struct dns_rr)*(header->authority_count));
   for (size_t i=0, j=header->answer_count; i<header->authority_count; ++i, ++j) {
     authority[i] = *H_INDEX(dns_rr_t, rrs, j);
   }

From 64c44522e90e8d4d47b4ebdc6fb03636ecc48b75 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 20:16:44 +0100
Subject: [PATCH 080/104] use new glue api in rr.c

---
 examples/rr.c | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/examples/rr.c b/examples/rr.c
index 94c4b7c..38edd18 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -20,10 +20,10 @@ bool validate_null(HParseResult *p) {
 const HParsedToken *act_null(const HParseResult *p) {
   dns_rr_null_t *null = H_ALLOC(dns_rr_null_t);
 
-  size_t len = p->ast->seq->used;
+  size_t len = h_seq_len(p->ast);
   uint8_t *buf = h_arena_malloc(p->arena, sizeof(uint8_t)*len);
   for (size_t i=0; i<len; ++i)
-    buf[i] = p->ast->seq->elements[i]->uint;
+    buf[i] = H_FIELD_UINT(i);
 
   return H_MAKE(dns_rr_null_t, null);
 }
@@ -31,16 +31,17 @@ const HParsedToken *act_null(const HParseResult *p) {
 const HParsedToken *act_txt(const HParseResult *p) {
   dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t);
 
-  const HCountedArray *arr = p->ast->seq;
+  const HCountedArray *arr = h_cast_seq(p->ast);
   uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
   for (size_t i=0; i<arr->used; ++i) {
-    uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->elements[i]->seq->used);
-    for (size_t j=0; j<arr->elements[i]->seq->used; ++j)
-      tmp[j] = arr->elements[i]->seq->elements[j]->uint;
+    size_t len = h_seq_len(arr->elements[i]);
+    uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*len);
+    for (size_t j=0; j<len; ++j)
+      tmp[j] = h_seq_index_uint(arr->elements[i], j);
     ret[i] = tmp;
   }
 
-  txt->count = p->ast->seq->elements[0]->seq->used;
+  txt->count = arr->used;
   txt->txt_data = ret;
 
   return H_MAKE(dns_rr_txt_t, txt);
@@ -49,10 +50,10 @@ const HParsedToken *act_txt(const HParseResult *p) {
 const HParsedToken* act_cstr(const HParseResult *p) {
   dns_cstr_t *cs = H_ALLOC(dns_cstr_t);
 
-  const HCountedArray *arr = p->ast->seq;
+  const HCountedArray *arr = h_cast_seq(p->ast);
   uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
   for (size_t i=0; i<arr->used; ++i)
-    ret[i] = arr->elements[i]->uint;
+    ret[i] = h_cast_uint(arr->elements[i]);
   assert(ret[arr->used-1] == '\0'); // XXX Is this right?! If so, shouldn't it be a validation?
   *cs = ret;
 
@@ -64,11 +65,11 @@ const HParsedToken* act_soa(const HParseResult *p) {
 
   soa->mname   = *H_FIELD(dns_domain_t, 0);
   soa->rname   = *H_FIELD(dns_domain_t, 1);
-  soa->serial  = p->ast->seq->elements[2]->uint;
-  soa->refresh = p->ast->seq->elements[3]->uint;
-  soa->retry   = p->ast->seq->elements[4]->uint;
-  soa->expire  = p->ast->seq->elements[5]->uint;
-  soa->minimum = p->ast->seq->elements[6]->uint;
+  soa->serial  = H_FIELD_UINT(2);
+  soa->refresh = H_FIELD_UINT(3);
+  soa->retry   = H_FIELD_UINT(4);
+  soa->expire  = H_FIELD_UINT(5);
+  soa->minimum = H_FIELD_UINT(6);
 
   return H_MAKE(dns_rr_soa_t, soa);
 }
@@ -76,12 +77,12 @@ const HParsedToken* act_soa(const HParseResult *p) {
 const HParsedToken* act_wks(const HParseResult *p) {
   dns_rr_wks_t *wks = H_ALLOC(dns_rr_wks_t);
 
-  wks->address  = p->ast->seq->elements[0]->uint;
-  wks->protocol = p->ast->seq->elements[1]->uint;
-  wks->len      = p->ast->seq->elements[2]->seq->used;
+  wks->address  = H_FIELD_UINT(0);
+  wks->protocol = H_FIELD_UINT(1);
+  wks->len      = H_FIELD_SEQ(2)->used;
   wks->bit_map  = h_arena_malloc(p->arena, sizeof(uint8_t)*wks->len);
   for (size_t i=0; i<wks->len; ++i)
-    wks->bit_map[i] = p->ast->seq->elements[2]->seq->elements[i]->uint;
+    wks->bit_map[i] = h_seq_index_uint(h_seq_index(p->ast, 2), i);
 
   return H_MAKE(dns_rr_wks_t, wks);
 }
@@ -107,7 +108,7 @@ const HParsedToken* act_minfo(const HParseResult *p) {
 const HParsedToken* act_mx(const HParseResult *p) {
   dns_rr_mx_t *mx = H_ALLOC(dns_rr_mx_t);
 
-  mx->preference = p->ast->seq->elements[0]->uint;
+  mx->preference = H_FIELD_UINT(0);
   mx->exchange   = *H_FIELD(dns_domain_t, 1);
 
   return H_MAKE(dns_rr_mx_t, mx);

From 811c68fdf268f9dd7838f7b7d8991b79e81d472a Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 20:47:51 +0100
Subject: [PATCH 081/104] let H_INDEX and H_FIELD accept varargs for nested
 sequence access

---
 examples/glue.c | 21 +++++++++++++++++++++
 examples/glue.h | 10 +++++++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 02087d4..1457f6f 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -135,6 +135,27 @@ HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
   return h_carray_index(p->seq, i);
 }
 
+HParsedToken *h_seq_index_path(HParsedToken *p, ...)
+{
+  va_list va;
+
+  va_start(va, p);
+  p = h_seq_index_vpath(p, va);
+  va_end(va);
+
+  return p;
+}
+
+HParsedToken *h_seq_index_vpath(HParsedToken *p, va_list va)
+{
+  int i;
+
+  while((i = va_arg(va, int)) >= 0)
+    p = h_seq_index(p, i);
+
+  return p;
+}
+
 HCountedArray *h_seq_index_seq(const HParsedToken *p, size_t i)
 {
   return h_cast_seq(h_seq_index(p, i));
diff --git a/examples/glue.h b/examples/glue.h
index b386625..cd5e4ef 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -83,6 +83,10 @@ HParsedToken **h_seq_elements(const HParsedToken *p);
 // Access a sequence element by index.
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 
+// Access an element in a nested sequence by a path of indices.
+HParsedToken *h_seq_index_path(HParsedToken *p, ...);
+HParsedToken *h_seq_index_vpath(HParsedToken *p, va_list va);
+
 // Convenience functions combining index access and h_cast_*.
 HCountedArray *h_seq_index_seq  (const HParsedToken *p, size_t i);
 HBytes         h_seq_index_bytes(const HParsedToken *p, size_t i);
@@ -91,11 +95,11 @@ uint64_t       h_seq_index_uint (const HParsedToken *p, size_t i);
 void *         h_seq_index_user (HTokenType type, const HParsedToken *p, size_t i);
 
 // Standard short-hand to access and cast a user-type sequence element.
-#define H_INDEX(TYP, SEQ, IDX) \
-  ((TYP *) h_seq_index_user(TT_ ## TYP, SEQ, IDX))
+#define H_INDEX(TYP, SEQ, ...) \
+  ((TYP *) h_cast(TT_ ## TYP, h_seq_index_path(SEQ, __VA_ARGS__, -1)))
 
 // Standard short-hand to access and cast elements on a sequence token.
-#define H_FIELD(TYP, IDX)  H_INDEX(TYP, p->ast, IDX)
+#define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
 #define H_FIELD_SEQ(IDX)   h_seq_index_seq(p->ast, IDX)
 #define H_FIELD_BYTES(IDX) h_seq_index_bytes(p->ast, IDX)
 #define H_FIELD_SINT(IDX)  h_seq_index_sint(p->ast, IDX)

From 18fbf6fd69fc0a3662b28304ca5cb2e910ecaea0 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 21:09:23 +0100
Subject: [PATCH 082/104] replace casting h_seq_index functions with H_INDEX
 macro family, supporting varargs

---
 examples/glue.c | 44 ++++++++++----------------------------------
 examples/glue.h | 28 +++++++++++++---------------
 examples/rr.c   |  4 ++--
 3 files changed, 25 insertions(+), 51 deletions(-)

diff --git a/examples/glue.c b/examples/glue.c
index 1457f6f..3a8f6cb 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -135,50 +135,26 @@ HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
   return h_carray_index(p->seq, i);
 }
 
-HParsedToken *h_seq_index_path(HParsedToken *p, ...)
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...)
 {
   va_list va;
 
-  va_start(va, p);
-  p = h_seq_index_vpath(p, va);
+  va_start(va, i);
+  HParsedToken *ret = h_seq_index_vpath(p, i, va);
   va_end(va);
 
-  return p;
+  return ret;
 }
 
-HParsedToken *h_seq_index_vpath(HParsedToken *p, va_list va)
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va)
 {
-  int i;
+  HParsedToken *ret = h_seq_index(p, i);
+  int j;
 
-  while((i = va_arg(va, int)) >= 0)
-    p = h_seq_index(p, i);
+  while((j = va_arg(va, int)) >= 0)
+    ret = h_seq_index(p, j);
 
-  return p;
-}
-
-HCountedArray *h_seq_index_seq(const HParsedToken *p, size_t i)
-{
-  return h_cast_seq(h_seq_index(p, i));
-}
-
-HBytes h_seq_index_bytes(const HParsedToken *p, size_t i)
-{
-  return h_cast_bytes(h_seq_index(p, i));
-}
-
-int64_t h_seq_index_sint(const HParsedToken *p, size_t i)
-{
-  return h_cast_sint(h_seq_index(p, i));
-}
-
-uint64_t h_seq_index_uint(const HParsedToken *p, size_t i)
-{
-  return h_cast_uint(h_seq_index(p, i));
-}
-
-void *h_seq_index_user(HTokenType type, const HParsedToken *p, size_t i)
-{
-  return h_cast(type, h_seq_index(p, i));
+  return ret;
 }
 
 void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
diff --git a/examples/glue.h b/examples/glue.h
index cd5e4ef..1dc2fbd 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -84,26 +84,24 @@ HParsedToken **h_seq_elements(const HParsedToken *p);
 HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
 
 // Access an element in a nested sequence by a path of indices.
-HParsedToken *h_seq_index_path(HParsedToken *p, ...);
-HParsedToken *h_seq_index_vpath(HParsedToken *p, va_list va);
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...);
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
 
-// Convenience functions combining index access and h_cast_*.
-HCountedArray *h_seq_index_seq  (const HParsedToken *p, size_t i);
-HBytes         h_seq_index_bytes(const HParsedToken *p, size_t i);
-int64_t        h_seq_index_sint (const HParsedToken *p, size_t i);
-uint64_t       h_seq_index_uint (const HParsedToken *p, size_t i);
-void *         h_seq_index_user (HTokenType type, const HParsedToken *p, size_t i);
-
-// Standard short-hand to access and cast a user-type sequence element.
+// Convenience macros combining (nested) index access and h_cast.
 #define H_INDEX(TYP, SEQ, ...) \
-  ((TYP *) h_cast(TT_ ## TYP, h_seq_index_path(SEQ, __VA_ARGS__, -1)))
+  ((TYP *) h_cast(TT_ ## TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__)))
+#define H_INDEX_SEQ(SEQ, ...)    h_cast_seq(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_BYTES(SEQ, ...)  h_cast_bytes(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SINT(SEQ, ...)   h_cast_sint(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_UINT(SEQ, ...)   h_cast_uint(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
 
 // Standard short-hand to access and cast elements on a sequence token.
 #define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
-#define H_FIELD_SEQ(IDX)   h_seq_index_seq(p->ast, IDX)
-#define H_FIELD_BYTES(IDX) h_seq_index_bytes(p->ast, IDX)
-#define H_FIELD_SINT(IDX)  h_seq_index_sint(p->ast, IDX)
-#define H_FIELD_UINT(IDX)  h_seq_index_uint(p->ast, IDX)
+#define H_FIELD_SEQ(...)   H_INDEX_SEQ(p->ast, __VA_ARGS__)
+#define H_FIELD_BYTES(...) H_INDEX_BYTES(p->ast, __VA_ARGS__)
+#define H_FIELD_SINT(...)  H_INDEX_SINT(p->ast, __VA_ARGS__)
+#define H_FIELD_UINT(...)  H_INDEX_UINT(p->ast, __VA_ARGS__)
 
 // Lower-level helper for h_seq_index.
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
diff --git a/examples/rr.c b/examples/rr.c
index 38edd18..1e74370 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -37,7 +37,7 @@ const HParsedToken *act_txt(const HParseResult *p) {
     size_t len = h_seq_len(arr->elements[i]);
     uint8_t *tmp = h_arena_malloc(arr->arena, sizeof(uint8_t)*len);
     for (size_t j=0; j<len; ++j)
-      tmp[j] = h_seq_index_uint(arr->elements[i], j);
+      tmp[j] = H_INDEX_UINT(arr->elements[i], j);
     ret[i] = tmp;
   }
 
@@ -82,7 +82,7 @@ const HParsedToken* act_wks(const HParseResult *p) {
   wks->len      = H_FIELD_SEQ(2)->used;
   wks->bit_map  = h_arena_malloc(p->arena, sizeof(uint8_t)*wks->len);
   for (size_t i=0; i<wks->len; ++i)
-    wks->bit_map[i] = h_seq_index_uint(h_seq_index(p->ast, 2), i);
+    wks->bit_map[i] = H_INDEX_UINT(p->ast, 2, i);
 
   return H_MAKE(dns_rr_wks_t, wks);
 }

From 7149260a13def22c236f11a6ecfd09ff6cd22ef7 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 25 Jan 2013 21:52:11 +0100
Subject: [PATCH 083/104] make h_cast a family of macros, add H_ASSERT family,
 make h_assert_type a macro

---
 examples/dns.c  | 32 ++++++++++++++++----------------
 examples/glue.c | 30 ------------------------------
 examples/glue.h | 35 +++++++++++++++++++++--------------
 examples/rr.c   |  6 +++---
 4 files changed, 40 insertions(+), 63 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index c404f6c..6c572a0 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -52,7 +52,7 @@ bool validate_message(HParseResult *p) {
 void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
-    data[i] = h_cast_uint(rdata->elements[i]);
+    data[i] = H_CAST_UINT(rdata->elements[i]);
 
   // Parse RDATA if possible.
   const HParseResult *p = NULL;
@@ -66,7 +66,7 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
 
   // Pack the parsed rdata into rr.
   switch(rr.type) {
-  case 1:  rr.a     = h_cast_uint(p->ast);             break;
+  case 1:  rr.a     = H_CAST_UINT(p->ast);             break;
   case 2:  rr.ns    = *H_CAST(dns_domain_t,   p->ast); break;
   case 3:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
   case 4:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
@@ -89,18 +89,18 @@ void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
 const HParsedToken* act_header(const HParseResult *p) {
   HParsedToken **fields = h_seq_elements(p->ast);
   dns_header_t header_ = {
-    .id     = h_cast_uint(fields[0]),
-    .qr     = h_cast_uint(fields[1]),
-    .opcode = h_cast_uint(fields[2]),
-    .aa     = h_cast_uint(fields[3]),
-    .tc     = h_cast_uint(fields[4]),
-    .rd     = h_cast_uint(fields[5]),
-    .ra     = h_cast_uint(fields[6]),
-    .rcode  = h_cast_uint(fields[7]),
-    .question_count   = h_cast_uint(fields[8]),
-    .answer_count     = h_cast_uint(fields[9]),
-    .authority_count  = h_cast_uint(fields[10]),
-    .additional_count = h_cast_uint(fields[11])
+    .id     = H_CAST_UINT(fields[0]),
+    .qr     = H_CAST_UINT(fields[1]),
+    .opcode = H_CAST_UINT(fields[2]),
+    .aa     = H_CAST_UINT(fields[3]),
+    .tc     = H_CAST_UINT(fields[4]),
+    .rd     = H_CAST_UINT(fields[5]),
+    .ra     = H_CAST_UINT(fields[6]),
+    .rcode  = H_CAST_UINT(fields[7]),
+    .question_count   = H_CAST_UINT(fields[8]),
+    .answer_count     = H_CAST_UINT(fields[9]),
+    .authority_count  = H_CAST_UINT(fields[10]),
+    .additional_count = H_CAST_UINT(fields[11])
   };
 
   dns_header_t *header = H_ALLOC(dns_header_t);
@@ -147,8 +147,8 @@ const HParsedToken* act_question(const HParseResult *p) {
     q->qname.labels[i] = *H_INDEX(dns_label_t, fields[0], i);
   }
 
-  q->qtype  = h_cast_uint(fields[1]);
-  q->qclass = h_cast_uint(fields[2]);
+  q->qtype  = H_CAST_UINT(fields[1]);
+  q->qclass = H_CAST_UINT(fields[2]);
 
   return H_MAKE(dns_question_t, q);
 }
diff --git a/examples/glue.c b/examples/glue.c
index 3a8f6cb..7f9c6fa 100644
--- a/examples/glue.c
+++ b/examples/glue.c
@@ -77,36 +77,6 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val)
   return ret;
 }
 
-void * h_cast(HTokenType type, const HParsedToken *p)
-{
-  assert(p->token_type == type);
-  return p->user;
-}
-
-HCountedArray *h_cast_seq(const HParsedToken *p)
-{
-  assert(p->token_type == TT_SEQUENCE);
-  return p->seq;
-}
-
-HBytes h_cast_bytes(const HParsedToken *p)
-{
-  assert(p->token_type == TT_BYTES);
-  return p->bytes;
-}
-
-int64_t h_cast_sint(const HParsedToken *p)
-{
-  assert(p->token_type == TT_SINT);
-  return p->sint;
-}
-
-uint64_t h_cast_uint(const HParsedToken *p)
-{
-  assert(p->token_type == TT_UINT);
-  return p->uint;
-}
-
 // XXX -> internal
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 {
diff --git a/examples/glue.h b/examples/glue.h
index 1dc2fbd..addcf18 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -61,16 +61,24 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 #define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
 #define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
 
-// Extract type-specific value back from HParsedTokens...
+// Extract (cast) type-specific value back from HParsedTokens...
 
-void *         h_cast      (HTokenType type, const HParsedToken *p);
-HCountedArray *h_cast_seq  (const HParsedToken *p);
-HBytes         h_cast_bytes(const HParsedToken *p);
-int64_t        h_cast_sint (const HParsedToken *p);
-uint64_t       h_cast_uint (const HParsedToken *p);
+// Pass-through assertion that a given token has the expected type.
+#define h_assert_type(T,P)  (assert(P->token_type == (HTokenType)T), P)
 
-// Standard short-hand to cast to a user type.
-#define H_CAST(TYP, TOK)  ((TYP *) h_cast(TT_ ## TYP, TOK))
+// Convenience short-hand forms of h_assert_type.
+#define H_ASSERT(TYP, TOK)   h_assert_type(TT_ ## TYP, TOK)
+#define H_ASSERT_SEQ(TOK)    h_assert_type(TT_SEQUENCE, TOK)
+#define H_ASSERT_BYTES(TOK)  h_assert_type(TT_BYTES, TOK)
+#define H_ASSERT_SINT(TOK)   h_assert_type(TT_SINT, TOK)
+#define H_ASSERT_UINT(TOK)   h_assert_type(TT_UINT, TOK)
+
+// Assert expected type and return contained value.
+#define H_CAST(TYP, TOK)   ((TYP *) H_ASSERT(TYP, TOK)->user)
+#define H_CAST_SEQ(TOK)    (H_ASSERT_SEQ(TOK)->seq)
+#define H_CAST_BYTES(TOK)  (H_ASSERT_BYTES(TOK)->bytes)
+#define H_CAST_SINT(TOK)   (H_ASSERT_SINT(TOK)->sint)
+#define H_CAST_UINT(TOK)   (H_ASSERT_UINT(TOK)->uint)
 
 // Sequence access...
 
@@ -88,12 +96,11 @@ HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...);
 HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
 
 // Convenience macros combining (nested) index access and h_cast.
-#define H_INDEX(TYP, SEQ, ...) \
-  ((TYP *) h_cast(TT_ ## TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__)))
-#define H_INDEX_SEQ(SEQ, ...)    h_cast_seq(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_BYTES(SEQ, ...)  h_cast_bytes(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_SINT(SEQ, ...)   h_cast_sint(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_UINT(SEQ, ...)   h_cast_uint(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX(TYP, SEQ, ...)   H_CAST(TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SEQ(SEQ, ...)    H_CAST_SEQ(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_BYTES(SEQ, ...)  H_CAST_BYTES(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SINT(SEQ, ...)   H_CAST_SINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_UINT(SEQ, ...)   H_CAST_UINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
 #define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
 
 // Standard short-hand to access and cast elements on a sequence token.
diff --git a/examples/rr.c b/examples/rr.c
index 1e74370..8c14e0a 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -31,7 +31,7 @@ const HParsedToken *act_null(const HParseResult *p) {
 const HParsedToken *act_txt(const HParseResult *p) {
   dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t);
 
-  const HCountedArray *arr = h_cast_seq(p->ast);
+  const HCountedArray *arr = H_CAST_SEQ(p->ast);
   uint8_t **ret = h_arena_malloc(arr->arena, sizeof(uint8_t*)*arr->used);
   for (size_t i=0; i<arr->used; ++i) {
     size_t len = h_seq_len(arr->elements[i]);
@@ -50,10 +50,10 @@ const HParsedToken *act_txt(const HParseResult *p) {
 const HParsedToken* act_cstr(const HParseResult *p) {
   dns_cstr_t *cs = H_ALLOC(dns_cstr_t);
 
-  const HCountedArray *arr = h_cast_seq(p->ast);
+  const HCountedArray *arr = H_CAST_SEQ(p->ast);
   uint8_t *ret = h_arena_malloc(arr->arena, sizeof(uint8_t)*arr->used);
   for (size_t i=0; i<arr->used; ++i)
-    ret[i] = h_cast_uint(arr->elements[i]);
+    ret[i] = H_CAST_UINT(arr->elements[i]);
   assert(ret[arr->used-1] == '\0'); // XXX Is this right?! If so, shouldn't it be a validation?
   *cs = ret;
 

From e95aef0b09a002ed9e513c5d0298eca2b4b76b32 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 1 Feb 2013 01:42:10 +0100
Subject: [PATCH 084/104] add some docs to glue.h

---
 examples/glue.h | 123 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)

diff --git a/examples/glue.h b/examples/glue.h
index addcf18..ccb488e 100644
--- a/examples/glue.h
+++ b/examples/glue.h
@@ -1,6 +1,25 @@
 //
 // API additions for writing grammar and semantic actions more concisely
 //
+//
+// Quick Overview:
+//
+// Grammars can be succinctly specified with the family of H_RULE macros.
+// H_RULE defines a plain parser variable. H_ARULE additionally attaches a
+// semantic action; H_VRULE attaches a validation. H_AVRULE and H_VARULE
+// combine both.
+//
+// A few standard semantic actions are defined below. The H_ACT_APPLY macro
+// allows semantic actions to be defined by "partial application" of
+// a generic action to fixed paramters.
+//
+// The definition of more complex semantic actions will usually consist of
+// extracting data from the given parse tree and constructing a token of custom
+// type to represent the result. A number of functions and convenience macros
+// are provided to capture the most common cases and idioms.
+//
+// See the leading comment blocks on the sections below for more details.
+//
 
 #ifndef HAMMER_EXAMPLES_GLUE__H
 #define HAMMER_EXAMPLES_GLUE__H
@@ -12,6 +31,29 @@
 //
 // Grammar specification
 //
+// H_RULE is simply a short-hand for the typical declaration and definition of
+// a parser variable. See its plain definition below. The goal is to save
+// horizontal space as well as to provide a clear and unified look together with
+// the other macro variants that stays close to an abstract PEG or BNF grammar.
+// The latter goal is more specifically enabled by H_ARULE, H_VRULE, and their
+// combinations as they allow the definition of syntax to be given without
+// intermingling it with the semantic specifications.
+//
+// H_ARULE defines a variable just like H_RULE but attaches a semantic action
+// to the result of the parser via h_action. The action is expected to be
+// named act_<rulename>.
+//
+// H_VRULE is analogous to H_ARULE but attaches a validation via h_attr_bool.
+// The validation is expected to be named validate_<rulename>.
+//
+// H_VARULE combines H_RULE with both an action and a validation. The action is
+// attached before the validation, i.e. the validation receives as input the
+// result of the action.
+//
+// H_AVRULE is like H_VARULE but the action is attached outside the validation,
+// i.e. the validation receives the uninterpreted AST as input.
+//
+
 
 #define H_RULE(rule, def) const HParser *rule = def
 #define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
@@ -26,6 +68,25 @@
 //
 // Pre-fab semantic actions
 //
+// A collection of generally useful semantic actions is provided.
+//
+// h_act_ignore is the action equivalent of the parser combinator h_ignore. It
+// simply causes the AST it is applied to to be replaced with NULL. This most
+// importantly causes it to be elided from the result of a surrounding
+// h_sequence.
+//
+// h_act_index is of note as it is not itself suitable to be passed to
+// h_action. It is parameterized by an index to be picked from a sequence
+// token. It must be wrapped in a proper HAction to be used. The H_ACT_APPLY
+// macro provides a concise way to define such a parameter-application wrapper.
+//
+// h_act_flatten acts on a token of possibly nested sequences by recursively
+// flattening it into a single sequence. Cf. h_seq_flatten below.
+//
+// H_ACT_APPLY implements "partial application" for semantic actions. It
+// defines a new action that supplies given parameters to a parameterized
+// action such as h_act_index.
+//
 
 const HParsedToken *h_act_ignore(const HParseResult *p);
 const HParsedToken *h_act_index(int i, const HParseResult *p);
@@ -42,6 +103,68 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 //
 // Working with HParsedTokens
 //
+// The type HParsedToken represents a dynamically-typed universe of values.
+// Declared below are constructors to turn ordinary values into their
+// HParsedToken equivalents, extractors to retrieve the original values from
+// inside an HParsedToken, and functions that inspect and modify tokens of
+// sequence type directly.
+//
+// In addition, there are a number of short-hand macros that work with some
+// conventions to eliminate common boilerplate. These conventions are listed
+// below. Be sure to follow them if you want to use the respective macros.
+//
+//  * The single argument to semantic actions should be called 'p'.
+//
+//    The H_MAKE macros suppy 'p->arena' to their underlying h_make
+//    counterparts. The H_FIELD macros supply 'p->ast' to their underlying
+//    H_INDEX counterparts.
+//
+//  * For each custom token type, there should be a typedef for the
+//    corresponding value type.
+//
+//    H_CAST, H_INDEX and H_FIELD cast the void * user field of such a token to
+//    a pointer to the given type. 
+//
+//  * For each custom token type, say 'foo_t', there must be an integer
+//    constant 'TT_foo_t' to identify the token type. This constant must have a
+//    value greater or equal than TT_USER.
+//
+//    One idiom is to define an enum for all custom token types and to assign a
+//    value of TT_USER to the first element. This can be viewed as extending
+//    the HTokenType enum.
+//
+//    The H_MAKE and H_ASSERT macros derive the name of the token type constant
+//    from the given type name.
+//
+//
+// The H_ALLOC macro is useful for allocating values of custom token types.
+//
+// The H_MAKE family of macros construct tokens of a given type. The native
+// token types are indicated by a corresponding suffix such as in H_MAKE_SEQ.
+// The form with no suffix is used for custom token types. This convention is
+// also used for other macro and function families.
+//
+// The H_ASSERT family simply asserts that a given token has the expected type.
+// It mainly serves as an implementation aid for H_CAST. Of note in that regard
+// is that, unlike the standard 'assert' macro, these form _expressions_ that
+// return the value of their token argument; thus they can be used in a
+// "pass-through" fashion inside other expressions.
+//
+// The H_CAST family combines a type assertion with access to the
+// statically-typed value inside a token.
+//
+// A number of functions h_seq_* operate on and inspect sequence tokens.
+// Note that H_MAKE_SEQ takes no arguments and constructs an empty sequence.
+// Therefore there are h_seq_snoc and h_seq_append to build up sequences.
+//
+// The macro families H_FIELD and H_INDEX combine index access on a sequence
+// with a cast to the appropriate result type. H_FIELD is used to access the
+// elements of the argument token 'p' in an action. H_INDEX allows any sequence
+// token to be specified. Both macro families take an arbitrary number of index
+// arguments, giving access to elements in nested sequences by path.
+// These macros are very useful to avoid spaghetti chains of unchecked pointer
+// dereferences.
+//
 
 // Standard short-hand for arena-allocating a variable in a semantic action.
 #define H_ALLOC(TYP)  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))

From 195e50a7f18b86fa8d17aaaa632a454ed67a2d7e Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 15:24:13 +0100
Subject: [PATCH 085/104] move glue.[ch] into the library proper

---
 examples/Makefile     |   3 +-
 examples/dns_common.h |   2 +-
 examples/glue.c       | 170 ----------------------------
 examples/glue.h       | 251 ------------------------------------------
 src/Makefile          |   2 +
 5 files changed, 4 insertions(+), 424 deletions(-)
 delete mode 100644 examples/glue.c
 delete mode 100644 examples/glue.h

diff --git a/examples/Makefile b/examples/Makefile
index 786af44..9c6ac1b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -15,13 +15,12 @@ LDFLAGS += $(pkg-config --libs glib-2.0)
 all: dns base64
 
 dns: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
-dns: dns.o rr.o dns_common.o glue.o
+dns: dns.o rr.o dns_common.o
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
 
 dns.o: ../src/hammer.h dns_common.h
 rr.o: ../src/hammer.h rr.h dns_common.h
 dns_common.o: ../src/hammer.h dns_common.h
-glue.o: ../src/hammer.h glue.h
 
 base64: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
 base64: base64.o
diff --git a/examples/dns_common.h b/examples/dns_common.h
index 6b04519..c1d8d7e 100644
--- a/examples/dns_common.h
+++ b/examples/dns_common.h
@@ -2,7 +2,7 @@
 #define HAMMER_DNS_COMMON__H
 
 #include "../src/hammer.h"
-#include "glue.h"
+#include "../src/glue.h"
 
 const HParser* init_domain();
 const HParser* init_character_string();
diff --git a/examples/glue.c b/examples/glue.c
deleted file mode 100644
index 7f9c6fa..0000000
--- a/examples/glue.c
+++ /dev/null
@@ -1,170 +0,0 @@
-#include "glue.h"
-#include "../src/internal.h"  // for h_carray_*
-
-
-// The action equivalent of h_ignore.
-const HParsedToken *h_act_ignore(const HParseResult *p)
-{
-  return NULL;
-}
-
-// Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *h_act_index(int i, const HParseResult *p)
-{
-    if(!p) return NULL;
-
-    const HParsedToken *tok = p->ast;
-
-    if(!tok || tok->token_type != TT_SEQUENCE)
-        return NULL;
-
-    const HCountedArray *seq = tok->seq;
-    size_t n = seq->used;
-
-    if(i<0 || (size_t)i>=n)
-        return NULL;
-    else
-        return tok->seq->elements[i];
-}
-
-// Action version of h_seq_flatten.
-const HParsedToken *h_act_flatten(const HParseResult *p) {
-  return h_seq_flatten(p->arena, p->ast);
-}
-
-// Low-level helper for the h_make family.
-HParsedToken *h_make_(HArena *arena, HTokenType type)
-{
-  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
-  ret->token_type = type;
-  return ret;
-}
-
-HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
-{
-  assert(type >= TT_USER);
-  HParsedToken *ret = h_make_(arena, type);
-  ret->user = value;
-  return ret;
-}
-
-HParsedToken *h_make_seq(HArena *arena)
-{
-  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
-  ret->seq = h_carray_new(arena);
-  return ret;
-}
-
-HParsedToken *h_make_bytes(HArena *arena, size_t len)
-{
-  HParsedToken *ret = h_make_(arena, TT_BYTES);
-  ret->bytes.len = len;
-  ret->bytes.token = h_arena_malloc(arena, len);
-  return ret;
-}
-
-HParsedToken *h_make_sint(HArena *arena, int64_t val)
-{
-  HParsedToken *ret = h_make_(arena, TT_SINT);
-  ret->sint = val;
-  return ret;
-}
-
-HParsedToken *h_make_uint(HArena *arena, uint64_t val)
-{
-  HParsedToken *ret = h_make_(arena, TT_UINT);
-  ret->uint = val;
-  return ret;
-}
-
-// XXX -> internal
-HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
-{
-  assert(i < a->used);
-  return a->elements[i];
-}
-
-size_t h_seq_len(const HParsedToken *p)
-{
-  assert(p != NULL);
-  assert(p->token_type == TT_SEQUENCE);
-  return p->seq->used;
-}
-
-HParsedToken **h_seq_elements(const HParsedToken *p)
-{
-  assert(p != NULL);
-  assert(p->token_type == TT_SEQUENCE);
-  return p->seq->elements;
-}
-
-HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
-{
-  assert(p != NULL);
-  assert(p->token_type == TT_SEQUENCE);
-  return h_carray_index(p->seq, i);
-}
-
-HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...)
-{
-  va_list va;
-
-  va_start(va, i);
-  HParsedToken *ret = h_seq_index_vpath(p, i, va);
-  va_end(va);
-
-  return ret;
-}
-
-HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va)
-{
-  HParsedToken *ret = h_seq_index(p, i);
-  int j;
-
-  while((j = va_arg(va, int)) >= 0)
-    ret = h_seq_index(p, j);
-
-  return ret;
-}
-
-void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
-{
-  assert(xs != NULL);
-  assert(xs->token_type == TT_SEQUENCE);
-
-  h_carray_append(xs->seq, (HParsedToken *)x);
-}
-
-void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
-{
-  assert(xs != NULL);
-  assert(xs->token_type == TT_SEQUENCE);
-  assert(ys != NULL);
-  assert(ys->token_type == TT_SEQUENCE);
-
-  for(size_t i; i<ys->seq->used; i++)
-    h_carray_append(xs->seq, ys->seq->elements[i]);
-}
-
-// Flatten nested sequences. Always returns a sequence.
-// If input element is not a sequence, returns it as a singleton sequence.
-const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
-{
-  assert(p != NULL);
-
-  HParsedToken *ret = h_make_seq(arena);
-  switch(p->token_type) {
-  case TT_SEQUENCE:
-    // Flatten and append all.
-    for(size_t i; i<p->seq->used; i++) {
-      h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
-    }
-    break;
-  default:
-    // Make singleton sequence.
-    h_seq_snoc(ret, p);
-    break;
-  }
-
-  return ret;
-}
diff --git a/examples/glue.h b/examples/glue.h
deleted file mode 100644
index ccb488e..0000000
--- a/examples/glue.h
+++ /dev/null
@@ -1,251 +0,0 @@
-//
-// API additions for writing grammar and semantic actions more concisely
-//
-//
-// Quick Overview:
-//
-// Grammars can be succinctly specified with the family of H_RULE macros.
-// H_RULE defines a plain parser variable. H_ARULE additionally attaches a
-// semantic action; H_VRULE attaches a validation. H_AVRULE and H_VARULE
-// combine both.
-//
-// A few standard semantic actions are defined below. The H_ACT_APPLY macro
-// allows semantic actions to be defined by "partial application" of
-// a generic action to fixed paramters.
-//
-// The definition of more complex semantic actions will usually consist of
-// extracting data from the given parse tree and constructing a token of custom
-// type to represent the result. A number of functions and convenience macros
-// are provided to capture the most common cases and idioms.
-//
-// See the leading comment blocks on the sections below for more details.
-//
-
-#ifndef HAMMER_EXAMPLES_GLUE__H
-#define HAMMER_EXAMPLES_GLUE__H
-
-#include <assert.h>
-#include "../src/hammer.h"
-
-
-//
-// Grammar specification
-//
-// H_RULE is simply a short-hand for the typical declaration and definition of
-// a parser variable. See its plain definition below. The goal is to save
-// horizontal space as well as to provide a clear and unified look together with
-// the other macro variants that stays close to an abstract PEG or BNF grammar.
-// The latter goal is more specifically enabled by H_ARULE, H_VRULE, and their
-// combinations as they allow the definition of syntax to be given without
-// intermingling it with the semantic specifications.
-//
-// H_ARULE defines a variable just like H_RULE but attaches a semantic action
-// to the result of the parser via h_action. The action is expected to be
-// named act_<rulename>.
-//
-// H_VRULE is analogous to H_ARULE but attaches a validation via h_attr_bool.
-// The validation is expected to be named validate_<rulename>.
-//
-// H_VARULE combines H_RULE with both an action and a validation. The action is
-// attached before the validation, i.e. the validation receives as input the
-// result of the action.
-//
-// H_AVRULE is like H_VARULE but the action is attached outside the validation,
-// i.e. the validation receives the uninterpreted AST as input.
-//
-
-
-#define H_RULE(rule, def) const HParser *rule = def
-#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
-#define H_VRULE(rule, def) const HParser *rule = \
-  h_attr_bool(def, validate_ ## rule)
-#define H_VARULE(rule, def) const HParser *rule = \
-  h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule)
-#define H_AVRULE(rule, def) const HParser *rule = \
-  h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
-
-
-//
-// Pre-fab semantic actions
-//
-// A collection of generally useful semantic actions is provided.
-//
-// h_act_ignore is the action equivalent of the parser combinator h_ignore. It
-// simply causes the AST it is applied to to be replaced with NULL. This most
-// importantly causes it to be elided from the result of a surrounding
-// h_sequence.
-//
-// h_act_index is of note as it is not itself suitable to be passed to
-// h_action. It is parameterized by an index to be picked from a sequence
-// token. It must be wrapped in a proper HAction to be used. The H_ACT_APPLY
-// macro provides a concise way to define such a parameter-application wrapper.
-//
-// h_act_flatten acts on a token of possibly nested sequences by recursively
-// flattening it into a single sequence. Cf. h_seq_flatten below.
-//
-// H_ACT_APPLY implements "partial application" for semantic actions. It
-// defines a new action that supplies given parameters to a parameterized
-// action such as h_act_index.
-//
-
-const HParsedToken *h_act_ignore(const HParseResult *p);
-const HParsedToken *h_act_index(int i, const HParseResult *p);
-const HParsedToken *h_act_flatten(const HParseResult *p);
-
-// Define 'myaction' as a specialization of 'paction' by supplying the leading
-// parameters.
-#define H_ACT_APPLY(myaction, paction, ...) \
-  const HParsedToken *myaction(const HParseResult *p) { \
-    return paction(__VA_ARGS__, p); \
-  }
-
-
-//
-// Working with HParsedTokens
-//
-// The type HParsedToken represents a dynamically-typed universe of values.
-// Declared below are constructors to turn ordinary values into their
-// HParsedToken equivalents, extractors to retrieve the original values from
-// inside an HParsedToken, and functions that inspect and modify tokens of
-// sequence type directly.
-//
-// In addition, there are a number of short-hand macros that work with some
-// conventions to eliminate common boilerplate. These conventions are listed
-// below. Be sure to follow them if you want to use the respective macros.
-//
-//  * The single argument to semantic actions should be called 'p'.
-//
-//    The H_MAKE macros suppy 'p->arena' to their underlying h_make
-//    counterparts. The H_FIELD macros supply 'p->ast' to their underlying
-//    H_INDEX counterparts.
-//
-//  * For each custom token type, there should be a typedef for the
-//    corresponding value type.
-//
-//    H_CAST, H_INDEX and H_FIELD cast the void * user field of such a token to
-//    a pointer to the given type. 
-//
-//  * For each custom token type, say 'foo_t', there must be an integer
-//    constant 'TT_foo_t' to identify the token type. This constant must have a
-//    value greater or equal than TT_USER.
-//
-//    One idiom is to define an enum for all custom token types and to assign a
-//    value of TT_USER to the first element. This can be viewed as extending
-//    the HTokenType enum.
-//
-//    The H_MAKE and H_ASSERT macros derive the name of the token type constant
-//    from the given type name.
-//
-//
-// The H_ALLOC macro is useful for allocating values of custom token types.
-//
-// The H_MAKE family of macros construct tokens of a given type. The native
-// token types are indicated by a corresponding suffix such as in H_MAKE_SEQ.
-// The form with no suffix is used for custom token types. This convention is
-// also used for other macro and function families.
-//
-// The H_ASSERT family simply asserts that a given token has the expected type.
-// It mainly serves as an implementation aid for H_CAST. Of note in that regard
-// is that, unlike the standard 'assert' macro, these form _expressions_ that
-// return the value of their token argument; thus they can be used in a
-// "pass-through" fashion inside other expressions.
-//
-// The H_CAST family combines a type assertion with access to the
-// statically-typed value inside a token.
-//
-// A number of functions h_seq_* operate on and inspect sequence tokens.
-// Note that H_MAKE_SEQ takes no arguments and constructs an empty sequence.
-// Therefore there are h_seq_snoc and h_seq_append to build up sequences.
-//
-// The macro families H_FIELD and H_INDEX combine index access on a sequence
-// with a cast to the appropriate result type. H_FIELD is used to access the
-// elements of the argument token 'p' in an action. H_INDEX allows any sequence
-// token to be specified. Both macro families take an arbitrary number of index
-// arguments, giving access to elements in nested sequences by path.
-// These macros are very useful to avoid spaghetti chains of unchecked pointer
-// dereferences.
-//
-
-// Standard short-hand for arena-allocating a variable in a semantic action.
-#define H_ALLOC(TYP)  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
-
-// Token constructors...
-
-HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
-HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
-HParsedToken *h_make_bytes(HArena *arena, size_t len);
-HParsedToken *h_make_sint(HArena *arena, int64_t val);
-HParsedToken *h_make_uint(HArena *arena, uint64_t val);
-
-// Standard short-hands to make tokens in an action.
-#define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
-#define H_MAKE_SEQ()      h_make_seq(p->arena)
-#define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
-#define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
-#define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
-
-// Extract (cast) type-specific value back from HParsedTokens...
-
-// Pass-through assertion that a given token has the expected type.
-#define h_assert_type(T,P)  (assert(P->token_type == (HTokenType)T), P)
-
-// Convenience short-hand forms of h_assert_type.
-#define H_ASSERT(TYP, TOK)   h_assert_type(TT_ ## TYP, TOK)
-#define H_ASSERT_SEQ(TOK)    h_assert_type(TT_SEQUENCE, TOK)
-#define H_ASSERT_BYTES(TOK)  h_assert_type(TT_BYTES, TOK)
-#define H_ASSERT_SINT(TOK)   h_assert_type(TT_SINT, TOK)
-#define H_ASSERT_UINT(TOK)   h_assert_type(TT_UINT, TOK)
-
-// Assert expected type and return contained value.
-#define H_CAST(TYP, TOK)   ((TYP *) H_ASSERT(TYP, TOK)->user)
-#define H_CAST_SEQ(TOK)    (H_ASSERT_SEQ(TOK)->seq)
-#define H_CAST_BYTES(TOK)  (H_ASSERT_BYTES(TOK)->bytes)
-#define H_CAST_SINT(TOK)   (H_ASSERT_SINT(TOK)->sint)
-#define H_CAST_UINT(TOK)   (H_ASSERT_UINT(TOK)->uint)
-
-// Sequence access...
-
-// Return the length of a sequence.
-size_t h_seq_len(const HParsedToken *p);
-
-// Access a sequence's element array.
-HParsedToken **h_seq_elements(const HParsedToken *p);
-
-// Access a sequence element by index.
-HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
-
-// Access an element in a nested sequence by a path of indices.
-HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...);
-HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
-
-// Convenience macros combining (nested) index access and h_cast.
-#define H_INDEX(TYP, SEQ, ...)   H_CAST(TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_SEQ(SEQ, ...)    H_CAST_SEQ(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_BYTES(SEQ, ...)  H_CAST_BYTES(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_SINT(SEQ, ...)   H_CAST_SINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_UINT(SEQ, ...)   H_CAST_UINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
-
-// Standard short-hand to access and cast elements on a sequence token.
-#define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
-#define H_FIELD_SEQ(...)   H_INDEX_SEQ(p->ast, __VA_ARGS__)
-#define H_FIELD_BYTES(...) H_INDEX_BYTES(p->ast, __VA_ARGS__)
-#define H_FIELD_SINT(...)  H_INDEX_SINT(p->ast, __VA_ARGS__)
-#define H_FIELD_UINT(...)  H_INDEX_UINT(p->ast, __VA_ARGS__)
-
-// Lower-level helper for h_seq_index.
-HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
-
-// Sequence modification...
-
-// Add elements to a sequence.
-void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
-void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
-
-// XXX TODO: Remove elements from a sequence.
-
-// Flatten nested sequences into one.
-const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
-
-
-#endif
diff --git a/src/Makefile b/src/Makefile
index 47e136d..ead0516 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -38,6 +38,7 @@ HAMMER_PARTS := \
 	system_allocator.o \
 	benchmark.o \
 	compile.o \
+	glue.o \
 	$(PARSERS:%=parsers/%.o) \
 	$(BACKENDS:%=backends/%.o)
 
@@ -67,6 +68,7 @@ libhammer.a: $(HAMMER_PARTS)
 
 bitreader.o: test_suite.h
 hammer.o: hammer.h
+glue.o: hammer.h glue.h
 
 all: libhammer.a
 

From 25df438832be00de17400404efecd4b1939951b9 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 15:25:19 +0100
Subject: [PATCH 086/104] oops, moved files without adding

---
 src/glue.c | 170 ++++++++++++++++++++++++++++++++++++
 src/glue.h | 251 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 421 insertions(+)
 create mode 100644 src/glue.c
 create mode 100644 src/glue.h

diff --git a/src/glue.c b/src/glue.c
new file mode 100644
index 0000000..7f9c6fa
--- /dev/null
+++ b/src/glue.c
@@ -0,0 +1,170 @@
+#include "glue.h"
+#include "../src/internal.h"  // for h_carray_*
+
+
+// The action equivalent of h_ignore.
+const HParsedToken *h_act_ignore(const HParseResult *p)
+{
+  return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *h_act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+// Action version of h_seq_flatten.
+const HParsedToken *h_act_flatten(const HParseResult *p) {
+  return h_seq_flatten(p->arena, p->ast);
+}
+
+// Low-level helper for the h_make family.
+HParsedToken *h_make_(HArena *arena, HTokenType type)
+{
+  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
+  ret->token_type = type;
+  return ret;
+}
+
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
+{
+  assert(type >= TT_USER);
+  HParsedToken *ret = h_make_(arena, type);
+  ret->user = value;
+  return ret;
+}
+
+HParsedToken *h_make_seq(HArena *arena)
+{
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new(arena);
+  return ret;
+}
+
+HParsedToken *h_make_bytes(HArena *arena, size_t len)
+{
+  HParsedToken *ret = h_make_(arena, TT_BYTES);
+  ret->bytes.len = len;
+  ret->bytes.token = h_arena_malloc(arena, len);
+  return ret;
+}
+
+HParsedToken *h_make_sint(HArena *arena, int64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_SINT);
+  ret->sint = val;
+  return ret;
+}
+
+HParsedToken *h_make_uint(HArena *arena, uint64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_UINT);
+  ret->uint = val;
+  return ret;
+}
+
+// XXX -> internal
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
+{
+  assert(i < a->used);
+  return a->elements[i];
+}
+
+size_t h_seq_len(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->used;
+}
+
+HParsedToken **h_seq_elements(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->elements;
+}
+
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return h_carray_index(p->seq, i);
+}
+
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...)
+{
+  va_list va;
+
+  va_start(va, i);
+  HParsedToken *ret = h_seq_index_vpath(p, i, va);
+  va_end(va);
+
+  return ret;
+}
+
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va)
+{
+  HParsedToken *ret = h_seq_index(p, i);
+  int j;
+
+  while((j = va_arg(va, int)) >= 0)
+    ret = h_seq_index(p, j);
+
+  return ret;
+}
+
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+
+  h_carray_append(xs->seq, (HParsedToken *)x);
+}
+
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+  assert(ys != NULL);
+  assert(ys->token_type == TT_SEQUENCE);
+
+  for(size_t i; i<ys->seq->used; i++)
+    h_carray_append(xs->seq, ys->seq->elements[i]);
+}
+
+// Flatten nested sequences. Always returns a sequence.
+// If input element is not a sequence, returns it as a singleton sequence.
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
+{
+  assert(p != NULL);
+
+  HParsedToken *ret = h_make_seq(arena);
+  switch(p->token_type) {
+  case TT_SEQUENCE:
+    // Flatten and append all.
+    for(size_t i; i<p->seq->used; i++) {
+      h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
+    }
+    break;
+  default:
+    // Make singleton sequence.
+    h_seq_snoc(ret, p);
+    break;
+  }
+
+  return ret;
+}
diff --git a/src/glue.h b/src/glue.h
new file mode 100644
index 0000000..90944ea
--- /dev/null
+++ b/src/glue.h
@@ -0,0 +1,251 @@
+//
+// API additions for writing grammar and semantic actions more concisely
+//
+//
+// Quick Overview:
+//
+// Grammars can be succinctly specified with the family of H_RULE macros.
+// H_RULE defines a plain parser variable. H_ARULE additionally attaches a
+// semantic action; H_VRULE attaches a validation. H_AVRULE and H_VARULE
+// combine both.
+//
+// A few standard semantic actions are defined below. The H_ACT_APPLY macro
+// allows semantic actions to be defined by "partial application" of
+// a generic action to fixed paramters.
+//
+// The definition of more complex semantic actions will usually consist of
+// extracting data from the given parse tree and constructing a token of custom
+// type to represent the result. A number of functions and convenience macros
+// are provided to capture the most common cases and idioms.
+//
+// See the leading comment blocks on the sections below for more details.
+//
+
+#ifndef HAMMER_GLUE__H
+#define HAMMER_GLUE__H
+
+#include <assert.h>
+#include "hammer.h"
+
+
+//
+// Grammar specification
+//
+// H_RULE is simply a short-hand for the typical declaration and definition of
+// a parser variable. See its plain definition below. The goal is to save
+// horizontal space as well as to provide a clear and unified look together with
+// the other macro variants that stays close to an abstract PEG or BNF grammar.
+// The latter goal is more specifically enabled by H_ARULE, H_VRULE, and their
+// combinations as they allow the definition of syntax to be given without
+// intermingling it with the semantic specifications.
+//
+// H_ARULE defines a variable just like H_RULE but attaches a semantic action
+// to the result of the parser via h_action. The action is expected to be
+// named act_<rulename>.
+//
+// H_VRULE is analogous to H_ARULE but attaches a validation via h_attr_bool.
+// The validation is expected to be named validate_<rulename>.
+//
+// H_VARULE combines H_RULE with both an action and a validation. The action is
+// attached before the validation, i.e. the validation receives as input the
+// result of the action.
+//
+// H_AVRULE is like H_VARULE but the action is attached outside the validation,
+// i.e. the validation receives the uninterpreted AST as input.
+//
+
+
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+#define H_VRULE(rule, def) const HParser *rule = \
+  h_attr_bool(def, validate_ ## rule)
+#define H_VARULE(rule, def) const HParser *rule = \
+  h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule)
+#define H_AVRULE(rule, def) const HParser *rule = \
+  h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
+
+
+//
+// Pre-fab semantic actions
+//
+// A collection of generally useful semantic actions is provided.
+//
+// h_act_ignore is the action equivalent of the parser combinator h_ignore. It
+// simply causes the AST it is applied to to be replaced with NULL. This most
+// importantly causes it to be elided from the result of a surrounding
+// h_sequence.
+//
+// h_act_index is of note as it is not itself suitable to be passed to
+// h_action. It is parameterized by an index to be picked from a sequence
+// token. It must be wrapped in a proper HAction to be used. The H_ACT_APPLY
+// macro provides a concise way to define such a parameter-application wrapper.
+//
+// h_act_flatten acts on a token of possibly nested sequences by recursively
+// flattening it into a single sequence. Cf. h_seq_flatten below.
+//
+// H_ACT_APPLY implements "partial application" for semantic actions. It
+// defines a new action that supplies given parameters to a parameterized
+// action such as h_act_index.
+//
+
+const HParsedToken *h_act_ignore(const HParseResult *p);
+const HParsedToken *h_act_index(int i, const HParseResult *p);
+const HParsedToken *h_act_flatten(const HParseResult *p);
+
+// Define 'myaction' as a specialization of 'paction' by supplying the leading
+// parameters.
+#define H_ACT_APPLY(myaction, paction, ...) \
+  const HParsedToken *myaction(const HParseResult *p) { \
+    return paction(__VA_ARGS__, p); \
+  }
+
+
+//
+// Working with HParsedTokens
+//
+// The type HParsedToken represents a dynamically-typed universe of values.
+// Declared below are constructors to turn ordinary values into their
+// HParsedToken equivalents, extractors to retrieve the original values from
+// inside an HParsedToken, and functions that inspect and modify tokens of
+// sequence type directly.
+//
+// In addition, there are a number of short-hand macros that work with some
+// conventions to eliminate common boilerplate. These conventions are listed
+// below. Be sure to follow them if you want to use the respective macros.
+//
+//  * The single argument to semantic actions should be called 'p'.
+//
+//    The H_MAKE macros suppy 'p->arena' to their underlying h_make
+//    counterparts. The H_FIELD macros supply 'p->ast' to their underlying
+//    H_INDEX counterparts.
+//
+//  * For each custom token type, there should be a typedef for the
+//    corresponding value type.
+//
+//    H_CAST, H_INDEX and H_FIELD cast the void * user field of such a token to
+//    a pointer to the given type. 
+//
+//  * For each custom token type, say 'foo_t', there must be an integer
+//    constant 'TT_foo_t' to identify the token type. This constant must have a
+//    value greater or equal than TT_USER.
+//
+//    One idiom is to define an enum for all custom token types and to assign a
+//    value of TT_USER to the first element. This can be viewed as extending
+//    the HTokenType enum.
+//
+//    The H_MAKE and H_ASSERT macros derive the name of the token type constant
+//    from the given type name.
+//
+//
+// The H_ALLOC macro is useful for allocating values of custom token types.
+//
+// The H_MAKE family of macros construct tokens of a given type. The native
+// token types are indicated by a corresponding suffix such as in H_MAKE_SEQ.
+// The form with no suffix is used for custom token types. This convention is
+// also used for other macro and function families.
+//
+// The H_ASSERT family simply asserts that a given token has the expected type.
+// It mainly serves as an implementation aid for H_CAST. Of note in that regard
+// is that, unlike the standard 'assert' macro, these form _expressions_ that
+// return the value of their token argument; thus they can be used in a
+// "pass-through" fashion inside other expressions.
+//
+// The H_CAST family combines a type assertion with access to the
+// statically-typed value inside a token.
+//
+// A number of functions h_seq_* operate on and inspect sequence tokens.
+// Note that H_MAKE_SEQ takes no arguments and constructs an empty sequence.
+// Therefore there are h_seq_snoc and h_seq_append to build up sequences.
+//
+// The macro families H_FIELD and H_INDEX combine index access on a sequence
+// with a cast to the appropriate result type. H_FIELD is used to access the
+// elements of the argument token 'p' in an action. H_INDEX allows any sequence
+// token to be specified. Both macro families take an arbitrary number of index
+// arguments, giving access to elements in nested sequences by path.
+// These macros are very useful to avoid spaghetti chains of unchecked pointer
+// dereferences.
+//
+
+// Standard short-hand for arena-allocating a variable in a semantic action.
+#define H_ALLOC(TYP)  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
+
+// Token constructors...
+
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
+HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
+HParsedToken *h_make_bytes(HArena *arena, size_t len);
+HParsedToken *h_make_sint(HArena *arena, int64_t val);
+HParsedToken *h_make_uint(HArena *arena, uint64_t val);
+
+// Standard short-hands to make tokens in an action.
+#define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
+#define H_MAKE_SEQ()      h_make_seq(p->arena)
+#define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
+#define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
+#define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
+
+// Extract (cast) type-specific value back from HParsedTokens...
+
+// Pass-through assertion that a given token has the expected type.
+#define h_assert_type(T,P)  (assert(P->token_type == (HTokenType)T), P)
+
+// Convenience short-hand forms of h_assert_type.
+#define H_ASSERT(TYP, TOK)   h_assert_type(TT_ ## TYP, TOK)
+#define H_ASSERT_SEQ(TOK)    h_assert_type(TT_SEQUENCE, TOK)
+#define H_ASSERT_BYTES(TOK)  h_assert_type(TT_BYTES, TOK)
+#define H_ASSERT_SINT(TOK)   h_assert_type(TT_SINT, TOK)
+#define H_ASSERT_UINT(TOK)   h_assert_type(TT_UINT, TOK)
+
+// Assert expected type and return contained value.
+#define H_CAST(TYP, TOK)   ((TYP *) H_ASSERT(TYP, TOK)->user)
+#define H_CAST_SEQ(TOK)    (H_ASSERT_SEQ(TOK)->seq)
+#define H_CAST_BYTES(TOK)  (H_ASSERT_BYTES(TOK)->bytes)
+#define H_CAST_SINT(TOK)   (H_ASSERT_SINT(TOK)->sint)
+#define H_CAST_UINT(TOK)   (H_ASSERT_UINT(TOK)->uint)
+
+// Sequence access...
+
+// Return the length of a sequence.
+size_t h_seq_len(const HParsedToken *p);
+
+// Access a sequence's element array.
+HParsedToken **h_seq_elements(const HParsedToken *p);
+
+// Access a sequence element by index.
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
+
+// Access an element in a nested sequence by a path of indices.
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...);
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
+
+// Convenience macros combining (nested) index access and h_cast.
+#define H_INDEX(TYP, SEQ, ...)   H_CAST(TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SEQ(SEQ, ...)    H_CAST_SEQ(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_BYTES(SEQ, ...)  H_CAST_BYTES(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SINT(SEQ, ...)   H_CAST_SINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_UINT(SEQ, ...)   H_CAST_UINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
+
+// Standard short-hand to access and cast elements on a sequence token.
+#define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
+#define H_FIELD_SEQ(...)   H_INDEX_SEQ(p->ast, __VA_ARGS__)
+#define H_FIELD_BYTES(...) H_INDEX_BYTES(p->ast, __VA_ARGS__)
+#define H_FIELD_SINT(...)  H_INDEX_SINT(p->ast, __VA_ARGS__)
+#define H_FIELD_UINT(...)  H_INDEX_UINT(p->ast, __VA_ARGS__)
+
+// Lower-level helper for h_seq_index.
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
+
+// Sequence modification...
+
+// Add elements to a sequence.
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
+
+// XXX TODO: Remove elements from a sequence.
+
+// Flatten nested sequences into one.
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
+
+
+#endif

From b06a98ce2600727fe18bcb19f402ab355d5f6467 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 15:47:18 +0100
Subject: [PATCH 087/104] fix linking of base64 example

---
 examples/Makefile | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 10ba9f1..663a214 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -22,12 +22,20 @@ dns: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
 dns: dns.o rr.o dns_common.o
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
 
-dns.o: ../src/hammer.h dns_common.h
-rr.o: ../src/hammer.h rr.h dns_common.h
-dns_common.o: ../src/hammer.h dns_common.h
+dns.o: ../src/hammer.h dns_common.h ../src/glue.h
+rr.o: ../src/hammer.h rr.h dns_common.h ../src/glue.h
+dns_common.o: ../src/hammer.h dns_common.h ../src/glue.h
 
-base64%: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
-base64%: base64%.o
+base64: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
+base64: base64.o
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
 
-base64%.o: ../src/hammer.h
+base64_sem1: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
+base64_sem1: base64_sem1.o
+	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
+
+base64_sem2: LDFLAGS:=-L../src -lhammer $(LDFLAGS)
+base64_sem2: base64_sem2.o
+	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS)
+
+base64%.o: ../src/hammer.h ../src/glue.h

From 38ddcc5ab4427bafb3a49d28e86141dba5991897 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 15:47:53 +0100
Subject: [PATCH 088/104] add semantic base64 examples to gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index db2ee3a..7f4d7d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,8 @@ src/test_suite
 lib/hush
 examples/dns
 examples/base64
+examples/base64_sem1
+examples/base64_sem2
 TAGS
 *.swp
 *.swo

From a5c579c23da8f250889c0afee438800dbe56c899 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 15:53:43 +0100
Subject: [PATCH 089/104] switch semantic base64 examples to pre-fab actions
 provided by glue.h

---
 examples/base64_sem1.c | 64 ++++++------------------------------------
 examples/base64_sem2.c | 45 +++--------------------------
 2 files changed, 12 insertions(+), 97 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 8de31db..9d1012f 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -13,50 +13,11 @@
 // base64_sem2.c for an alternative approach using a single top-level action.
 
 #include "../src/hammer.h"
+#include "../src/glue.h"
 #include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
 
 
-#define H_RULE(rule, def) const HParser *rule = def
-#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
-
-
-///
-// Semantic action helpers.
-// These might be candidates for inclusion in the library.
-///
-
-// The action equivalent of h_ignore.
-const HParsedToken *act_ignore(const HParseResult *p)
-{
-    return NULL;
-}
-
-// Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *act_index(int i, const HParseResult *p)
-{
-    if(!p) return NULL;
-
-    const HParsedToken *tok = p->ast;
-
-    if(!tok || tok->token_type != TT_SEQUENCE)
-        return NULL;
-
-    const HCountedArray *seq = tok->seq;
-    size_t n = seq->used;
-
-    if(i<0 || (size_t)i>=n)
-        return NULL;
-    else
-        return tok->seq->elements[i];
-}
-
-const HParsedToken *act_index0(const HParseResult *p)
-{
-    return act_index(0, p);
-}
-
-
 ///
 // Semantic actions for the grammar below, each corresponds to an "ARULE".
 // They must be named act_<rulename>.
@@ -84,11 +45,13 @@ const HParsedToken *act_bsfdig(const HParseResult *p)
     return res;
 }
 
+H_ACT_APPLY(act_index0, h_act_index, 0);
+
 #define act_bsfdig_4bit  act_bsfdig
 #define act_bsfdig_2bit  act_bsfdig
 
-#define act_equals       act_ignore
-#define act_ws           act_ignore
+#define act_equals       h_act_ignore
+#define act_ws           h_act_ignore
 
 #define act_document     act_index0
 
@@ -124,20 +87,9 @@ const HParsedToken *act_base64_n(int n, const HParseResult *p)
     return res;
 }
 
-const HParsedToken *act_base64_3(const HParseResult *p)
-{
-    return act_base64_n(3, p);
-}
-
-const HParsedToken *act_base64_2(const HParseResult *p)
-{
-    return act_base64_n(2, p);
-}
-
-const HParsedToken *act_base64_1(const HParseResult *p)
-{
-    return act_base64_n(1, p);
-}
+H_ACT_APPLY(act_base64_3, act_base64_n, 3);
+H_ACT_APPLY(act_base64_2, act_base64_n, 2);
+H_ACT_APPLY(act_base64_1, act_base64_n, 1);
 
 // Helper to concatenate two arrays.
 void carray_concat(HCountedArray *a, const HCountedArray *b)
diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
index 11b0660..4b886c6 100644
--- a/examples/base64_sem2.c
+++ b/examples/base64_sem2.c
@@ -14,50 +14,11 @@
 // transformation.
 
 #include "../src/hammer.h"
+#include "../src/glue.h"
 #include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
 
 
-#define H_RULE(rule, def) const HParser *rule = def
-#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
-
-
-///
-// Semantic action helpers.
-// These might be candidates for inclusion in the library.
-///
-
-// The action equivalent of h_ignore.
-const HParsedToken *act_ignore(const HParseResult *p)
-{
-    return NULL;
-}
-
-// Helper to build HAction's that pick one index out of a sequence.
-const HParsedToken *act_index(int i, const HParseResult *p)
-{
-    if(!p) return NULL;
-
-    const HParsedToken *tok = p->ast;
-
-    if(!tok || tok->token_type != TT_SEQUENCE)
-        return NULL;
-
-    const HCountedArray *seq = tok->seq;
-    size_t n = seq->used;
-
-    if(i<0 || (size_t)i>=n)
-        return NULL;
-    else
-        return tok->seq->elements[i];
-}
-
-const HParsedToken *act_index0(const HParseResult *p)
-{
-    return act_index(0, p);
-}
-
-
 ///
 // Semantic actions for the grammar below, each corresponds to an "ARULE".
 // They must be named act_<rulename>.
@@ -150,7 +111,9 @@ const HParsedToken *act_base64(const HParseResult *p)
     return res;
 }
 
-#define act_ws           act_ignore
+H_ACT_APPLY(act_index0, h_act_index, 0);
+
+#define act_ws           h_act_ignore
 #define act_document     act_index0
 
 

From b83be8472e044b012ac3812daf76c37ab41a3a85 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:29:54 +0100
Subject: [PATCH 090/104] fix counter init in h_seq_append

---
 src/glue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/glue.c b/src/glue.c
index 7f9c6fa..2735e48 100644
--- a/src/glue.c
+++ b/src/glue.c
@@ -142,7 +142,7 @@ void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
   assert(ys != NULL);
   assert(ys->token_type == TT_SEQUENCE);
 
-  for(size_t i; i<ys->seq->used; i++)
+  for(size_t i=0; i<ys->seq->used; i++)
     h_carray_append(xs->seq, ys->seq->elements[i]);
 }
 

From af23f3bbf31856fbd79359db62697d82e6457f66 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:45:29 +0100
Subject: [PATCH 091/104] add more glue functions in base64_sem1

---
 examples/base64_sem1.c | 37 ++++++++++---------------------------
 1 file changed, 10 insertions(+), 27 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 9d1012f..1c318e3 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -25,12 +25,10 @@
 
 const HParsedToken *act_bsfdig(const HParseResult *p)
 {
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
+    HParsedToken *res = H_MAKE_UINT(0);
 
-    assert(p->ast->token_type == TT_UINT);
-    uint8_t c = p->ast->uint;
+    uint8_t c = H_CAST_UINT(p->ast);
 
-    res->token_type = TT_UINT;
     if(c >= 0x40 && c <= 0x5A) // A-Z
         res->uint = c - 0x41;
     else if(c >= 0x60 && c <= 0x7A) // a-z
@@ -58,13 +56,11 @@ H_ACT_APPLY(act_index0, h_act_index, 0);
 // General-form action to turn a block of base64 digits into bytes.
 const HParsedToken *act_base64_n(int n, const HParseResult *p)
 {
-    assert(p->ast->token_type == TT_SEQUENCE);
-
     HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
     res->token_type = TT_SEQUENCE;
     res->seq = h_carray_new_sized(p->arena, n);
 
-    HParsedToken **digits = p->ast->seq->elements;
+    HParsedToken **digits = h_seq_elements(p->ast);
 
     uint32_t x = 0;
     int bits = 0;
@@ -75,9 +71,7 @@ const HParsedToken *act_base64_n(int n, const HParseResult *p)
     x >>= bits%8;   // align, i.e. cut off extra bits
 
     for(int i=0; i<n; i++) {
-        HParsedToken *item = h_arena_malloc(p->arena, sizeof(HParsedToken));
-        item->token_type = TT_UINT;
-        item->uint = x & 0xFF;
+        HParsedToken *item = H_MAKE_UINT(x & 0xFF);
 
         res->seq->elements[n-1-i] = item;   // output the last byte and
         x >>= 8;                            // discard it
@@ -91,34 +85,23 @@ H_ACT_APPLY(act_base64_3, act_base64_n, 3);
 H_ACT_APPLY(act_base64_2, act_base64_n, 2);
 H_ACT_APPLY(act_base64_1, act_base64_n, 1);
 
-// Helper to concatenate two arrays.
-void carray_concat(HCountedArray *a, const HCountedArray *b)
-{
-    for(size_t i=0; i<b->used; i++)
-        h_carray_append(a, b->elements[i]);
-}
-
 const HParsedToken *act_base64(const HParseResult *p)
 {
     assert(p->ast->token_type == TT_SEQUENCE);
     assert(p->ast->seq->used == 2);
     assert(p->ast->seq->elements[0]->token_type == TT_SEQUENCE);
 
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new(p->arena);
+    HParsedToken *res = H_MAKE_SEQ();
 
     // concatenate base64_3 blocks
-    HCountedArray *seq = p->ast->seq->elements[0]->seq;
-    for(size_t i=0; i<seq->used; i++) {
-        assert(seq->elements[i]->token_type == TT_SEQUENCE);
-        carray_concat(res->seq, seq->elements[i]->seq);
-    }
+    HCountedArray *seq = H_FIELD_SEQ(0);
+    for(size_t i=0; i<seq->used; i++)
+        h_seq_append(res, seq->elements[i]);
 
     // append one trailing base64_2 or _1 block
-    const HParsedToken *tok = p->ast->seq->elements[1];
+    const HParsedToken *tok = h_seq_index(p->ast, 1);
     if(tok->token_type == TT_SEQUENCE)
-        carray_concat(res->seq, tok->seq);
+        h_seq_append(res, tok);
 
     return res;
 }

From 662357ccb43afceb55bc63aca287984a1ed59bbf Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:47:36 +0100
Subject: [PATCH 092/104] add glue helper to construct sized sequences

---
 src/glue.c | 7 +++++++
 src/glue.h | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/src/glue.c b/src/glue.c
index 2735e48..f1e086a 100644
--- a/src/glue.c
+++ b/src/glue.c
@@ -55,6 +55,13 @@ HParsedToken *h_make_seq(HArena *arena)
   return ret;
 }
 
+HParsedToken *h_make_seqn(HArena *arena, size_t n)
+{
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new_sized(arena, n);
+  return ret;
+}
+
 HParsedToken *h_make_bytes(HArena *arena, size_t len)
 {
   HParsedToken *ret = h_make_(arena, TT_BYTES);
diff --git a/src/glue.h b/src/glue.h
index 90944ea..3125ae0 100644
--- a/src/glue.h
+++ b/src/glue.h
@@ -173,6 +173,7 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
+HParsedToken *h_make_seqn(HArena *arena, size_t n);  // Makes empty sequence of expected size n.
 HParsedToken *h_make_bytes(HArena *arena, size_t len);
 HParsedToken *h_make_sint(HArena *arena, int64_t val);
 HParsedToken *h_make_uint(HArena *arena, uint64_t val);
@@ -180,6 +181,7 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 // Standard short-hands to make tokens in an action.
 #define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
 #define H_MAKE_SEQ()      h_make_seq(p->arena)
+#define H_MAKE_SEQN(N)    h_make_seqn(p->arena, N)
 #define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
 #define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
 #define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)

From d9c2c921c431c609003d2ba3e7f309ce3709aeec Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:48:35 +0100
Subject: [PATCH 093/104] use H_MAKE_SEQN in base64_sem1

---
 examples/base64_sem1.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 1c318e3..8074352 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -56,9 +56,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0);
 // General-form action to turn a block of base64 digits into bytes.
 const HParsedToken *act_base64_n(int n, const HParseResult *p)
 {
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new_sized(p->arena, n);
+    HParsedToken *res = H_MAKE_SEQN(n);
 
     HParsedToken **digits = h_seq_elements(p->ast);
 

From 51b90828379cc7840901e2e74532bc188658d1f6 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:49:25 +0100
Subject: [PATCH 094/104] remove obsolete dependency on internal.h

---
 examples/base64_sem1.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 8074352..f2a3e82 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -14,7 +14,6 @@
 
 #include "../src/hammer.h"
 #include "../src/glue.h"
-#include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
 
 

From f1f7c4f488d8ba0b43ad13e7186730afe4a37ad9 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 17:00:04 +0100
Subject: [PATCH 095/104] remove dependency on internal.h for base64_sem2

---
 examples/base64_sem2.c | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
index 4b886c6..32afe5b 100644
--- a/examples/base64_sem2.c
+++ b/examples/base64_sem2.c
@@ -15,7 +15,6 @@
 
 #include "../src/hammer.h"
 #include "../src/glue.h"
-#include "../src/internal.h"    // for h_carray functions (XXX ?!)
 #include <assert.h>
 
 
@@ -47,13 +46,7 @@ uint8_t bsfdig_value(const HParsedToken *p)
 }
 
 // helper: append a byte value to a sequence
-void seq_append_byte(HCountedArray *a, uint8_t b)
-{
-    HParsedToken *item = h_arena_malloc(a->arena, sizeof(HParsedToken));
-    item->token_type = TT_UINT;
-    item->uint = b;
-    h_carray_append(a, item);
-}
+#define seq_append_byte(res, b) h_seq_snoc(res, H_MAKE_UINT(b))
 
 const HParsedToken *act_base64(const HParseResult *p)
 {
@@ -75,9 +68,7 @@ const HParsedToken *act_base64(const HParseResult *p)
         b64_1 = NULL;
 
     // allocate result sequence
-    HParsedToken *res = h_arena_malloc(p->arena, sizeof(HParsedToken));
-    res->token_type = TT_SEQUENCE;
-    res->seq = h_carray_new(p->arena);
+    HParsedToken *res = H_MAKE_SEQ();
 
     // concatenate base64_3 blocks
     for(size_t i=0; i<b64_3->seq->used; i++) {
@@ -88,9 +79,9 @@ const HParsedToken *act_base64(const HParseResult *p)
         x <<= 6; x |= bsfdig_value(digits[1]);
         x <<= 6; x |= bsfdig_value(digits[2]);
         x <<= 6; x |= bsfdig_value(digits[3]);
-        seq_append_byte(res->seq, (x >> 16) & 0xFF);
-        seq_append_byte(res->seq, (x >> 8) & 0xFF);
-        seq_append_byte(res->seq, x & 0xFF);
+        seq_append_byte(res, (x >> 16) & 0xFF);
+        seq_append_byte(res, (x >> 8) & 0xFF);
+        seq_append_byte(res, x & 0xFF);
     }
 
     // append one trailing base64_2 or _1 block
@@ -99,13 +90,13 @@ const HParsedToken *act_base64(const HParseResult *p)
         uint32_t x = bsfdig_value(digits[0]);
         x <<= 6; x |= bsfdig_value(digits[1]);
         x <<= 6; x |= bsfdig_value(digits[2]);
-        seq_append_byte(res->seq, (x >> 10) & 0xFF);
-        seq_append_byte(res->seq, (x >> 2) & 0xFF);
+        seq_append_byte(res, (x >> 10) & 0xFF);
+        seq_append_byte(res, (x >> 2) & 0xFF);
     } else if(b64_1) {
         HParsedToken **digits = b64_1->seq->elements;
         uint32_t x = bsfdig_value(digits[0]);
         x <<= 6; x |= bsfdig_value(digits[1]);
-        seq_append_byte(res->seq, (x >> 4) & 0xFF);
+        seq_append_byte(res, (x >> 4) & 0xFF);
     }
 
     return res;

From 866e66ccf9538e3c19d03739e8c804ab2fc4029e Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:29:54 +0100
Subject: [PATCH 096/104] fix counter init in h_seq_append

---
 src/glue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/glue.c b/src/glue.c
index 7f9c6fa..2735e48 100644
--- a/src/glue.c
+++ b/src/glue.c
@@ -142,7 +142,7 @@ void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
   assert(ys != NULL);
   assert(ys->token_type == TT_SEQUENCE);
 
-  for(size_t i; i<ys->seq->used; i++)
+  for(size_t i=0; i<ys->seq->used; i++)
     h_carray_append(xs->seq, ys->seq->elements[i]);
 }
 

From 0cfec9781ab7dfa0718b04bd5a34fd3d2eb72394 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 17 Feb 2013 16:47:36 +0100
Subject: [PATCH 097/104] add glue helper to construct sized sequences

---
 src/glue.c | 7 +++++++
 src/glue.h | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/src/glue.c b/src/glue.c
index 2735e48..f1e086a 100644
--- a/src/glue.c
+++ b/src/glue.c
@@ -55,6 +55,13 @@ HParsedToken *h_make_seq(HArena *arena)
   return ret;
 }
 
+HParsedToken *h_make_seqn(HArena *arena, size_t n)
+{
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new_sized(arena, n);
+  return ret;
+}
+
 HParsedToken *h_make_bytes(HArena *arena, size_t len)
 {
   HParsedToken *ret = h_make_(arena, TT_BYTES);
diff --git a/src/glue.h b/src/glue.h
index 90944ea..3125ae0 100644
--- a/src/glue.h
+++ b/src/glue.h
@@ -173,6 +173,7 @@ const HParsedToken *h_act_flatten(const HParseResult *p);
 
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
+HParsedToken *h_make_seqn(HArena *arena, size_t n);  // Makes empty sequence of expected size n.
 HParsedToken *h_make_bytes(HArena *arena, size_t len);
 HParsedToken *h_make_sint(HArena *arena, int64_t val);
 HParsedToken *h_make_uint(HArena *arena, uint64_t val);
@@ -180,6 +181,7 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 // Standard short-hands to make tokens in an action.
 #define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
 #define H_MAKE_SEQ()      h_make_seq(p->arena)
+#define H_MAKE_SEQN(N)    h_make_seqn(p->arena, N)
 #define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
 #define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
 #define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)

From 928391d547851f588588394ff83c6272b78d7e71 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Sun, 17 Feb 2013 19:57:12 -0500
Subject: [PATCH 098/104] Added irc channel to README.md

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index d88a153..ffd5cff 100644
--- a/README.md
+++ b/README.md
@@ -48,3 +48,7 @@ Examples
 The `examples/` directory contains some simple examples, currently including:
 * base64
 * DNS
+
+Community
+=========
+Please join us at `#hammer` on `irc.upstandinghackers.com` if you have any questions or just want to talk about parsing. 

From 3a0da2fd844c6f60cfdfcd0102846844f92630e9 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <clonearmy@gmail.com>
Date: Mon, 18 Feb 2013 20:03:19 -0500
Subject: [PATCH 099/104] Added contact info

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ffd5cff..e008b12 100644
--- a/README.md
+++ b/README.md
@@ -51,4 +51,8 @@ The `examples/` directory contains some simple examples, currently including:
 
 Community
 =========
-Please join us at `#hammer` on `irc.upstandinghackers.com` if you have any questions or just want to talk about parsing. 
+Please join us at `#hammer` on `irc.upstandinghackers.com` if you have any questions or just want to talk about parsing.
+
+Contact
+=======
+You can also email us at <hammer@upstandinghackers.com>.

From 4041a5865cb40c20160b23ad0e6757f3fc760754 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sat, 13 Apr 2013 14:42:36 +0200
Subject: [PATCH 100/104] pass rr by reference to set_rdata

---
 examples/dns.c | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/examples/dns.c b/examples/dns.c
index 6c572a0..7887ba6 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -49,40 +49,40 @@ bool validate_message(HParseResult *p) {
 ///
 
 // Helper: Parse and pack the RDATA field of a Resource Record.
-void set_rdata(struct dns_rr rr, HCountedArray *rdata) {
+void set_rdata(struct dns_rr *rr, HCountedArray *rdata) {
   uint8_t *data = h_arena_malloc(rdata->arena, sizeof(uint8_t)*rdata->used);
   for (size_t i=0; i<rdata->used; ++i)
     data[i] = H_CAST_UINT(rdata->elements[i]);
 
   // Parse RDATA if possible.
   const HParseResult *p = NULL;
-  const HParser *parser = init_rdata(rr.type);
+  const HParser *parser = init_rdata(rr->type);
   if (parser)
     p = h_parse(parser, (const uint8_t*)data, rdata->used);
 
   // If the RR doesn't parse, set its type to 0.
   if (!p) 
-    rr.type = 0;
+    rr->type = 0;
 
   // Pack the parsed rdata into rr.
-  switch(rr.type) {
-  case 1:  rr.a     = H_CAST_UINT(p->ast);             break;
-  case 2:  rr.ns    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 3:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 4:  rr.md    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 5:  rr.cname = *H_CAST(dns_domain_t,   p->ast); break;
-  case 6:  rr.soa   = *H_CAST(dns_rr_soa_t,   p->ast); break;
-  case 7:  rr.mb    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 8:  rr.mg    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 9:  rr.mr    = *H_CAST(dns_domain_t,   p->ast); break;
-  case 10: rr.null  = *H_CAST(dns_rr_null_t,  p->ast); break;
-  case 11: rr.wks   = *H_CAST(dns_rr_wks_t,   p->ast); break;
-  case 12: rr.ptr   = *H_CAST(dns_domain_t,   p->ast); break;
-  case 13: rr.hinfo = *H_CAST(dns_rr_hinfo_t, p->ast); break;
-  case 14: rr.minfo = *H_CAST(dns_rr_minfo_t, p->ast); break;
-  case 15: rr.mx    = *H_CAST(dns_rr_mx_t,    p->ast); break;
-  case 16: rr.txt   = *H_CAST(dns_rr_txt_t,   p->ast); break;
-  default:                                             break;
+  switch(rr->type) {
+  case 1:  rr->a     = H_CAST_UINT(p->ast);             break;
+  case 2:  rr->ns    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 3:  rr->md    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 4:  rr->md    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 5:  rr->cname = *H_CAST(dns_domain_t,   p->ast); break;
+  case 6:  rr->soa   = *H_CAST(dns_rr_soa_t,   p->ast); break;
+  case 7:  rr->mb    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 8:  rr->mg    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 9:  rr->mr    = *H_CAST(dns_domain_t,   p->ast); break;
+  case 10: rr->null  = *H_CAST(dns_rr_null_t,  p->ast); break;
+  case 11: rr->wks   = *H_CAST(dns_rr_wks_t,   p->ast); break;
+  case 12: rr->ptr   = *H_CAST(dns_domain_t,   p->ast); break;
+  case 13: rr->hinfo = *H_CAST(dns_rr_hinfo_t, p->ast); break;
+  case 14: rr->minfo = *H_CAST(dns_rr_minfo_t, p->ast); break;
+  case 15: rr->mx    = *H_CAST(dns_rr_mx_t,    p->ast); break;
+  case 16: rr->txt   = *H_CAST(dns_rr_txt_t,   p->ast); break;
+  default:                                              break;
   }
 }
 
@@ -131,7 +131,7 @@ const HParsedToken* act_rr(const HParseResult *p) {
   rr->rdlength = H_FIELD_SEQ(4)->used;
 
   // Parse and pack RDATA.
-  set_rdata(*rr, H_FIELD_SEQ(4));
+  set_rdata(rr, H_FIELD_SEQ(4));
 
   return H_MAKE(dns_rr_t, rr);
 }

From 1529c0641e5f2599739373fa20f49632b85e4746 Mon Sep 17 00:00:00 2001
From: aegis <github@xn--g-3fa.is>
Date: Tue, 23 Apr 2013 17:53:26 +0200
Subject: [PATCH 101/104] Added JNI example, this is what usage hsould look
 like soon.

---
 jni/Example.java                             | 49 ++++++++++++
 jni/com/upstandinghackers/hammer/Hammer.java | 81 ++++++++++----------
 2 files changed, 89 insertions(+), 41 deletions(-)
 create mode 100644 jni/Example.java

diff --git a/jni/Example.java b/jni/Example.java
new file mode 100644
index 0000000..5871d79
--- /dev/null
+++ b/jni/Example.java
@@ -0,0 +1,49 @@
+/**
+* Example JHammer usage
+*/
+public class Example
+{
+
+private HParser initParser()
+{
+    HParser digit = Hammer.chRange(0x30, 0x39);
+    HParser alpha = Hammer.choice({Hammer.chRange(0x41, 0x5a), Hammer.chRange(0x61, 0x7a)});
+    
+    HParser plus = Hammer.ch('+');
+    HParser slash = Hammer.ch('/');
+    HParser equals = Hammer.ch('=');
+    HParser bsfdig = Hammer.choice({alpha, digit, plus, slash});
+
+    byte[] AEIMQUYcgkosw048 = "AEIMQUYcgkosw048".getBytes();
+    HParser bsfdig_4bit = Hammer.in(AEIMQUYcgkosw048, AEIMQUYcgkosw048.length);
+    byte[] AQgw = "AQgw".getBytes();
+    HParser bsfdig_2bit = Hammer.in(AQgw, AQgw.length);
+    HParser base64_3 = Hammer.repeatN(bsfdig, 4);
+    HParser base64_2 = Hammer.sequence({bsfdig, bsfdig, bsfdig_4bit, equals});
+    HParser base64_1 = Hammer.sequence({bsfdig, bsfdig_2bit, equals, equals});
+    HParser base64 = Hammer.sequence({  Hammer.many(base64_3),
+                                        Hammer.optional(Hammer.choice({base64_2, base64_1}))
+                                    });
+
+    return Hammer.sequence({Hammer.whitespace(base64), Hammer.whitespace(Hammer.endP()}});
+}
+
+public static void main(String args[])
+{
+    byte[] input = "RXMgaXN0IFNwYXJnZWx6ZWl0IQo=".getBytes();
+    int length = input.length;
+    HParsedResult result = Hammer.parse(initParser(), input, length);
+    if(result == null)
+    {
+        System.out.println("FAIL");
+    }
+    else
+    {
+        System.out.println("PASS");
+        //TODO: Pretty print
+    }
+}
+
+
+
+}
diff --git a/jni/com/upstandinghackers/hammer/Hammer.java b/jni/com/upstandinghackers/hammer/Hammer.java
index 08b75b1..e7ad571 100644
--- a/jni/com/upstandinghackers/hammer/Hammer.java
+++ b/jni/com/upstandinghackers/hammer/Hammer.java
@@ -16,51 +16,50 @@ public class Hammer
         TT_SEQUENCE(16),
         TT_ERR(32),
         TT_USER(64),
-        TT_MAX(128);
 
         private int value;
         public int getValue() { return this.value; }
         private HTokenType(int value) { this.value = value; }
     }
 
-    public static native HParseResult hParse(HParser parser, byte[] input, int length);
-    public static native HParser hToken(byte[] str, int length);
-    public static native HParser hCh(byte c);
-    public static native HParser hChRange(byte from, byte to);
-    public static native HParser hIntRange(HParser p, int lower, int upper);
-    public static native HParser hBits(int len, boolean sign);
-    public static native HParser hInt64();
-    public static native HParser hInt32();
-    public static native HParser hInt16();
-    public static native HParser hInt8();
-    public static native HParser hUInt64();
-    public static native HParser hUInt32();
-    public static native HParser hUInt16();
-    public static native HParser hUInt8();
-    public static native HParser hWhitespace(HParser p);
-    public static native HParser hLeft(HParser p, HParser q);
-    public static native HParser hRight(HParser p, HParser q);
-    public static native HParser hMiddle(HParser p, HParser x, HParser q);
-    public static native HParser hAction(HParser p, HAction a);
-    public static native HParser hIn(byte[] charset, int length);
-    public static native HParser hEndP();
-    public static native HParser hNothingP();
-    public static native HParser hSequence(HParser[] parsers);
-    public static native HParser hChoice(HParser[] parsers);
-    public static native HParser hButNot(HParser p1, HParser p2);
-    public static native HParser hDifference(HParser p1, HParser p2);
-    public static native HParser hXor(HParser p1, HParser p2);
-    public static native HParser hMany(HParser p);
-    public static native HParser hMany1(HParser p);
-    public static native HParser hRepeatN(HParser p, int n);
-    public static native HParser hOptional(HParser p);
-    public static native HParser hIgnore(HParser p);
-    public static native HParser hSepBy(HParser p, HParser sep);
-    public static native HParser hSepBy1(HParser p, HParser sep);
-    public static native HParser hEpsilonP();
-    public static native HParser hLengthValue(HParser length, HParser value);
-    public static native HParser hAttrBool(HParser p, HPredicate pred);
-    public static native HParser hAnd(HParser p);
-    public static native HParser hNot(HParser p);
-    public static native HParser hIndirect();
+    public static native HParseResult parse(HParser parser, byte[] input, int length);
+    public static native HParser token(byte[] str, int length);
+    public static native HParser ch(byte c);
+    public static native HParser chRange(byte from, byte to);
+    public static native HParser intRange(HParser p, int lower, int upper);
+    public static native HParser bits(int len, boolean sign);
+    public static native HParser int64();
+    public static native HParser int32();
+    public static native HParser int16();
+    public static native HParser int8();
+    public static native HParser uInt64();
+    public static native HParser uInt32();
+    public static native HParser uInt16();
+    public static native HParser uInt8();
+    public static native HParser whitespace(HParser p);
+    public static native HParser left(HParser p, HParser q);
+    public static native HParser right(HParser p, HParser q);
+    public static native HParser middle(HParser p, HParser x, HParser q);
+    public static native HParser action(HParser p, HAction a);
+    public static native HParser in(byte[] charset, int length);
+    public static native HParser endP();
+    public static native HParser nothingP();
+    public static native HParser sequence(HParser[] parsers);
+    public static native HParser choice(HParser[] parsers);
+    public static native HParser butNot(HParser p1, HParser p2);
+    public static native HParser difference(HParser p1, HParser p2);
+    public static native HParser xor(HParser p1, HParser p2);
+    public static native HParser many(HParser p);
+    public static native HParser many1(HParser p);
+    public static native HParser repeatN(HParser p, int n);
+    public static native HParser optional(HParser p);
+    public static native HParser ignore(HParser p);
+    public static native HParser sepBy(HParser p, HParser sep);
+    public static native HParser sepBy1(HParser p, HParser sep);
+    public static native HParser epsilonP();
+    public static native HParser lengthValue(HParser length, HParser value);
+    public static native HParser attrBool(HParser p, HPredicate pred);
+    public static native HParser and(HParser p);
+    public static native HParser not(HParser p);
+    public static native HParser indirect();
 }

From 106b8bb6a752d3aa064350806b19bd30241049c0 Mon Sep 17 00:00:00 2001
From: aegis <github@xn--g-3fa.is>
Date: Fri, 10 May 2013 12:05:20 +0200
Subject: [PATCH 102/104] JNI ready for testing. -fPIC enabled for objects in
 src and jni.

---
 Makefile                                      |   2 +-
 common.mk                                     |   2 +-
 jni/Example.java                              |  97 +++--
 jni/Makefile                                  |  42 +++
 jni/NOTES                                     |  30 +-
 jni/com/upstandinghackers/hammer/Action.java  |   8 +
 jni/com/upstandinghackers/hammer/HAction.java |   8 -
 .../hammer/HParseResult.java                  |   9 -
 .../hammer/HParsedToken.java                  |  28 --
 jni/com/upstandinghackers/hammer/HParser.java |   6 -
 .../upstandinghackers/hammer/HPredicate.java  |   6 -
 jni/com/upstandinghackers/hammer/Hammer.java  | 109 +++---
 .../upstandinghackers/hammer/ParseResult.java |  15 +
 .../upstandinghackers/hammer/ParsedToken.java |  40 +++
 jni/com/upstandinghackers/hammer/Parser.java  |  11 +
 .../upstandinghackers/hammer/Predicate.java   |   6 +
 jni/com_upstandinghackers_hammer_Hammer.c     | 335 ++++++++++++++++++
 ...com_upstandinghackers_hammer_ParseResult.c |  45 +++
 ...com_upstandinghackers_hammer_ParsedToken.c | 195 ++++++++++
 jni/com_upstandinghackers_hammer_Parser.c     |  15 +
 jni/jhammer.h                                 |  31 ++
 src/Makefile                                  |   2 +
 src/hammer.h                                  |   8 +-
 src/parsers/choice.c                          |  24 ++
 src/parsers/sequence.c                        |  25 ++
 25 files changed, 954 insertions(+), 145 deletions(-)
 create mode 100644 jni/Makefile
 create mode 100644 jni/com/upstandinghackers/hammer/Action.java
 delete mode 100644 jni/com/upstandinghackers/hammer/HAction.java
 delete mode 100644 jni/com/upstandinghackers/hammer/HParseResult.java
 delete mode 100644 jni/com/upstandinghackers/hammer/HParsedToken.java
 delete mode 100644 jni/com/upstandinghackers/hammer/HParser.java
 delete mode 100644 jni/com/upstandinghackers/hammer/HPredicate.java
 create mode 100644 jni/com/upstandinghackers/hammer/ParseResult.java
 create mode 100644 jni/com/upstandinghackers/hammer/ParsedToken.java
 create mode 100644 jni/com/upstandinghackers/hammer/Parser.java
 create mode 100644 jni/com/upstandinghackers/hammer/Predicate.java
 create mode 100644 jni/com_upstandinghackers_hammer_Hammer.c
 create mode 100644 jni/com_upstandinghackers_hammer_ParseResult.c
 create mode 100644 jni/com_upstandinghackers_hammer_ParsedToken.c
 create mode 100644 jni/com_upstandinghackers_hammer_Parser.c
 create mode 100644 jni/jhammer.h

diff --git a/Makefile b/Makefile
index bd383a2..a70cc40 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@
 # and kick off a recursive make
 # Also, "make src/all" turns into "make -C src all"
 
-SUBDIRS = src examples
+SUBDIRS = src examples jni
 
 include config.mk
 
diff --git a/common.mk b/common.mk
index a57429d..0259d5f 100644
--- a/common.mk
+++ b/common.mk
@@ -8,7 +8,7 @@ include $(TOPLEVEL)/config.mk
 TEST_CFLAGS = $(shell pkg-config --cflags glib-2.0) -DINCLUDE_TESTS
 TEST_LDFLAGS = $(shell pkg-config --libs glib-2.0)
 
-CFLAGS := -std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes
+CFLAGS := -std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -g
 LDFLAGS :=
 
 CC ?= gcc
diff --git a/jni/Example.java b/jni/Example.java
index 5871d79..8cfa724 100644
--- a/jni/Example.java
+++ b/jni/Example.java
@@ -1,38 +1,18 @@
+import com.upstandinghackers.hammer.*;
+import java.util.Arrays;
 /**
 * Example JHammer usage
 */
+
 public class Example
 {
 
-private HParser initParser()
-{
-    HParser digit = Hammer.chRange(0x30, 0x39);
-    HParser alpha = Hammer.choice({Hammer.chRange(0x41, 0x5a), Hammer.chRange(0x61, 0x7a)});
-    
-    HParser plus = Hammer.ch('+');
-    HParser slash = Hammer.ch('/');
-    HParser equals = Hammer.ch('=');
-    HParser bsfdig = Hammer.choice({alpha, digit, plus, slash});
-
-    byte[] AEIMQUYcgkosw048 = "AEIMQUYcgkosw048".getBytes();
-    HParser bsfdig_4bit = Hammer.in(AEIMQUYcgkosw048, AEIMQUYcgkosw048.length);
-    byte[] AQgw = "AQgw".getBytes();
-    HParser bsfdig_2bit = Hammer.in(AQgw, AQgw.length);
-    HParser base64_3 = Hammer.repeatN(bsfdig, 4);
-    HParser base64_2 = Hammer.sequence({bsfdig, bsfdig, bsfdig_4bit, equals});
-    HParser base64_1 = Hammer.sequence({bsfdig, bsfdig_2bit, equals, equals});
-    HParser base64 = Hammer.sequence({  Hammer.many(base64_3),
-                                        Hammer.optional(Hammer.choice({base64_2, base64_1}))
-                                    });
-
-    return Hammer.sequence({Hammer.whitespace(base64), Hammer.whitespace(Hammer.endP()}});
+static {
+    System.loadLibrary("jhammer");
 }
 
-public static void main(String args[])
+private static void handle(ParseResult result)
 {
-    byte[] input = "RXMgaXN0IFNwYXJnZWx6ZWl0IQo=".getBytes();
-    int length = input.length;
-    HParsedResult result = Hammer.parse(initParser(), input, length);
     if(result == null)
     {
         System.out.println("FAIL");
@@ -40,10 +20,73 @@ public static void main(String args[])
     else
     {
         System.out.println("PASS");
-        //TODO: Pretty print
+        handleToken(result.getAst());
     }
 }
 
+private static void handleToken(ParsedToken p)
+{
+    if(p==null)
+    {
+        System.out.println("Empty AST");
+        return;
+    }
+    switch(p.getTokenType())
+    {
+        case NONE: out("NONE token type"); break;
+        case BYTES: out("BYTES token type, value: " + Arrays.toString(p.getBytesValue())); break;
+        case SINT: out("SINT token type, value: " + p.getSIntValue()); break;
+        case UINT: out("UINT token type, value: " + p.getUIntValue()); break;
+        case SEQUENCE: out("SEQUENCE token type"); for(ParsedToken tok : p.getSeqValue()) {handleToken(tok);} break;
+        case ERR: out("ERR token type"); break;
+        case USER: out("USER token type"); break;
+    }
+}
+
+private static void out(String msg)
+{
+    System.out.println(">> " + msg);
+}
+
+public static void main(String args[])
+{
+    out("chRange");
+    handle(Hammer.parse(Hammer.chRange((byte)0x30, (byte)0x39), "1".getBytes(), 1));
+    handle(Hammer.parse(Hammer.chRange((byte)0x30, (byte)0x39), "a".getBytes(), 1));
+    
+    out("ch");
+    handle(Hammer.parse(Hammer.ch((byte)0x31), "1".getBytes(), 1));
+    handle(Hammer.parse(Hammer.ch((byte)0x31), "0".getBytes(), 1));
+    
+    out("token");
+    handle(Hammer.parse(Hammer.token("herp".getBytes(), 4), "herp".getBytes(), 4));
+    handle(Hammer.parse(Hammer.token("herp".getBytes(), 4), "derp".getBytes(), 4));
+    
+    out("intRange");
+    byte inbytes[] = {0x31, 0x31, 0x31, 0x31};
+    handle(Hammer.parse(Hammer.intRange(Hammer.uInt8(), 0L, 0x32), inbytes, inbytes.length));
+    handle(Hammer.parse(Hammer.intRange(Hammer.uInt8(), 0L, 0x30), inbytes, inbytes.length));
+
+    out("bits");
+    handle(Hammer.parse(Hammer.bits(7, false), inbytes, inbytes.length));
+    
+    out("int64");
+    byte ints[] = {(byte)0x8F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF};
+    handle(Hammer.parse(Hammer.int64(), ints, ints.length));
+    handle(Hammer.parse(Hammer.int64(), inbytes, inbytes.length));
+
+    out("choice");
+    Parser two32s[] = {Hammer.intRange(Hammer.uInt32(), 0x00, 0x01), Hammer.int32()};
+    handle(Hammer.parse(Hammer.choice(two32s), ints, ints.length));
+
+    out("sequence");
+    byte i3[] = {(byte)'i', (byte)3, (byte)0xFF};
+    Parser i3parsers[] = {Hammer.ch((byte)'i'), Hammer.uInt8(), Hammer.int8()};
+    handle(Hammer.parse(Hammer.sequence(i3parsers), i3, i3.length));
+
+    
+}
+
 
 
 }
diff --git a/jni/Makefile b/jni/Makefile
new file mode 100644
index 0000000..85be973
--- /dev/null
+++ b/jni/Makefile
@@ -0,0 +1,42 @@
+JSOURCES := Action.java Hammer.java ParsedToken.java ParseResult.java Parser.java Predicate.java
+JSOURCES_NATIVE := Hammer ParsedToken Parser ParseResult
+CSOURCES := com_upstandinghackers_hammer_Hammer.c com_upstandinghackers_hammer_ParsedToken.c com_upstandinghackers_hammer_Parser.c com_upstandinghackers_hammer_ParseResult.c
+
+# ls *.h *.o *.so com/upstandinghackers/hammer/*.class | grep -v jhammer.h | tr '\n' ' '; replace single $ with $$
+OUTPUTS := com/upstandinghackers/hammer/Action.class com/upstandinghackers/hammer/Hammer.class com_upstandinghackers_hammer_Hammer.h com_upstandinghackers_hammer_Hammer.o com/upstandinghackers/hammer/Hammer\$TokenType.class com_upstandinghackers_hammer_Hammer_TokenType.h com/upstandinghackers/hammer/ParsedToken.class com_upstandinghackers_hammer_ParsedToken.h com_upstandinghackers_hammer_ParsedToken.o com/upstandinghackers/hammer/Parser.class com/upstandinghackers/hammer/ParseResult.class com_upstandinghackers_hammer_ParseResult.h com_upstandinghackers_hammer_ParseResult.o com_upstandinghackers_hammer_Parser.h com_upstandinghackers_hammer_Parser.o com/upstandinghackers/hammer/Predicate.class libjhammer.so
+
+TOPLEVEL := ../
+
+JC=javac
+JH=javah
+CP=com/upstandinghackers/hammer
+PACKAGE=com.upstandinghackers.hammer
+
+include ../common.mk
+
+JNI_INCLUDE := /usr/lib/jvm/java-6-openjdk/include/
+CFLAGS += -fPIC -I. -I $(TOPLEVEL)/src/ -I jni -I $(JNI_INCLUDE) 
+
+%.java: $(call ifsilent,| $(HUSH))
+	$(call hush, "Compiling Java source $@") $(JC) $(CP)/$@
+
+all: javacc prepare compile link
+
+link: compile
+	$(call hush, "Generating libjhammer.so") $(CC) -shared $(CFLAGS) -o libjhammer.so *.o ../src/*.o ../src/backends/*.o ../src/parsers/*.o
+
+$(CSOURCES): prepare
+	$(call hush, "Compiling $@") $(CC) -c $(CFLAGS) $@
+
+compile: prepare $(CSOURCES)
+
+prepare: javacc $(JSOURCES_NATIVE)
+
+$(JSOURCES_NATIVE): javacc
+	$(call hush, "Generating JNI headers for $@") $(JH) $(PACKAGE).$@
+
+javacc: $(JSOURCES)
+
+#TODO make this not-as-hardcoded
+#clean:
+#	rm $(CP)/*.class && rm com_upstandinghackers_*.h && rm com_upstandinghackers_*.o && rm libjhammer.so
diff --git a/jni/NOTES b/jni/NOTES
index 2e6e5bb..564b3f5 100644
--- a/jni/NOTES
+++ b/jni/NOTES
@@ -1,12 +1,26 @@
-Compilation:
-javac com/upstandinghackers/hammer/*.java
+USING THE JNI BINDINGS:
+1. import com.upstandinghackers.hammer.*;
+2. Add a static initializer block that loads the correct library, like this: static { System.loadLibrary("jhammer");  }
+3. Code stuff. Just look at Example.java for a few handy snippets (for walking the resulting syntax tree [AST] etc)
+4. Compile your java sources like always
+5. Add the folder containing libhammer.so/dll/whatever to Java's library path to run it, for example: java -Djava.library.path=. <CLASS>
 
-Conversion to JNI headers:
-find -name "*.class" | sed -e 's/.class$//' | tr '/' '.' | cut -c 3- | xargs javah
-
-Not working:
-enums aren't converted at all, no idea why
+UNIMPLEMENTED:
+User-defined types, predicates and actions are unimplemented.
+Memory leaks because there is no reliable garbage collection.
 
 TODO:
-Implement the entire JNI side
 Testing
+
+
+TYPE MAPPING:
+Hammer      Java        JNI
+uint8_t     byte        jbyte       jbyte/byte is signed
+char        byte        jbyte       jchar would be 16 bit wide
+size_t      int         jint        signed as well; jsize == jint, actually
+int64_t     long        jlong       
+uint64_t     long        jlong       signed!
+bool        boolean     jboolean    JNI_TRUE / JNI_FALSE
+float       float       jfloat
+double      double      jdouble
+void        void        void
diff --git a/jni/com/upstandinghackers/hammer/Action.java b/jni/com/upstandinghackers/hammer/Action.java
new file mode 100644
index 0000000..aecb713
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/Action.java
@@ -0,0 +1,8 @@
+package com.upstandinghackers.hammer;
+
+import java.util.List;
+
+public interface Action
+{
+    public List<ParsedToken> execute(ParseResult p);
+}
diff --git a/jni/com/upstandinghackers/hammer/HAction.java b/jni/com/upstandinghackers/hammer/HAction.java
deleted file mode 100644
index 09aeaaf..0000000
--- a/jni/com/upstandinghackers/hammer/HAction.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.upstandinghackers.hammer;
-
-import java.util.List;
-
-public interface HAction
-{
-    public List<HParsedToken> execute(HParseResult p);
-}
diff --git a/jni/com/upstandinghackers/hammer/HParseResult.java b/jni/com/upstandinghackers/hammer/HParseResult.java
deleted file mode 100644
index ed5c85b..0000000
--- a/jni/com/upstandinghackers/hammer/HParseResult.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.upstandinghackers.hammer;
-
-import java.util.List;
-
-public class HParseResult
-{
-    public native List<HParsedToken> getAst();
-    public native long getBitLength();
-}
diff --git a/jni/com/upstandinghackers/hammer/HParsedToken.java b/jni/com/upstandinghackers/hammer/HParsedToken.java
deleted file mode 100644
index d8a70d6..0000000
--- a/jni/com/upstandinghackers/hammer/HParsedToken.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package com.upstandinghackers.hammer;
-
-import java.util.List;
-
-public class HParsedToken
-{
-    public native Hammer.HTokenType getTokenType();
-    public native int getIndex();
-    public native byte getBitOffset();
-    public native byte[] getBytesValue();
-    public native long getSIntValue();
-    public native long getUIntValue();
-    public native double getDoubleValue();
-    public native float getFloatValue();
-    public native List<HParsedToken> getSeqValue();
-    public native Object getUserValue();
-
-    native void setTokenType(Hammer.HTokenType type);
-    native void setIndex(int index);
-    native void setBitOffset(byte offset);
-    native void setBytesValue(byte[] value);
-    native void setSIntValue(long value);
-    native void setUIntValue(long value);
-    native void setDoubleValue(double value);
-    native void setFloatValue(float value);
-    native void setSeqValue(List<HParsedToken> value);
-    native void setUserValue(Object value);
-}
diff --git a/jni/com/upstandinghackers/hammer/HParser.java b/jni/com/upstandinghackers/hammer/HParser.java
deleted file mode 100644
index 9f2387a..0000000
--- a/jni/com/upstandinghackers/hammer/HParser.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package com.upstandinghackers.hammer;
-
-public class HParser
-{
-    public native void bindIndirect(HParser inner);
-}
diff --git a/jni/com/upstandinghackers/hammer/HPredicate.java b/jni/com/upstandinghackers/hammer/HPredicate.java
deleted file mode 100644
index 9c87875..0000000
--- a/jni/com/upstandinghackers/hammer/HPredicate.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package com.upstandinghackers.hammer;
-
-public interface HPredicate
-{
-    public boolean apply(HParseResult p);
-}
diff --git a/jni/com/upstandinghackers/hammer/Hammer.java b/jni/com/upstandinghackers/hammer/Hammer.java
index e7ad571..cd252d9 100644
--- a/jni/com/upstandinghackers/hammer/Hammer.java
+++ b/jni/com/upstandinghackers/hammer/Hammer.java
@@ -1,4 +1,5 @@
 package com.upstandinghackers.hammer;
+import java.util.HashMap;
 
 public class Hammer
 {
@@ -7,59 +8,69 @@ public class Hammer
     public final static byte BYTE_LITTLE_ENDIAN = 0x0;
     public final static byte BIT_LITTLE_ENDIAN = 0x0;
    
-    public enum HTokenType
+    static final HashMap<Integer, TokenType> tokenTypeMap = new HashMap<Integer, TokenType>();
+
+    public enum TokenType
     {
-        TT_NONE(1),
-        TT_BYTES(2),
-        TT_SINT(4),
-        TT_UINT(8),
-        TT_SEQUENCE(16),
-        TT_ERR(32),
-        TT_USER(64),
+        NONE(1),
+        BYTES(2),
+        SINT(4),
+        UINT(8),
+        SEQUENCE(16),
+        ERR(32),
+        USER(64);
 
         private int value;
         public int getValue() { return this.value; }
-        private HTokenType(int value) { this.value = value; }
+        private TokenType(int value) { this.value = value; }
     }
 
-    public static native HParseResult parse(HParser parser, byte[] input, int length);
-    public static native HParser token(byte[] str, int length);
-    public static native HParser ch(byte c);
-    public static native HParser chRange(byte from, byte to);
-    public static native HParser intRange(HParser p, int lower, int upper);
-    public static native HParser bits(int len, boolean sign);
-    public static native HParser int64();
-    public static native HParser int32();
-    public static native HParser int16();
-    public static native HParser int8();
-    public static native HParser uInt64();
-    public static native HParser uInt32();
-    public static native HParser uInt16();
-    public static native HParser uInt8();
-    public static native HParser whitespace(HParser p);
-    public static native HParser left(HParser p, HParser q);
-    public static native HParser right(HParser p, HParser q);
-    public static native HParser middle(HParser p, HParser x, HParser q);
-    public static native HParser action(HParser p, HAction a);
-    public static native HParser in(byte[] charset, int length);
-    public static native HParser endP();
-    public static native HParser nothingP();
-    public static native HParser sequence(HParser[] parsers);
-    public static native HParser choice(HParser[] parsers);
-    public static native HParser butNot(HParser p1, HParser p2);
-    public static native HParser difference(HParser p1, HParser p2);
-    public static native HParser xor(HParser p1, HParser p2);
-    public static native HParser many(HParser p);
-    public static native HParser many1(HParser p);
-    public static native HParser repeatN(HParser p, int n);
-    public static native HParser optional(HParser p);
-    public static native HParser ignore(HParser p);
-    public static native HParser sepBy(HParser p, HParser sep);
-    public static native HParser sepBy1(HParser p, HParser sep);
-    public static native HParser epsilonP();
-    public static native HParser lengthValue(HParser length, HParser value);
-    public static native HParser attrBool(HParser p, HPredicate pred);
-    public static native HParser and(HParser p);
-    public static native HParser not(HParser p);
-    public static native HParser indirect();
+    static
+    {
+        for(TokenType tt : TokenType.values())
+        {
+            Hammer.tokenTypeMap.put(new Integer(tt.getValue()), tt);
+        }
+    }
+
+    public static native ParseResult parse(Parser parser, byte[] input, int length);
+    public static native Parser token(byte[] str, int length);
+    public static native Parser ch(byte c);
+    public static native Parser chRange(byte from, byte to);
+    public static native Parser intRange(Parser p, long lower, long upper);
+    public static native Parser bits(int len, boolean sign);
+    public static native Parser int64();
+    public static native Parser int32();
+    public static native Parser int16();
+    public static native Parser int8();
+    public static native Parser uInt64();
+    public static native Parser uInt32();
+    public static native Parser uInt16();
+    public static native Parser uInt8();
+    public static native Parser whitespace(Parser p);
+    public static native Parser left(Parser p, Parser q);
+    public static native Parser right(Parser p, Parser q);
+    public static native Parser middle(Parser p, Parser x, Parser q);
+//    public static native Parser action(Parser p, Action a);
+    public static native Parser in(byte[] charset, int length);
+    public static native Parser endP();
+    public static native Parser nothingP();
+    public static native Parser sequence(Parser[] parsers);
+    public static native Parser choice(Parser[] parsers);
+    public static native Parser butNot(Parser p1, Parser p2);
+    public static native Parser difference(Parser p1, Parser p2);
+    public static native Parser xor(Parser p1, Parser p2);
+    public static native Parser many(Parser p);
+    public static native Parser many1(Parser p);
+    public static native Parser repeatN(Parser p, int n);
+    public static native Parser optional(Parser p);
+    public static native Parser ignore(Parser p);
+    public static native Parser sepBy(Parser p, Parser sep);
+    public static native Parser sepBy1(Parser p, Parser sep);
+    public static native Parser epsilonP();
+    public static native Parser lengthValue(Parser length, Parser value);
+//    public static native Parser attrBool(Parser p, Predicate pred);
+    public static native Parser and(Parser p);
+    public static native Parser not(Parser p);
+    public static native Parser indirect();
 }
diff --git a/jni/com/upstandinghackers/hammer/ParseResult.java b/jni/com/upstandinghackers/hammer/ParseResult.java
new file mode 100644
index 0000000..e5ad6c9
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/ParseResult.java
@@ -0,0 +1,15 @@
+package com.upstandinghackers.hammer;
+
+import java.util.List;
+
+public class ParseResult
+{
+    public native ParsedToken getAst();
+    public native long getBitLength();
+    
+    public native void free();
+    public long getInner() {return this.inner;}
+
+    private long inner;
+    ParseResult(long inner) {this.inner=inner;}
+}
diff --git a/jni/com/upstandinghackers/hammer/ParsedToken.java b/jni/com/upstandinghackers/hammer/ParsedToken.java
new file mode 100644
index 0000000..efbc8ed
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/ParsedToken.java
@@ -0,0 +1,40 @@
+package com.upstandinghackers.hammer;
+
+public class ParsedToken
+{
+    public Hammer.TokenType getTokenType()
+    {
+        int tt = this.getTokenTypeInternal();
+        if(0==tt)
+            return null;
+        return Hammer.tokenTypeMap.get(new Integer(tt));
+    }
+
+    private native int getTokenTypeInternal();
+    public native int getIndex();
+    public native byte getBitOffset();
+    public native byte[] getBytesValue();
+    public native long getSIntValue();
+    public native long getUIntValue();
+    public native double getDoubleValue();
+    public native float getFloatValue();
+    public native ParsedToken[] getSeqValue();
+//    public native Object getUserValue();
+
+    native void setTokenType(Hammer.TokenType type);
+    native void setIndex(int index);
+    native void setBitOffset(byte offset);
+    native void setBytesValue(byte[] value);
+    native void setSIntValue(long value);
+    native void setUIntValue(long value);
+    native void setDoubleValue(double value);
+    native void setFloatValue(float value);
+    native void setSeqValue(ParsedToken value[]);
+//    native void setUserValue(Object value);
+    
+//    public native void free();
+    public long getInner() {return this.inner;}
+
+    private long inner;
+    ParsedToken(long inner) {this.inner=inner;}
+}
diff --git a/jni/com/upstandinghackers/hammer/Parser.java b/jni/com/upstandinghackers/hammer/Parser.java
new file mode 100644
index 0000000..2e924cb
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/Parser.java
@@ -0,0 +1,11 @@
+package com.upstandinghackers.hammer;
+
+public class Parser
+{
+    public native void bindIndirect(Parser inner);
+    public native void free();
+    public long getInner() {return this.inner;}
+
+    private long inner;
+    Parser(long inner) {this.inner=inner;}
+}
diff --git a/jni/com/upstandinghackers/hammer/Predicate.java b/jni/com/upstandinghackers/hammer/Predicate.java
new file mode 100644
index 0000000..52ca4bf
--- /dev/null
+++ b/jni/com/upstandinghackers/hammer/Predicate.java
@@ -0,0 +1,6 @@
+package com.upstandinghackers.hammer;
+
+public interface Predicate
+{
+    public boolean apply(ParseResult p);
+}
diff --git a/jni/com_upstandinghackers_hammer_Hammer.c b/jni/com_upstandinghackers_hammer_Hammer.c
new file mode 100644
index 0000000..f83414b
--- /dev/null
+++ b/jni/com_upstandinghackers_hammer_Hammer.c
@@ -0,0 +1,335 @@
+#include "jhammer.h"
+#include "com_upstandinghackers_hammer_Hammer.h"
+#include <stdlib.h>
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_parse
+  (JNIEnv *env, jclass class, jobject obj, jbyteArray input_, jint length_)
+{
+    HParser *parser;
+    uint8_t* input;
+    size_t length;
+    HParseResult *result;
+    jclass resultClass;
+    jobject retVal;
+
+    parser = UNWRAP(env, obj);
+    
+    input = (uint8_t *) ((*env)->GetByteArrayElements(env, input_, NULL));
+    length = (size_t) length_;
+
+    result = h_parse(parser, input, length);
+    
+    if(result==NULL)
+        return NULL;
+
+    FIND_CLASS(resultClass, env, "com/upstandinghackers/hammer/ParseResult");
+    
+    NEW_INSTANCE(retVal, env, resultClass, result);
+ 
+    return retVal;
+}
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_token
+  (JNIEnv *env, jclass class, jbyteArray str, jint len)
+{
+    RETURNWRAP(env, h_token((uint8_t *) ((*env)->GetByteArrayElements(env, str, NULL)), (size_t) len));
+}
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_ch
+  (JNIEnv *env, jclass class, jbyte c)
+{
+    RETURNWRAP(env, h_ch((uint8_t) c));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_chRange
+  (JNIEnv *env, jclass class, jbyte lower, jbyte upper)
+{
+    
+    RETURNWRAP(env, h_ch_range((uint8_t) lower, (uint8_t) upper));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_intRange
+  (JNIEnv *env, jclass class, jobject obj, jlong lower, jlong upper)
+{
+    HParser *parser;
+    parser = UNWRAP(env, obj); 
+    RETURNWRAP(env, h_int_range(parser, (int64_t) lower, (int64_t) upper));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_bits
+  (JNIEnv *env, jclass class, jint len, jboolean sign)
+{
+    RETURNWRAP(env, h_bits((size_t) len, (bool)(sign & JNI_TRUE)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_int64
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_int64());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_int32
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_int32());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_int16
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_int16()); 
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_int8
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_int8());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_uInt64
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_uint64());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_uInt32
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_uint32()); 
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_uInt16
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_uint16());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_uInt8
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_uint8());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_whitespace
+  (JNIEnv *env, jclass class, jobject parser)
+{
+    RETURNWRAP(env, h_whitespace(UNWRAP(env, parser)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_left
+  (JNIEnv *env, jclass class, jobject p, jobject q)
+{
+    RETURNWRAP(env, h_left(UNWRAP(env, p), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_right
+  (JNIEnv *env, jclass class, jobject p, jobject q)
+{
+    RETURNWRAP(env, h_right(UNWRAP(env, p), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_middle
+  (JNIEnv *env, jclass class, jobject p, jobject x, jobject q)
+{
+    RETURNWRAP(env, h_middle(UNWRAP(env, p), UNWRAP(env, x), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_in
+  (JNIEnv *env, jclass class, jbyteArray charset, jint length)
+{
+    RETURNWRAP(env, h_in((uint8_t *) ((*env)->GetByteArrayElements(env, charset, NULL)), (size_t)length));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_endP
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_end_p());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_nothingP
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_nothing_p());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_sequence
+  (JNIEnv *env, jclass class, jobjectArray sequence)
+{
+    jsize length;
+    void **parsers;
+    int i;
+    jobject current;
+    const HParser *result;
+
+    length = (*env)->GetArrayLength(env, sequence);
+    parsers = malloc(sizeof(void *)*(length+1));
+    if(NULL==parsers)
+    {
+        return NULL;
+    }
+
+    for(i=0; i<length; i++)
+    {
+        current = (*env)->GetObjectArrayElement(env, sequence, (jsize)i);
+        parsers[i] = UNWRAP(env, current);
+    }
+    parsers[length] = NULL;
+    
+    result = h_sequence__a(parsers);
+    RETURNWRAP(env, result);
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_choice
+  (JNIEnv *env, jclass class, jobjectArray choices)
+{
+    jsize length;
+    void **parsers;
+    int i;
+    jobject current;
+    const HParser *result;
+
+    length = (*env)->GetArrayLength(env, choices);
+    parsers = malloc(sizeof(HParser *)*(length+1));
+    if(NULL==parsers)
+    {
+        return NULL;
+    }
+
+    for(i=0; i<length; i++)
+    {
+        current = (*env)->GetObjectArrayElement(env, choices, (jsize)i);
+        parsers[i] = UNWRAP(env, current);
+    }
+    parsers[length] = NULL;
+
+    result = h_choice__a(parsers);
+    RETURNWRAP(env, result);
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_butNot
+  (JNIEnv *env, jclass class, jobject p, jobject q)
+{
+    RETURNWRAP(env, h_butnot(UNWRAP(env, p), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_difference
+  (JNIEnv *env, jclass class, jobject p, jobject q)
+{
+    RETURNWRAP(env, h_difference(UNWRAP(env, p), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_xor
+  (JNIEnv *env, jclass class, jobject p, jobject q)
+{
+    RETURNWRAP(env, h_xor(UNWRAP(env, p), UNWRAP(env, q)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_many
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_many(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_many1
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_many1(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_repeatN
+  (JNIEnv *env, jclass class, jobject p, jint n)
+{
+    RETURNWRAP(env, h_repeat_n(UNWRAP(env, p), (size_t)n));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_optional
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_optional(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_ignore
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_ignore(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_sepBy
+  (JNIEnv *env, jclass class, jobject p, jobject sep)
+{
+    RETURNWRAP(env, h_sepBy(UNWRAP(env, p), UNWRAP(env, sep)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_sepBy1
+  (JNIEnv *env, jclass class, jobject p, jobject sep)
+{
+    RETURNWRAP(env, h_sepBy1(UNWRAP(env, p), UNWRAP(env, sep)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_epsilonP
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_epsilon_p());
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_lengthValue
+  (JNIEnv *env, jclass class, jobject length, jobject value)
+{
+    RETURNWRAP(env, h_length_value(UNWRAP(env, length), UNWRAP(env, value)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_and
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_and(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_not
+  (JNIEnv *env, jclass class, jobject p)
+{
+    RETURNWRAP(env, h_not(UNWRAP(env, p)));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_indirect
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_indirect());
+}
+
+
+
diff --git a/jni/com_upstandinghackers_hammer_ParseResult.c b/jni/com_upstandinghackers_hammer_ParseResult.c
new file mode 100644
index 0000000..ad45fe3
--- /dev/null
+++ b/jni/com_upstandinghackers_hammer_ParseResult.c
@@ -0,0 +1,45 @@
+#include "jhammer.h"
+#include "com_upstandinghackers_hammer_ParseResult.h"
+
+HParseResult *unwrap_parse_result(JNIEnv *env, jobject obj)
+{
+    jclass parseResultClass;
+    jfieldID parseResultInner;
+    FIND_CLASS(parseResultClass, env, "com/upstandinghackers/hammer/ParseResult");
+    parseResultInner =  (*env)->GetFieldID(env, parseResultClass, "inner", "J");
+    return (HParseResult *)((*env)->GetLongField(env, obj, parseResultInner));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_ParseResult_getAst
+  (JNIEnv *env, jobject this)
+{
+    HParseResult *inner;
+    jclass parsedTokenClass;
+    jobject retVal;
+
+    if(this == NULL)
+        return NULL; // parse unsuccessful
+    inner = unwrap_parse_result(env, this);
+    if(inner->ast == NULL)
+        return NULL; // parse successful, but empty
+
+    FIND_CLASS(parsedTokenClass, env, "com/upstandinghackers/hammer/ParsedToken");
+    NEW_INSTANCE(retVal, env, parsedTokenClass, inner->ast);
+    return retVal;
+    
+}
+
+JNIEXPORT jlong JNICALL Java_com_upstandinghackers_hammer_ParseResult_getBitLength
+  (JNIEnv *env, jobject this)
+{
+    HParseResult *inner = unwrap_parse_result(env, this);
+    return (jlong) (inner->bit_length);
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParseResult_free
+  (JNIEnv *env, jobject this)
+{
+    //XXX: NOT IMPLEMENTED
+}
+
diff --git a/jni/com_upstandinghackers_hammer_ParsedToken.c b/jni/com_upstandinghackers_hammer_ParsedToken.c
new file mode 100644
index 0000000..e863738
--- /dev/null
+++ b/jni/com_upstandinghackers_hammer_ParsedToken.c
@@ -0,0 +1,195 @@
+#include "jhammer.h"
+#include "com_upstandinghackers_hammer_ParsedToken.h"
+
+#define HPT_UNWRAP(env, this) HParsedToken *inner = unwrap_parsed_token(env, this); assert(inner!=NULL)
+
+HParsedToken *unwrap_parsed_token(JNIEnv *env, jobject obj)
+{
+    jclass parsedTokenClass;
+    jfieldID parsedTokenInner;
+    FIND_CLASS(parsedTokenClass, env, "com/upstandinghackers/hammer/ParsedToken");
+    parsedTokenInner =  (*env)->GetFieldID(env, parsedTokenClass, "inner", "J");
+    return (HParsedToken *)((*env)->GetLongField(env, obj, parsedTokenInner));
+}
+
+
+JNIEXPORT jint JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getTokenTypeInternal
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    if(inner==NULL)
+        return (jint)0;
+    return (jint)(inner->token_type);
+}
+
+JNIEXPORT jint JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getIndex
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jint) (inner->index);
+}
+
+JNIEXPORT jbyte JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getBitOffset
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jbyte) (inner->bit_offset);
+}
+
+JNIEXPORT jbyteArray JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getBytesValue
+  (JNIEnv *env, jobject this)
+{
+    jbyteArray outArray;
+    HPT_UNWRAP(env, this);
+    outArray = (*env)->NewByteArray(env, (jsize)inner->bytes.len);
+    (*env)->SetByteArrayRegion(env, outArray, (jsize) 0, (jsize)(inner->bytes.len), (jbyte *)(inner->bytes.token));
+    return outArray; 
+}
+
+JNIEXPORT jlong JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getSIntValue
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jlong) (inner->sint);
+}
+
+JNIEXPORT jlong JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getUIntValue
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jlong) (inner->uint);
+}
+
+JNIEXPORT jdouble JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getDoubleValue
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jdouble) (inner->dbl);
+}
+
+JNIEXPORT jfloat JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getFloatValue
+  (JNIEnv *env, jobject this)
+{
+    HPT_UNWRAP(env, this);
+    return (jfloat) (inner->flt);
+}
+
+JNIEXPORT jobjectArray JNICALL Java_com_upstandinghackers_hammer_ParsedToken_getSeqValue
+  (JNIEnv *env, jobject this)
+{
+    jsize i;
+    HPT_UNWRAP(env, this);
+    jsize returnSize = inner->seq->used;
+    jobject currentObject;
+    jclass returnClass;
+    FIND_CLASS(returnClass, env, "com/upstandinghackers/hammer/ParsedToken");
+    jobjectArray retVal = (*env)->NewObjectArray(env, returnSize, returnClass, NULL);
+    for(i = 0; i<returnSize; i++)
+    {
+        NEW_INSTANCE(currentObject, env, returnClass, inner->seq->elements[i]);
+        (*env)->SetObjectArrayElement(env, retVal, i, currentObject);
+    }
+    return retVal;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setTokenType
+  (JNIEnv *env, jobject this, jobject tokenType)
+{
+    jclass tokenTypeClass;
+    jmethodID getValue;
+    jint typeVal;
+    HPT_UNWRAP(env, this);
+    
+    FIND_CLASS(tokenTypeClass, env, "com/upstandinghackers/hammer/Hammer$TokenType");
+    getValue = (*env)->GetMethodID(env, tokenTypeClass, "getValue", "()I");
+    typeVal = (*env)->CallIntMethod(env, tokenType, getValue);
+
+    inner->token_type = (int32_t) typeVal; // unsafe cast, but enums should be of type int
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setIndex
+  (JNIEnv *env, jobject this, jint index)
+{
+    HPT_UNWRAP(env, this);
+    inner->index = (size_t)index;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setBitOffset
+  (JNIEnv *env, jobject this, jbyte bit_offset)
+{
+    HPT_UNWRAP(env, this);
+    inner->bit_offset = (char)bit_offset;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setBytesValue
+  (JNIEnv *env, jobject this, jbyteArray bytes_)
+{
+    HBytes bytes;
+    HPT_UNWRAP(env, this);
+    
+    bytes.token = (uint8_t *) ((*env)->GetByteArrayElements(env, bytes_, NULL));
+    bytes.len = (size_t) (*env)->GetArrayLength(env, bytes_);
+
+    inner->bytes = bytes;
+    inner->token_type = TT_BYTES;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setSIntValue
+  (JNIEnv *env, jobject this, jlong sint)
+{
+    HPT_UNWRAP(env, this);
+    inner->token_type = TT_SINT;
+    inner->sint = (int64_t)sint;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setUIntValue
+  (JNIEnv *env, jobject this, jlong uint)
+{
+    HPT_UNWRAP(env, this);
+    inner->token_type = TT_UINT;
+    inner->uint = (uint64_t)uint;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setDoubleValue
+  (JNIEnv *env, jobject this, jdouble dbl)
+{
+    HPT_UNWRAP(env, this);
+    //token_type?
+    inner->dbl = (double)dbl;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setFloatValue
+  (JNIEnv *env, jobject this, jfloat flt)
+{
+    HPT_UNWRAP(env, this);
+    //token_type?
+    inner->flt = (float)flt;
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_ParsedToken_setSeqValue
+  (JNIEnv *env, jobject this, jobjectArray values)
+{
+    HArena *arena;
+    size_t len, i;
+    jobject currentValue;
+    HParsedToken *currentValueInner;
+    HCountedArray *seq;
+    HPT_UNWRAP(env, this);
+    len = (size_t) (*env)->GetArrayLength(env, values); 
+    arena = h_new_arena(&system_allocator, 0);
+    seq = h_carray_new_sized(arena, len);
+    
+    // unwrap each value and append it to the new HCountedArray
+    for(i = 0; i<len; i++)
+    {
+        currentValue = (*env)->GetObjectArrayElement(env, values, (jsize)i);
+        if(NULL == currentValue) 
+            continue;
+        currentValueInner = unwrap_parsed_token(env, currentValue);
+        if(currentValueInner)
+            h_carray_append(seq, (void *)currentValueInner);
+    }
+    
+    inner->token_type = TT_SEQUENCE; 
+    inner->seq = seq;
+}
diff --git a/jni/com_upstandinghackers_hammer_Parser.c b/jni/com_upstandinghackers_hammer_Parser.c
new file mode 100644
index 0000000..4e95e23
--- /dev/null
+++ b/jni/com_upstandinghackers_hammer_Parser.c
@@ -0,0 +1,15 @@
+#include "jhammer.h"
+#include "com_upstandinghackers_hammer_Parser.h"
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_Parser_bindIndirect
+  (JNIEnv *env, jobject this, jobject parser)
+{
+    h_bind_indirect(UNWRAP(env, this), UNWRAP(env, parser));
+}
+
+JNIEXPORT void JNICALL Java_com_upstandinghackers_hammer_Parser_free
+  (JNIEnv *env, jobject this)
+{
+    //XXX NOT IMPLEMENTED
+    //h_free(UNWRAP(env, this));
+}
diff --git a/jni/jhammer.h b/jni/jhammer.h
new file mode 100644
index 0000000..077777b
--- /dev/null
+++ b/jni/jhammer.h
@@ -0,0 +1,31 @@
+#ifndef JHAMMER_H
+#define JHAMMER_H
+#include <jni.h>
+#include "internal.h"
+#include <assert.h>
+
+// Unsafe (non-asserting) helpers
+#define FIND_CLASS_(env, class) (*env)->FindClass(env, class)
+#define REFCONSTRUCTOR_(env, class) (*env)->GetMethodID(env, class, "<init>", "(J)V")
+#define NEW_INSTANCE_(env, class, inner) (*env)->NewObject(env, class, REFCONSTRUCTOR_(env, class), (jlong)inner)
+
+// Safer versions, assert that the result is not NULL
+// If one of those asserts fails, it most likely means that there's a typo (wrong class name or method signature) or big trouble (OOM)
+#define FIND_CLASS(target, env, class) target = FIND_CLASS_(env, class); assert(target != NULL)
+#define REFCONSTRUCTOR(target, env, class) target = REFCONSTRUCTOR_(env, class); assert(target != NULL)
+#define NEW_INSTANCE(target, env, class, inner) target = NEW_INSTANCE_(env, class, inner); assert(target != NULL)
+
+
+// Since there's a LOT of wrapping/unwrapping HParsers, these macros make it a bit more readable
+#define PARSER_CLASS "com/upstandinghackers/hammer/Parser"
+#define PARSER_REF(env) (*env)->GetFieldID(env, FIND_CLASS_(env, PARSER_CLASS), "inner", "J")
+
+#define RETURNWRAP(env, inner) jclass __cls=FIND_CLASS_(env, PARSER_CLASS); \
+                                assert(__cls != NULL); \
+                                jmethodID __constructor = REFCONSTRUCTOR_(env, __cls); \
+                                assert(__constructor != NULL); \
+                                return (*env)->NewObject(env, __cls, __constructor, (jlong)inner)
+
+#define UNWRAP(env, object) (HParser *)((*env)->GetLongField(env, object, PARSER_REF(env)))
+
+#endif
diff --git a/src/Makefile b/src/Makefile
index ead0516..18d3c3f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -62,6 +62,8 @@ include ../common.mk
 $(TESTS): CFLAGS += $(TEST_CFLAGS)
 $(TESTS): LDFLAGS += $(TEST_LDFLAGS)
 
+CFLAGS += -fPIC
+
 all: libhammer.a
 
 libhammer.a: $(HAMMER_PARTS)
diff --git a/src/hammer.h b/src/hammer.h
index d1e8688..31bc021 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -167,14 +167,18 @@ typedef struct HBenchmarkResults_ {
   rtype_t name(__VA_ARGS__, ...);					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
   rtype_t name##__mv(HAllocator* mm__, __VA_ARGS__, va_list ap);	\
-  rtype_t name##__v(__VA_ARGS__, va_list ap)
+  rtype_t name##__v(__VA_ARGS__, va_list ap);   \
+  rtype_t name##__a(void *args[]);  \
+  rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // Note: this drops the attributes on the floor for the __v versions
 #define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...)		\
   rtype_t name(__VA_ARGS__, ...) attr;					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr;		\
   rtype_t name##__mv(HAllocator* mm__, __VA_ARGS__, va_list ap);	\
-  rtype_t name##__v(__VA_ARGS__, va_list ap)
+  rtype_t name##__v(__VA_ARGS__, va_list ap); \
+  rtype_t name##__a(void *args[]);  \
+  rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // }}}
 
diff --git a/src/parsers/choice.c b/src/parsers/choice.c
index 73dedde..6ea7c5e 100644
--- a/src/parsers/choice.c
+++ b/src/parsers/choice.c
@@ -72,3 +72,27 @@ const HParser* h_choice__mv(HAllocator* mm__, const HParser* p, va_list ap_) {
   return ret;
 }
 
+const HParser* h_choice__a(void *args[]) {
+  return h_choice__ma(&system_allocator, args);
+}
+
+const HParser* h_choice__ma(HAllocator* mm__, void *args[]) {
+  size_t len = -1; // because do...while
+  const HParser *arg;
+
+  do {
+    arg=((HParser **)args)[++len];
+  } while(arg);
+
+  HSequence *s = h_new(HSequence, 1);
+  s->p_array = h_new(const HParser *, len);
+
+  for (size_t i = 0; i < len; i++) {
+    s->p_array[i] = ((HParser **)args)[i];
+  }
+
+  s->len = len;
+  HParser *ret = h_new(HParser, 1);
+  ret->vtable = &choice_vt; ret->env = (void*)s;
+  return ret;
+}
diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c
index 21ae31d..564cd14 100644
--- a/src/parsers/sequence.c
+++ b/src/parsers/sequence.c
@@ -73,3 +73,28 @@ const HParser* h_sequence__mv(HAllocator* mm__, const HParser *p, va_list ap_) {
   ret->vtable = &sequence_vt; ret->env = (void*)s;
   return ret;
 }
+
+const HParser* h_sequence__a(void *args[]) {
+  return h_sequence__ma(&system_allocator, args);
+}
+
+const HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
+  size_t len = -1; // because do...while
+  const HParser *arg;
+  
+  do {
+    arg=((HParser **)args)[++len];
+  } while(arg);
+  
+  HSequence *s = h_new(HSequence, 1);
+  s->p_array = h_new(const HParser *, len);
+
+  for (size_t i = 0; i < len; i++) {
+    s->p_array[i] = ((HParser **)args)[i];
+  }
+  
+  s->len = len;
+  HParser *ret = h_new(HParser, 1);
+  ret->vtable = &sequence_vt; ret->env = (void*)s;
+  return ret;
+}

From 62501e544768460399116b175d4cd60518fd0a86 Mon Sep 17 00:00:00 2001
From: aegis <github@xn--g-3fa.is>
Date: Fri, 10 May 2013 14:55:28 +0200
Subject: [PATCH 103/104] Updated gitignore for JNI

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 7f4d7d9..da41ffe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
 *.o
 *~
 *.a
+*.class
+*.so
+jni/com*.h
 src/test_suite
 lib/hush
 examples/dns

From 8976238bcc2f6952af6b5f93e6ee1a7eba4d0e27 Mon Sep 17 00:00:00 2001
From: aegis <github@xn--g-3fa.is>
Date: Thu, 16 May 2013 11:32:37 +0200
Subject: [PATCH 104/104] Switched sequence and choice in JNI to varargs

---
 jni/Example.java                             | 4 ++--
 jni/com/upstandinghackers/hammer/Hammer.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/jni/Example.java b/jni/Example.java
index 8cfa724..4e6d768 100644
--- a/jni/Example.java
+++ b/jni/Example.java
@@ -77,12 +77,12 @@ public static void main(String args[])
 
     out("choice");
     Parser two32s[] = {Hammer.intRange(Hammer.uInt32(), 0x00, 0x01), Hammer.int32()};
-    handle(Hammer.parse(Hammer.choice(two32s), ints, ints.length));
+    handle(Hammer.parse(Hammer.choice(Hammer.intRange(Hammer.uInt32(), 0x00, 0x01), Hammer.int32()), ints, ints.length));
 
     out("sequence");
     byte i3[] = {(byte)'i', (byte)3, (byte)0xFF};
     Parser i3parsers[] = {Hammer.ch((byte)'i'), Hammer.uInt8(), Hammer.int8()};
-    handle(Hammer.parse(Hammer.sequence(i3parsers), i3, i3.length));
+    handle(Hammer.parse(Hammer.sequence(Hammer.ch((byte)'i'), Hammer.uInt8(), Hammer.int8()), i3, i3.length));
 
     
 }
diff --git a/jni/com/upstandinghackers/hammer/Hammer.java b/jni/com/upstandinghackers/hammer/Hammer.java
index cd252d9..3e06a91 100644
--- a/jni/com/upstandinghackers/hammer/Hammer.java
+++ b/jni/com/upstandinghackers/hammer/Hammer.java
@@ -55,8 +55,8 @@ public class Hammer
     public static native Parser in(byte[] charset, int length);
     public static native Parser endP();
     public static native Parser nothingP();
-    public static native Parser sequence(Parser[] parsers);
-    public static native Parser choice(Parser[] parsers);
+    public static native Parser sequence(Parser... parsers);
+    public static native Parser choice(Parser... parsers);
     public static native Parser butNot(Parser p1, Parser p2);
     public static native Parser difference(Parser p1, Parser p2);
     public static native Parser xor(Parser p1, Parser p2);