From 78b7904c5a74fd8044f557355e2e2485060dc414 Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Fri, 1 Nov 2013 18:00:50 -0400
Subject: [PATCH 01/23] We declared some functions that were never implemented.
 Implement them.

---
 src/hammer.c           | 4 ++++
 src/hammer.h           | 3 ++-
 src/parsers/indirect.c | 4 ++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/hammer.c b/src/hammer.c
index 7fc80db..2456bdc 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t*
   return backends[parser->backend]->parse(mm__, parser, &input_stream);
 }
 
+void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) {
+  h_parse_result_free(result);
+}
+
 void h_parse_result_free(HParseResult *result) {
   if(result == NULL) return;
   h_delete_arena(result->arena);
diff --git a/src/hammer.h b/src/hammer.h
index 487fa97..37a273b 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -124,6 +124,7 @@ typedef struct HCFChoice_ HCFChoice;
 typedef struct HRVMProg_ HRVMProg;
 typedef struct HParserVtable_ HParserVtable;
 
+// TODO: Make this internal
 typedef struct HParser_ {
   const HParserVtable *vtable;
   HParserBackend backend;
@@ -570,7 +571,7 @@ char* h_write_result_unamb(const HParsedToken* tok);
  * Format token to the given output stream. Indent starting at
  * [indent] spaces, with [delta] spaces between levels.
  */
-HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta);
+void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);
 
 /**
  * Build parse tables for the given parser backend. See the
diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c
index 2217a20..c91eaab 100644
--- a/src/parsers/indirect.c
+++ b/src/parsers/indirect.c
@@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = {
   .compile_to_rvm = h_not_regular,
 };
 
+void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) {
+  h_bind_indirect(indirect, inner);
+}
+
 void h_bind_indirect(HParser* indirect, const HParser* inner) {
   assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser");
   indirect->env = (void*)inner;

From f8604f5c7658d1bc051267ed36c0b12e67350da1 Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Fri, 1 Nov 2013 18:01:44 -0400
Subject: [PATCH 02/23] Add CFFI python bindings

---
 src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++
 1 file changed, 244 insertions(+)
 create mode 100644 src/bindings/python/hammer.py

diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py
new file mode 100644
index 0000000..d3d1e3d
--- /dev/null
+++ b/src/bindings/python/hammer.py
@@ -0,0 +1,244 @@
+from cffi import FFI
+
+ffi = FFI()
+
+## Types
+ffi.cdef("typedef struct HAllocator_ HAllocator;")
+ffi.cdef("typedef struct HArena_ HArena;")
+ffi.cdef("typedef int bool;")
+ffi.cdef("typedef struct HParseState_ HParseState;")
+ffi.cdef("""
+typedef enum HParserBackend_ {
+  PB_MIN = 0,
+  PB_PACKRAT = 0, // PB_MIN is always the default.
+  PB_REGULAR,
+  PB_LLk,
+  PB_LALR,
+  PB_GLR
+// TODO: support PB_MAX
+} HParserBackend;
+""")
+ffi.cdef("""
+typedef enum HTokenType_ {
+  // Before you change the explicit values of these, think of the poor bindings ;_;
+  TT_NONE = 1,
+  TT_BYTES = 2,
+  TT_SINT = 4,
+  TT_UINT = 8,
+  TT_SEQUENCE = 16,
+  TT_RESERVED_1, // reserved for backend-specific internal use
+  TT_ERR = 32,
+  TT_USER = 64,
+  TT_MAX
+} HTokenType;
+""")
+ffi.cdef("""
+typedef struct HCountedArray_ {
+  size_t capacity;
+  size_t used;
+  HArena * arena;
+  struct HParsedToken_ **elements;
+} HCountedArray;
+""")
+ffi.cdef("""
+typedef struct HBytes_ {
+  const uint8_t *token;
+  size_t len;
+} HBytes;
+""")
+ffi.cdef("""
+typedef struct HParsedToken_ {
+  HTokenType token_type;
+  union {
+    HBytes bytes;
+    int64_t sint;
+    uint64_t uint;
+    double dbl;
+    float flt;
+    HCountedArray *seq; // a sequence of HParsedToken's
+    void *user;
+  };
+  size_t index;
+  char bit_offset;
+} HParsedToken;
+""")
+ffi.cdef("""
+typedef struct HParseResult_ {
+  const HParsedToken *ast;
+  long long bit_length;
+  HArena * arena;
+} HParseResult;
+""")
+
+ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
+ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
+ffi.cdef("""
+typedef struct HCFChoice_ HCFChoice;
+typedef struct HRVMProg_ HRVMProg;
+typedef struct HParserVtable_ HParserVtable;
+""")
+
+ffi.cdef("typedef struct HParser_ HParser;")
+ffi.cdef("""
+typedef struct HParserTestcase_ {
+  unsigned char* input;
+  size_t length;
+  char* output_unambiguous;
+} HParserTestcase;
+
+typedef struct HCaseResult_ {
+  bool success;
+  union {
+    const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
+    size_t parse_time; // on success, filled in with time for a single parse, in nsec
+  };
+} HCaseResult;
+
+typedef struct HBackendResults_ {
+  HParserBackend backend;
+  bool compile_success;
+  size_t n_testcases;
+  size_t failed_testcases; // actually a count...
+  HCaseResult *cases;
+} HBackendResults;
+
+typedef struct HBenchmarkResults_ {
+  size_t len;
+  HBackendResults *results;
+} HBenchmarkResults;
+""")
+
+## The following section was generated by
+## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' 
+ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
+ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
+ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
+ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
+ffi.cdef("HParser* h_ch(const uint8_t c);")
+ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
+ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
+ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
+ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
+ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
+ffi.cdef("HParser* h_bits(size_t len, bool sign);")
+ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
+ffi.cdef("HParser* h_int64(void);")
+ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_int32(void);")
+ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_int16(void);")
+ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_int8(void);")
+ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_uint64(void);")
+ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_uint32(void);")
+ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_uint16(void);")
+ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_uint8(void);")
+ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_whitespace(const HParser* p);")
+ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
+ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
+ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
+ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
+ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
+ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
+ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
+ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
+ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+ffi.cdef("HParser* h_end_p(void);")
+ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_nothing_p(void);")
+ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_sequence(HParser* p, ...);")
+ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
+ffi.cdef("HParser* h_sequence__a(void* args);")
+ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
+ffi.cdef("HParser* h_choice(HParser* p, ...);")
+ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
+ffi.cdef("HParser* h_choice__a(void* args);")
+ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
+ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+ffi.cdef("HParser* h_many(const HParser* p);")
+ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_many1(const HParser* p);")
+ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
+ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
+ffi.cdef("HParser* h_optional(const HParser* p);")
+ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_ignore(const HParser* p);")
+ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
+ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
+ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+ffi.cdef("HParser* h_epsilon_p(void);")
+ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
+ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
+ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
+ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
+ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
+ffi.cdef("HParser* h_and(const HParser* p);")
+ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_not(const HParser* p);")
+ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
+ffi.cdef("HParser* h_indirect(void);")
+ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
+ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
+ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
+ffi.cdef("void h_parse_result_free(HParseResult *result);")
+ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
+ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
+ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
+ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
+ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
+ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
+
+lib = ffi.verify("#include <hammer/hammer.h>",
+                 libraries=['hammer'])
+
+
+# Quick test
+def fromCobj(cobj):
+    # TODO: Free the toplevel parser
+    tt = cobj.token_type
+    if cobj.token_type == lib.TT_BYTES:
+        return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
+    elif cobj.token_type == lib.TT_ERR:
+        # I have no idea what this is for
+        pass
+    elif cobj.token_type == lib.TT_NONE:
+        return None
+    elif cobj.token_type == lib.TT_SEQUENCE:
+        return [fromCobj(cobj.seq.elements[i])
+                for i in range(cobj.seq.used)]
+    elif cobj.token_type == lib.TT_SINT:
+        return cobj.sint
+    elif cobj.token_type == lib.TT_UINT:
+        return cobj.uint
+
+def fromParseResult(cobj):
+    ret = fromCobj(cobj.ast)
+    lib.h_parse_result_free(cobj)
+    return ret
+
+def run_test():
+    p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')),
+                                       lib.h_ch(ord('2')),
+                                       lib.h_ch(ord('3')),
+                                       ffi.NULL),
+                          lib.h_ch(ord(',')))
+    return fromParseResult(lib.h_parse(p_test, "1,2,3", 5))
+

From 8928c04456e8bb190d02d06869fc13278aa9f3bf Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Thu, 7 Nov 2013 22:34:27 -0500
Subject: [PATCH 03/23] Working python bindings

---
 src/bindings/desugar-header.pl |  22 ++
 src/bindings/python/hammer.py  | 462 +++++++++++++++++++++++----------
 src/parsers/token.c            |   4 +-
 3 files changed, 353 insertions(+), 135 deletions(-)
 create mode 100644 src/bindings/desugar-header.pl

diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl
new file mode 100644
index 0000000..5bdd11e
--- /dev/null
+++ b/src/bindings/desugar-header.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl -w
+
+
+my $arg = qr/[^,]*/;
+
+while(<>) {
+  chomp;
+  if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) {
+    print "$1 $2(void);\n";
+    print "$1 $2__m(HAllocator* mm__);\n";
+  } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) {
+    print "$1 $2($3);\n";
+    print "$1 $2__m(HAllocator* mm__, $3);\n";
+  } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) {
+    print "$2 $3($4, ...);\n";
+    print "$2 $3__m(HAllocator *mm__, $4, ...);\n";
+    print "$2 $3__a(void* args);\n";
+    print "$2 $3__ma(HAllocator* mm__, void* args);\n";
+  } elsif (/^HAMMER_FN_DECL/) {
+    print "\e[1;31m!!!\e[0m " . $_ . "\n";
+  }
+}
diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py
index d3d1e3d..e7e0822 100644
--- a/src/bindings/python/hammer.py
+++ b/src/bindings/python/hammer.py
@@ -1,13 +1,15 @@
 from cffi import FFI
+import threading
+import sys
 
-ffi = FFI()
+_ffi = FFI()
 
 ## Types
-ffi.cdef("typedef struct HAllocator_ HAllocator;")
-ffi.cdef("typedef struct HArena_ HArena;")
-ffi.cdef("typedef int bool;")
-ffi.cdef("typedef struct HParseState_ HParseState;")
-ffi.cdef("""
+_ffi.cdef("typedef struct HAllocator_ HAllocator;")
+_ffi.cdef("typedef struct HArena_ HArena;")
+_ffi.cdef("typedef int bool;")
+_ffi.cdef("typedef struct HParseState_ HParseState;")
+_ffi.cdef("""
 typedef enum HParserBackend_ {
   PB_MIN = 0,
   PB_PACKRAT = 0, // PB_MIN is always the default.
@@ -18,7 +20,7 @@ typedef enum HParserBackend_ {
 // TODO: support PB_MAX
 } HParserBackend;
 """)
-ffi.cdef("""
+_ffi.cdef("""
 typedef enum HTokenType_ {
   // Before you change the explicit values of these, think of the poor bindings ;_;
   TT_NONE = 1,
@@ -32,7 +34,7 @@ typedef enum HTokenType_ {
   TT_MAX
 } HTokenType;
 """)
-ffi.cdef("""
+_ffi.cdef("""
 typedef struct HCountedArray_ {
   size_t capacity;
   size_t used;
@@ -40,13 +42,13 @@ typedef struct HCountedArray_ {
   struct HParsedToken_ **elements;
 } HCountedArray;
 """)
-ffi.cdef("""
+_ffi.cdef("""
 typedef struct HBytes_ {
   const uint8_t *token;
   size_t len;
 } HBytes;
 """)
-ffi.cdef("""
+_ffi.cdef("""
 typedef struct HParsedToken_ {
   HTokenType token_type;
   union {
@@ -62,7 +64,7 @@ typedef struct HParsedToken_ {
   char bit_offset;
 } HParsedToken;
 """)
-ffi.cdef("""
+_ffi.cdef("""
 typedef struct HParseResult_ {
   const HParsedToken *ast;
   long long bit_length;
@@ -70,16 +72,16 @@ typedef struct HParseResult_ {
 } HParseResult;
 """)
 
-ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
-ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
-ffi.cdef("""
+_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
+_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
+_ffi.cdef("""
 typedef struct HCFChoice_ HCFChoice;
 typedef struct HRVMProg_ HRVMProg;
 typedef struct HParserVtable_ HParserVtable;
 """)
 
-ffi.cdef("typedef struct HParser_ HParser;")
-ffi.cdef("""
+_ffi.cdef("typedef struct HParser_ HParser;")
+_ffi.cdef("""
 typedef struct HParserTestcase_ {
   unsigned char* input;
   size_t length;
@@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ {
 } HBenchmarkResults;
 """)
 
-## The following section was generated by
-## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' 
-ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
-ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
-ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
-ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
-ffi.cdef("HParser* h_ch(const uint8_t c);")
-ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
-ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
-ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
-ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
-ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
-ffi.cdef("HParser* h_bits(size_t len, bool sign);")
-ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
-ffi.cdef("HParser* h_int64(void);")
-ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_int32(void);")
-ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_int16(void);")
-ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_int8(void);")
-ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_uint64(void);")
-ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_uint32(void);")
-ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_uint16(void);")
-ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_uint8(void);")
-ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_whitespace(const HParser* p);")
-ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
-ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
-ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
-ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
-ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
-ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
-ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
-ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
-ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
-ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
-ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
-ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
-ffi.cdef("HParser* h_end_p(void);")
-ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_nothing_p(void);")
-ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_sequence(HParser* p, ...);")
-ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
-ffi.cdef("HParser* h_sequence__a(void* args);")
-ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
-ffi.cdef("HParser* h_choice(HParser* p, ...);")
-ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
-ffi.cdef("HParser* h_choice__a(void* args);")
-ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
-ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
-ffi.cdef("HParser* h_many(const HParser* p);")
-ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_many1(const HParser* p);")
-ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
-ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
-ffi.cdef("HParser* h_optional(const HParser* p);")
-ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_ignore(const HParser* p);")
-ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
-ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
-ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
-ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
-ffi.cdef("HParser* h_epsilon_p(void);")
-ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
-ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
-ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
-ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
-ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
-ffi.cdef("HParser* h_and(const HParser* p);")
-ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_not(const HParser* p);")
-ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
-ffi.cdef("HParser* h_indirect(void);")
-ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
-ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
-ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
-ffi.cdef("void h_parse_result_free(HParseResult *result);")
-ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
-ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
-ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
-ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
-ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
-ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
+## Arena functions
+_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);")
+_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);")
 
-lib = ffi.verify("#include <hammer/hammer.h>",
+## The following section was generated by
+## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/'
+_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_ch(const uint8_t c);")
+_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
+_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_bits(size_t len, bool sign);")
+_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
+_ffi.cdef("HParser* h_int64(void);")
+_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int32(void);")
+_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int16(void);")
+_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int8(void);")
+_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint64(void);")
+_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint32(void);")
+_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint16(void);")
+_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint8(void);")
+_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_whitespace(const HParser* p);")
+_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_end_p(void);")
+_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_nothing_p(void);")
+_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_sequence(HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__a(void* args);")
+_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_choice(HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__a(void* args);")
+_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_many(const HParser* p);")
+_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_many1(const HParser* p);")
+_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_optional(const HParser* p);")
+_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_ignore(const HParser* p);")
+_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_epsilon_p(void);")
+_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_and(const HParser* p);")
+_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_not(const HParser* p);")
+_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_indirect(void);")
+_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
+_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_parse_result_free(HParseResult *result);")
+_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
+_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
+_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
+_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
+
+_lib = _ffi.verify("#include <hammer/hammer.h>",
                  libraries=['hammer'])
 
+_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45
 
-# Quick test
-def fromCobj(cobj):
+class _DynamicScopeHolder(threading.local):
+    """A dynamically-scoped holder of python objects, which may or may not
+    otherwise appear in the object graph. Intended for use with CFFI """
+    def __init__(self):
+        self._ctxstack = []
+    def __enter__(self):
+        self._ctxstack.append([])
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._ctxstack.pop()
+        return False
+    def stash(self, *objs):
+        if len(self._ctxstack) < 1:
+            raise Exception("Not in any dynamic scope")
+        for obj in objs:
+            self._ctxstack[-1].append(obj)
+def _fromHParsedToken(cobj):
     # TODO: Free the toplevel parser
     tt = cobj.token_type
-    if cobj.token_type == lib.TT_BYTES:
-        return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
-    elif cobj.token_type == lib.TT_ERR:
+
+    if cobj.token_type == _lib.TT_BYTES:
+        return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
+    elif cobj.token_type == _lib.TT_ERR:
         # I have no idea what this is for
         pass
-    elif cobj.token_type == lib.TT_NONE:
+    elif cobj.token_type == _lib.TT_NONE:
         return None
-    elif cobj.token_type == lib.TT_SEQUENCE:
-        return [fromCobj(cobj.seq.elements[i])
+    elif cobj.token_type == _lib.TT_SEQUENCE:
+        return [_fromHParsedToken(cobj.seq.elements[i])
                 for i in range(cobj.seq.used)]
-    elif cobj.token_type == lib.TT_SINT:
+    elif cobj.token_type == _lib.TT_SINT:
         return cobj.sint
-    elif cobj.token_type == lib.TT_UINT:
+    elif cobj.token_type == _lib.TT_UINT:
         return cobj.uint
+    elif cobj.token_type == _lib.TT_PYTHON:
+        return _ffi.from_handle(cobj.user)
 
-def fromParseResult(cobj):
-    ret = fromCobj(cobj.ast)
-    lib.h_parse_result_free(cobj)
+_parser_result_holder = _DynamicScopeHolder()
+def _toHParsedToken(arena, pyobj):
+    if pyobj is None:
+        return _ffi.NULL
+    cobj = _ffi.new_handle(pyobj)
+    _parser_result_holder.stash(cobj)
+
+    hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken")))
+    hpt.token_type = _lib.TT_PYTHON
+    hpt.user = cobj
+    hpt.bit_offset = 127;
+    hpt.index = 0;
+    return hpt
+
+def _fromParseResult(cobj):
+    ret = _fromHParsedToken(cobj.ast)
+    _lib.h_parse_result_free(cobj)
     return ret
 
-def run_test():
-    p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')),
-                                       lib.h_ch(ord('2')),
-                                       lib.h_ch(ord('3')),
-                                       ffi.NULL),
-                          lib.h_ch(ord(',')))
-    return fromParseResult(lib.h_parse(p_test, "1,2,3", 5))
+def _to_haction(fn):
+    """Turn a function that transforms a parsed value into an HAction"""
+    def action(parse_result):
+        res = _toHParsedToken(parse_result.arena,  fn(_fromParseResult(parse_result)))
+        if res != _ffi.NULL and parse_result.ast != _ffi.NULL:
+            res.index = parse_result.ast.index
+            res.bit_offset = parse_result.ast.bit_offset
+        return res
+    return _ffi.callback("HParsedToken*(HParseResult*)", action)
 
+def _to_hpredicate(fn):
+    """Turn a function that transforms a parsed value into an HAction"""
+    def predicate(parse_result):
+        res = fn(_fromParseResult(parse_result))
+        # TODO: Handle exceptions; parse should fail.
+        if type(res) != bool:
+            raise TypeError("Predicates should return a bool")
+        return res
+    return _ffi.callback("bool(HParseResult*)", action)
+
+class Parser(object):
+    # TODO: Map these to individually garbage-collected blocks of
+    # memory. Perhaps with an arena allocator with block size of 1?
+    # There has to be something more efficient than that, though.
+
+    # TODO: How do we handle encodings? By default, we're using UTF-8
+    def __init__(self, internal, deps):
+        """Create a new parser from an FFI object. Not for user code"""
+        self._parser = internal
+        self._deps = deps
+
+    def parse(self, string):
+        with _parser_result_holder:
+            pres = _lib.h_parse(self._parser, string, len(string))
+            if pres:
+                return _fromParseResult(pres)
+            else:
+                return None
+
+class IndirectParser(Parser):
+    def bind(self, inner):
+        _lib.h_bind_indirect(self._parser, inner._parser)
+        self._deps = (inner,)
+
+class BitsParser(Parser):
+    pass
+
+def token(token):
+    # TODO: Does not clone argument.
+    if isinstance(token, unicode):
+        token = token.encode("utf-8")
+    return Parser(_lib.h_token(token, len(token)), ())
+
+def ch(char):
+    return token(char)
+
+def ch_range(chr1, chr2):
+    if not isinstance(chr1, str) or not isinstance(chr2, str):
+        raise TypeError("ch_range can't handle unicode")
+    return Parser(_lib.h_ch_range(chr1, chr2), ())
+
+def int_range(parser, i1, i2):
+    if type(parser) != BitsParser:
+        raise TypeError("int_range is only valid when used with a bits parser")
+    return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,))
+
+def bits(length, signedp):
+    return BitsParser(_lib.h_bits(length, signedp), ())
+
+def int64(): return bits(64, True)
+def int32(): return bits(32, True)
+def int16(): return bits(16, True)
+def int8 (): return bits(8,  True)
+def uint64(): return bits(64, False)
+def uint32(): return bits(32, False)
+def uint16(): return bits(16, False)
+def uint8 (): return bits(8,  False)
+
+def whitespace(p):
+    return Parser(_lib.h_whitespace(p._parser), (p,))
+def left(p1, p2):
+    return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2))
+def right(p1, p2):
+    return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2))
+def middle(p1, p2, p3):
+    return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3))
+def action(parser, action):
+    caction = _to_haction(action)
+    return Parser(_lib.h_action(parser._parser, caction), (parser, caction))
+def in_(charset):
+    if typeof(charset) is not str:
+        # TODO/Python3: change str to bytes
+        raise TypeError("in_ can't deal with unicode")
+    return Parser(_lib.h_in(charset, len(charset)), ())
+def not_in(charset):
+    if typeof(charset) is not str:
+        # TODO/Python3: change str to bytes
+        raise TypeError("in_ can't deal with unicode")
+    return Parser(_lib.h_not_in(charset, len(charset)), ())
+def end_p():
+    return Parser(_lib.h_end_p(), ())
+def nothing_p():
+    return Parser(_lib.h_nothing_p(), ())
+def sequence(*parsers):
+    plist = [p._parser for p in parsers]
+    plist.append(_ffi.NULL)
+    return Parser(_lib.h_sequence(*plist), (plist,))
+def choice(*parsers):
+    plist = [p._parser for p in parsers]
+    plist.append(_ffi.NULL)
+    return Parser(_lib.h_choice(*plist), (plist,))
+def butnot(p1, p2):
+    return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2))
+def difference(p1, p2):
+    return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2))
+def xor(p1, p2):
+    return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2))
+def many(p1):
+    return Parser(_lib.h_many(p1._parser), (p1,))
+def many1(p1):
+    return Parser(_lib.h_many1(p1._parser), (p1,))
+def repeat_n(p1, n):
+    return Parser(_lib.h_repeat_n(p1._parser, n), (p1,))
+def optional(p1):
+    return Parser(_lib.h_optional(p1._parser), (p1,))
+def ignore(p1):
+    return Parser(_lib.h_ignore(p1._parser), (p1,))
+def sepBy(p, sep):
+    return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep))
+def sepBy1(p, sep):
+    return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep))
+def epsilon_p():
+    return Parser(_lib.h_epsilon_p(), ())
+def length_value(p_len, p_value):
+    return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value))
+def attr_bool(parser, predicate):
+    cpredicate = _to_hpredicate(predicate)
+    return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate))
+def and_(parser):
+    return Parser(_lib.h_and(parser._parser), (parser,))
+def not_(parser):
+    return Parser(_lib.h_not(parser._parser), (parser,))
+def indirect():
+    return IndirectParser(_lib.h_indirect(), ())
+def bind_indirect(indirect, inner):
+    indirect.bind(inner)
+
+def parse(parser):
+    return parser.parse()
+
+# Unfortunately, "in", "and", and "not" are keywords. This makes them
+# show up in the module namespace for the use of automated tools. Do
+# not attempt to use them by hand; only use the mangled forms (with
+# the '_')
+sys.modules[__name__].__dict__["in"] = in_
+sys.modules[__name__].__dict__["and"] = and_
+sys.modules[__name__].__dict__["not"] = not_
+
+def run_test():
+    p_test = sepBy1(choice(ch('1'),
+                           ch('2'),
+                           ch('3')),
+                    ch(','))
+    return p_test.parse("1,2,3")
diff --git a/src/parsers/token.c b/src/parsers/token.c
index 97886b7..7a05636 100644
--- a/src/parsers/token.c
+++ b/src/parsers/token.c
@@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) {
 }
 HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { 
   HToken *t = h_new(HToken, 1);
-  t->str = (uint8_t*)str, t->len = len;
+  uint8_t *str_cpy = h_new(uint8_t, len);
+  memcpy(str_cpy, str, len);
+  t->str = str_cpy, t->len = len;
   return h_new_parser(mm__, &token_vt, t);
 }

From 390bbd32f12ef45fd801fcb633ff836e26c5e565 Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Fri, 8 Nov 2013 17:20:00 -0500
Subject: [PATCH 04/23] Added auto-construction, tidied h_ch a bit

---
 src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 8 deletions(-)

diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py
index e7e0822..a25dd93 100644
--- a/src/bindings/python/hammer.py
+++ b/src/bindings/python/hammer.py
@@ -4,7 +4,8 @@ import sys
 
 _ffi = FFI()
 
-## Types
+# {{{ Types
+
 _ffi.cdef("typedef struct HAllocator_ HAllocator;")
 _ffi.cdef("typedef struct HArena_ HArena;")
 _ffi.cdef("typedef int bool;")
@@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ {
 } HBenchmarkResults;
 """)
 
-## Arena functions
+# }}}
+# {{{ Arena functions
 _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);")
 _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);")
-
+# }}}
+# {{{ cdefs
 ## The following section was generated by
 ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/'
 _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
@@ -216,7 +219,7 @@ _lib = _ffi.verify("#include <hammer/hammer.h>",
                  libraries=['hammer'])
 
 _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45
-
+# }}}
 class _DynamicScopeHolder(threading.local):
     """A dynamically-scoped holder of python objects, which may or may not
     otherwise appear in the object graph. Intended for use with CFFI """
@@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj):
     hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken")))
     hpt.token_type = _lib.TT_PYTHON
     hpt.user = cobj
-    hpt.bit_offset = 127;
-    hpt.index = 0;
+    hpt.bit_offset = chr(127)
+    hpt.index = 0
     return hpt
 
 def _fromParseResult(cobj):
@@ -311,6 +314,11 @@ class Parser(object):
             else:
                 return None
 
+    def __mul__(self, count):
+        return repeat_n(self, count)
+
+
+                
 class IndirectParser(Parser):
     def bind(self, inner):
         _lib.h_bind_indirect(self._parser, inner._parser)
@@ -326,12 +334,20 @@ def token(token):
     return Parser(_lib.h_token(token, len(token)), ())
 
 def ch(char):
-    return token(char)
+    """Returns either a token or an int, depending on the type of the
+    argument"""
+    if isinstance(char, int):
+        return Parser(_lib.h_ch(char), ())
+    else:
+        return token(char)
 
 def ch_range(chr1, chr2):
     if not isinstance(chr1, str) or not isinstance(chr2, str):
         raise TypeError("ch_range can't handle unicode")
-    return Parser(_lib.h_ch_range(chr1, chr2), ())
+    def my_action(pr):
+        # print "In action: ", pr
+        return pr
+    return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action)
 
 def int_range(parser, i1, i2):
     if type(parser) != BitsParser:
@@ -436,3 +452,36 @@ def run_test():
                            ch('3')),
                     ch(','))
     return p_test.parse("1,2,3")
+
+# {{{ Automatic parser construction... python specific
+
+# TODO: Implement Parsable metaclass, which requires the existence of
+# a "parse" method.
+
+# This is expected to be extended by user code. As a general rule,
+# only provide auto-parsers for your own types.
+AUTO_PARSERS = {
+    str: token,
+    unicode: token,
+}
+
+def _auto_seq(lst):
+    return sequence(*(auto_1(p, default_method=_auto_choice)
+                      for p in lst))
+        
+def _auto_choice(lst):
+    return choice(*(auto_1(p, default_method=_auto_seq)
+                    for p in lst))
+
+def auto_1(arg, default_method=_auto_choice):
+    if isinstance(arg, Parser):
+        return arg
+    elif type(arg) in AUTO_PARSERS:
+        return AUTO_PARSERS[type(arg)](arg)
+    else:
+        return default_method(arg)
+
+def auto(*args):
+    return auto_1(args, default_method=_auto_choice)
+
+# }}}

From 06aa19e206ea0155aa17825dbcf5645d1be57685 Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Tue, 12 Nov 2013 19:07:32 -0600
Subject: [PATCH 05/23] Added a few tests

---
 src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 src/bindings/python/hammer_tests.py

diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
new file mode 100644
index 0000000..b040141
--- /dev/null
+++ b/src/bindings/python/hammer_tests.py
@@ -0,0 +1,29 @@
+import unittest
+import hammer as h
+
+class TestTokenParser(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.token("95\xa2")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("95\xa2"), "95\xa2")
+    def test_partial_fails(self):
+        self.assertEqual(self.parser.parse("95"), None)
+
+class TestChParser(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser_int = h.ch(0xa2)
+        cls.parser_chr = h.ch("\xa2")
+    def test_success(self):
+        self.assertEqual(self.parser_int.parse("\xa2"), 0xa2)
+        self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2")
+    def test_failure(self):
+        self.assertEqual(self.parser_int.parse("\xa3"), None)
+        self.assertEqual(self.parser_chr.parse("\xa3"), None)
+
+class TestChRange(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = 
+        

From c9c282dcb468daebc45073193b568f5f55ae4f01 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Thu, 14 Nov 2013 15:50:58 +0100
Subject: [PATCH 06/23] Finished writing unit tests. Fixed a few small bugs in
 hammer.py. Further issues:

 * "in_" and "not_in" should coerce their results to strings (i.e., chr(result))
 * TestLeftrec: success case 2 fails
 * TestChRange: success case segfaults
 * TestWhitespaceEnd: success case segfaults
 * TestAction: success case segfaults with "corrupted double-linked list"
 * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range
 * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted"
---
 src/bindings/python/hammer.py       |  15 +-
 src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++-
 2 files changed, 486 insertions(+), 8 deletions(-)

diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py
index a25dd93..36b78c8 100644
--- a/src/bindings/python/hammer.py
+++ b/src/bindings/python/hammer.py
@@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj):
     cobj = _ffi.new_handle(pyobj)
     _parser_result_holder.stash(cobj)
 
-    hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken")))
+    hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken")))
     hpt.token_type = _lib.TT_PYTHON
     hpt.user = cobj
     hpt.bit_offset = chr(127)
@@ -293,7 +293,7 @@ def _to_hpredicate(fn):
         if type(res) != bool:
             raise TypeError("Predicates should return a bool")
         return res
-    return _ffi.callback("bool(HParseResult*)", action)
+    return _ffi.callback("bool(HParseResult*)", predicate)
 
 class Parser(object):
     # TODO: Map these to individually garbage-collected blocks of
@@ -352,7 +352,7 @@ def ch_range(chr1, chr2):
 def int_range(parser, i1, i2):
     if type(parser) != BitsParser:
         raise TypeError("int_range is only valid when used with a bits parser")
-    return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,))
+    return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,))
 
 def bits(length, signedp):
     return BitsParser(_lib.h_bits(length, signedp), ())
@@ -373,17 +373,18 @@ def left(p1, p2):
 def right(p1, p2):
     return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2))
 def middle(p1, p2, p3):
-    return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3))
+    return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3))
 def action(parser, action):
     caction = _to_haction(action)
     return Parser(_lib.h_action(parser._parser, caction), (parser, caction))
+
 def in_(charset):
-    if typeof(charset) is not str:
+    if not isinstance(charset, str):
         # TODO/Python3: change str to bytes
         raise TypeError("in_ can't deal with unicode")
     return Parser(_lib.h_in(charset, len(charset)), ())
 def not_in(charset):
-    if typeof(charset) is not str:
+    if not isinstance(charset, str):
         # TODO/Python3: change str to bytes
         raise TypeError("in_ can't deal with unicode")
     return Parser(_lib.h_not_in(charset, len(charset)), ())
@@ -402,7 +403,7 @@ def choice(*parsers):
 def butnot(p1, p2):
     return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2))
 def difference(p1, p2):
-    return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2))
+    return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2))
 def xor(p1, p2):
     return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2))
 def many(p1):
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index b040141..a56d669 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase):
 class TestChRange(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = 
+        cls.parser = h.ch_range("a", "c")
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("d"), None)
+
+class TestInt64(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int64()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None)
+
+class TestInt32(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int32()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000)
+        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xff\xfe\x00"), None)
+        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestInt16(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int16()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xfe\x00"), -0x200)
+        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xfe"), None)
+        self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestInt8(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int8()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x88"), -0x78)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+
+class TestUint64(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint64()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None)
+
+class TestUint32(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint32()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestUint16(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint16()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestUint8(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint8()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x78"), 0x78)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
         
+class TestIntRange(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int_range(h.uint8(), 3, 10)
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x05"), 5)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x0b"), None)
+
+class TestWhitespace(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.whitespace(h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse(" a"), "a")
+        self.assertEqual(self.parser.parse("  a"), "a")
+        self.assertEqual(self.parser.parse("\ta"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("_a"), None)
+
+class TestWhitespaceEnd(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.whitespace(h.end_p())
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse(""), "")
+#        self.assertEqual(self.parser.parse("  "), "")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("  x"), None)
+
+class TestLeft(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.left(h.ch("a"), h.ch(" "))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a "), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse("ab"), None)
+
+class TestRight(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.right(h.ch(" "), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(" a"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse("ba"), None)
+
+class TestMiddle(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" "))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(" a "), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse(" a"), None)
+        self.assertEqual(self.parser.parse("a "), None)
+        self.assertEqual(self.parser.parse(" b "), None)
+        self.assertEqual(self.parser.parse("ba "), None)
+        self.assertEqual(self.parser.parse(" ab"), None)
+
+class TestAction(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x])
+### fails with "corrupted double-linked list"
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("ab"), ["A", "B"])
+#        self.assertEqual(self.parser.parse("AB"), ["A", "B"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("XX"), None)
+
+class TestIn(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.in_("abc")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("d"), None)
+
+class TestNotIn(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.not_in("abc")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("d"), "d")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+class TestEndP(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.end_p())
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("aa"), None)
+
+class TestNothingP(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.nothing_p()
+    def test_success(self):
+        pass
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+class TestSequence(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse("b"), None)
+
+class TestSequenceWhitespace(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+        self.assertEqual(self.parser.parse("a b"), ["a", "b"])
+        self.assertEqual(self.parser.parse("a  b"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a  c"), None)
+
+class TestChoice(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.choice(h.ch("a"), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("c"), None)
+
+class TestButNot(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.butnot(h.ch("a"), h.token("ab"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("aa"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("ab"), None)
+
+### fails with malloc() memory corruption
+#class TestButNotRange(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6"))
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("4"), "4")
+### this segfaults
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("6"), None)
+
+class TestDifference(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.difference(h.token("ab"), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), "ab")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+#class TestXor(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9"))
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("0"), "0")
+#        self.assertEqual(self.parser.parse("9"), "9")
+### fails with "malloc(): smallbin double linked list corrupted"
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("5"), None)
+#        self.assertEqual(self.parser.parse("a"), None)
+
+class TestMany(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.many(h.choice(h.ch("a"), h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(""), [])
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("b"), ["b"])
+        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+    def test_failure(self):
+        pass
+
+class TestMany1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("b"), ["b"])
+        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+        self.assertEqual(self.parser.parse("daabbabadef"), None)
+
+class TestRepeatN(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2)
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abdef"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("adef"), None)
+        self.assertEqual(self.parser.parse("dabdef"), None)
+
+class TestOptional(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"])
+        self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"])
+        self.assertEqual(self.parser.parse("ad"), ["a", None, "d"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("aed"), None)
+        self.assertEqual(self.parser.parse("ab"), None)
+        self.assertEqual(self.parser.parse("ac"), None)
+
+class TestIgnore(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abc"), ["a", "c"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("ac"), None)
+
+class TestSepBy(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
+        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
+        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
+        self.assertEqual(self.parser.parse("3"), ["3"])
+        self.assertEqual(self.parser.parse(""), [])
+    def test_failure(self):
+        pass
+
+class TestSepBy1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
+        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
+        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
+        self.assertEqual(self.parser.parse("3"), ["3"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+
+class TestEpsilonP1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+    def test_failure(self):
+        pass
+
+class TestEpsilonP2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.epsilon_p(), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        pass
+
+class TestEpsilonP3(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.epsilon_p())
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        pass
+
+# this has a double-free problem
+#class TestAttrBool(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1])
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
+#        self.assertEqual(self.parser.parse("bb"), ["b", "b"])
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("ab"), None)
+
+class TestAnd1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("0"), ["0"])
+    def test_failure(self):
+        pass
+
+class TestAnd2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1"))
+    def test_success(self):
+        pass
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("0"), None)
+
+class TestAnd3(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("12"), ["1"])
+    def test_failure(self):
+        pass
+
+class TestNot1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a++b"), None)
+
+class TestNot2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"])
+        self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"])
+    def test_failure(self):
+        pass
+
+class TestLeftrec(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.indirect()
+        a = h.ch("a")
+        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
+        self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"])
+    def test_failure(self):
+        pass
+
+class TestRightrec(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.indirect()
+        a = h.ch("a")
+        h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p()))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("aa"), ["a", ["a"]])
+        self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]])
+    def test_failure(self):
+        pass
+
+#class TestAmbiguous(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.indirect()
+#        d = h.ch("d")
+#        p = h.ch("+")
+#        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d))
+#        # this is supposed to be flattened
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("d"), ["d"])
+#        self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"])
+#        self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"])
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("d+"), None)
+

From 9061318dd691359578322736fa131c90305c7005 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sat, 16 Nov 2013 07:56:47 +0100
Subject: [PATCH 07/23] Import combinator and allocator APIs into SWIG. Next is
 glue. Need to fix unions in HParsedToken and HCaseResult.

---
 src/allocator.h            |  4 ++++
 src/bindings/swig/hammer.i |  4 ++++
 src/hammer.h               | 20 +++++++++++---------
 3 files changed, 19 insertions(+), 9 deletions(-)
 create mode 100644 src/bindings/swig/hammer.i

diff --git a/src/allocator.h b/src/allocator.h
index 2dfc14e..a02d97c 100644
--- a/src/allocator.h
+++ b/src/allocator.h
@@ -29,7 +29,11 @@ typedef struct HAllocator_ {
 typedef struct HArena_ HArena ; // hidden implementation
 
 HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
+#ifndef SWIG
 void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) ));
+#else
+void* h_arena_malloc(HArena *arena, size_t count);
+#endif
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
 
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
new file mode 100644
index 0000000..95870e2
--- /dev/null
+++ b/src/bindings/swig/hammer.i
@@ -0,0 +1,4 @@
+%module hammer
+
+%import "hammer/allocator.h"
+%import "hammer/hammer.h"
diff --git a/src/hammer.h b/src/hammer.h
index f5a9e13..e435587 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -29,7 +29,9 @@
 #define BIT_LITTLE_ENDIAN 0x0
 #define BYTE_LITTLE_ENDIAN 0x0
 
+#ifndef HAMMER_INTERNAL__NO_STDARG_H
 typedef int bool;
+#endif // HAMMER_INTERNAL__NO_STDARG_H
 
 typedef struct HParseState_ HParseState;
 
@@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ {
   rtype_t name(__VA_ARGS__) attr;					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr
 
-#ifndef HAMMER_INTERNAL__NO_STDARG_H
+#ifndef SWIG
 #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...)			\
   rtype_t name(__VA_ARGS__, ...);					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
@@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ {
   rtype_t name##__a(void *args[]);					\
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 #else
-#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...)			\
-  rtype_t name(__VA_ARGS__, ...);					\
-  rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
-  rtype_t name##__a(void *args[]);					\
+#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...)  \
+  rtype_t name(params, ...);				  \
+  rtype_t name##__m(HAllocator* mm__, params, ...);    	  \
+  rtype_t name##__a(void *args[]);			 \
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // Note: this drops the attributes on the floor for the __v versions
-#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...)		\
-  rtype_t name(__VA_ARGS__, ...) attr;					\
-  rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr;		\
-  rtype_t name##__a(void *args[]);					\
+#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...)		\
+  rtype_t name(params, ...);				\
+  rtype_t name##__m(HAllocator* mm__, params, ...);       	\
+  rtype_t name##__a(void *args[]);				\
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 #endif // HAMMER_INTERNAL__NO_STDARG_H
 // }}}

From 529aa3e1a3d67272e2c8c0dea4d620252e5c8447 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sat, 16 Nov 2013 08:12:29 +0100
Subject: [PATCH 08/23] fixed nested unions

---
 src/bindings/swig/hammer.i |  1 +
 src/hammer.h               | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index 95870e2..81274c3 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -2,3 +2,4 @@
 
 %import "hammer/allocator.h"
 %import "hammer/hammer.h"
+
diff --git a/src/hammer.h b/src/hammer.h
index e435587..a7046d0 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -70,8 +70,21 @@ typedef struct HBytes_ {
   size_t len;
 } HBytes;
 
+#ifdef SWIG
+typedef union {
+  HBytes bytes;
+  int64_t sint;
+  uint64_t uint;
+  double dbl;
+  float flt;
+  HCountedArray *seq;
+  void *user;
+} HTokenData;
+#endif
+
 typedef struct HParsedToken_ {
   HTokenType token_type;
+#ifndef SWIG
   union {
     HBytes bytes;
     int64_t sint;
@@ -81,6 +94,9 @@ typedef struct HParsedToken_ {
     HCountedArray *seq; // a sequence of HParsedToken's
     void *user;
   };
+#else
+  HTokenData token_data;
+#endif
   size_t index;
   char bit_offset;
 } HParsedToken;
@@ -144,12 +160,23 @@ typedef struct HParserTestcase_ {
   char* output_unambiguous;
 } HParserTestcase;
 
+#ifdef SWIG
+typedef union {
+  const char* actual_results;
+  size_t parse_time;
+} HResultTiming;
+#endif
+
 typedef struct HCaseResult_ {
   bool success;
+#ifndef SWIG
   union {
     const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
     size_t parse_time; // on success, filled in with time for a single parse, in nsec
   };
+#else
+  HResultTiming timestamp;
+#endif
 } HCaseResult;
 
 typedef struct HBackendResults_ {

From c54f63871af927d238f361bf3a3565a32e8ff0a6 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sat, 16 Nov 2013 08:15:01 +0100
Subject: [PATCH 09/23] invoke swig with relative path instead, -I../../

---
 src/bindings/swig/hammer.i | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index 81274c3..357018b 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,5 +1,5 @@
 %module hammer
 
-%import "hammer/allocator.h"
-%import "hammer/hammer.h"
+%import "allocator.h"
+%import "hammer.h"
 

From fc63ee5d1bd1451db8bb80829be07772e452422f Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sat, 16 Nov 2013 20:24:05 +0100
Subject: [PATCH 10/23] SWIG bindings build, compile, and appear to work for
 python. Will port over tests next.

---
 src/bindings/swig/hammer.i | 10 ++++++++--
 src/hammer.h               |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index 357018b..ac8c47f 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,5 +1,11 @@
 %module hammer
 
-%import "allocator.h"
-%import "hammer.h"
+%{
+#include "allocator.h"
+#include "hammer.h"
+#include "internal.h"
+%}
+%include "allocator.h"
+%include "hammer.h"
+
 
diff --git a/src/hammer.h b/src/hammer.h
index a7046d0..d8c613d 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p);
 // {{{ Benchmark functions
 HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases);
 void h_benchmark_report(FILE* stream, HBenchmarkResults* results);
-void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
+//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
 // }}}
 
 #endif // #ifndef HAMMER_HAMMER__H

From a5da5804570a5b2449fcd2b3d06ffbff5b55efe9 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sun, 17 Nov 2013 15:55:38 -0600
Subject: [PATCH 11/23] There's a typemap problem between const uint8_t* and
 target-language strings, but I can create parsers in python and php.

---
 src/bindings/swig/hammer.i | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index ac8c47f..f784f31 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,5 +1,9 @@
 %module hammer
 
+%include "typemaps.i"
+%include "stdint.i"
+
+ // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind.
 %{
 #include "allocator.h"
 #include "hammer.h"
@@ -8,4 +12,5 @@
 %include "allocator.h"
 %include "hammer.h"
 
+%apply const char* { const uint8_t* }
 

From 804b8f7bd1f0c4667e63d1e93de98bd0f6a69e66 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sun, 17 Nov 2013 20:26:51 -0600
Subject: [PATCH 12/23] SWIG python bindings, has same typemap problem as PHP

---
 src/SConscript                 |  2 +-
 src/bindings/python/SConscript | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 src/bindings/python/SConscript

diff --git a/src/SConscript b/src/SConscript
index 9b5c868..cb5d7f6 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -1,7 +1,7 @@
 # -*- python -*-
 Import('env')
 
-bindings = []
+bindings = ['python']
 
 dist_headers = [
     "hammer.h",
diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
new file mode 100644
index 0000000..29d1ad9
--- /dev/null
+++ b/src/bindings/python/SConscript
@@ -0,0 +1,14 @@
+# -*- python -*-
+Import('env')
+
+pythonenv = env.Clone()
+
+pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7'])
+pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
+pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python'])
+
+pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
+
+swig = ['hammer.i']
+
+libhammer_python = pythonenv.SharedLibrary('hammer', swig)
\ No newline at end of file

From 6effa7ae3ebfc8acdd7858facdf456cb1ec50a52 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sun, 17 Nov 2013 20:31:10 -0600
Subject: [PATCH 13/23] fix travis config here too

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 1458ecd..dd66860 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,9 @@ language: c
 compiler:
   - gcc
   - clang
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq swig python-dev
 script: 
   - scons
 notifications:

From 0eab9112587315b7e9089fd022e434412cb22e7a Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sun, 17 Nov 2013 20:41:56 -0600
Subject: [PATCH 14/23] helps to link against the library...

---
 src/bindings/python/SConscript | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index 29d1ad9..2b1c4cc 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -5,6 +5,8 @@ pythonenv = env.Clone()
 
 pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7'])
 pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
+pythonenv.Append(LIBS = ['hammer'])
+pythonenv.Append(LIBPATH = ['../../'])
 pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python'])
 
 pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))

From f1e6e1fa9532043ed515fff6523252e9e5e6a5c9 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Sun, 17 Nov 2013 20:56:03 -0600
Subject: [PATCH 15/23] sync with php-bindings SConscript

---
 src/bindings/python/SConscript | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index 2b1c4cc..a08e55f 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -1,7 +1,7 @@
 # -*- python -*-
 Import('env')
 
-pythonenv = env.Clone()
+pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 
 pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7'])
 pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
@@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 
 swig = ['hammer.i']
 
-libhammer_python = pythonenv.SharedLibrary('hammer', swig)
\ No newline at end of file
+libhammer_python = pythonenv.SharedLibrary('hammer', swig)

From 83fdfd93fda0d583562e35c62ebbe2726efa3da5 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Mon, 18 Nov 2013 17:19:46 -0600
Subject: [PATCH 16/23] typemap to fix conversion from python strings to
 uint8_t*

---
 src/bindings/swig/hammer.i | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index f784f31..bef82b9 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,7 +1,16 @@
 %module hammer
 
-%include "typemaps.i"
 %include "stdint.i"
+%include "typemaps.i"
+%apply char [ANY] { uint8_t [ANY] };
+
+#if defined(SWIGPYTHON)
+%typemap(in) uint8_t* {
+  $1 = (uint8_t*)PyString_AsString($input);
+ }
+#else
+  #warning no "in" typemap defined
+#endif
 
  // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind.
 %{
@@ -12,5 +21,5 @@
 %include "allocator.h"
 %include "hammer.h"
 
-%apply const char* { const uint8_t* }
+
 

From e8b2c17026070e47d00835da3f30a29235d91bb4 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Mon, 18 Nov 2013 21:14:44 -0600
Subject: [PATCH 17/23] python tests baked into scons; h_ch needs the first
 char of its input string as its input

---
 src/bindings/python/SConscript      |   5 +-
 src/bindings/python/hammer_tests.py | 444 ++++++++++++++--------------
 src/bindings/swig/hammer.i          |   5 +-
 3 files changed, 226 insertions(+), 228 deletions(-)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index a08e55f..318103e 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 
 swig = ['hammer.i']
 
-libhammer_python = pythonenv.SharedLibrary('hammer', swig)
+libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
+
+pytestenv = pythonenv.Clone()
+pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE")
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index a56d669..3f0596c 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -4,503 +4,495 @@ import hammer as h
 class TestTokenParser(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.token("95\xa2")
+        cls.parser = h.h_token("95\xa2", 3)
     def test_success(self):
-        self.assertEqual(self.parser.parse("95\xa2"), "95\xa2")
+        self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2")
     def test_partial_fails(self):
-        self.assertEqual(self.parser.parse("95"), None)
+        self.assertEqual(h.h_parse(self.parser, "95", 2), None)
 
 class TestChParser(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser_int = h.ch(0xa2)
-        cls.parser_chr = h.ch("\xa2")
+        cls.parser_int = h.h_ch(0xa2)
+        cls.parser_chr = h.h_ch("\xa2")
     def test_success(self):
-        self.assertEqual(self.parser_int.parse("\xa2"), 0xa2)
-        self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2")
+        self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2)
+        self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2")
     def test_failure(self):
-        self.assertEqual(self.parser_int.parse("\xa3"), None)
-        self.assertEqual(self.parser_chr.parse("\xa3"), None)
+        self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None)
+        self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None)
 
 class TestChRange(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.ch_range("a", "c")
-### this segfaults
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("b"), "b")
+        cls.parser = h.h_ch_range("a", "c")
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("d"), None)
+        self.assertEqual(h.h_parse(self.parser, "d", 1), None)
 
 class TestInt64(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.int64()
+        cls.parser = h.h_int64()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000)
+        self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None)
+        self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None)
 
 class TestInt32(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.int32()
+        cls.parser = h.h_int32()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000)
-        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+        self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\xff\xfe\x00"), None)
-        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+        self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None)
 
 class TestInt16(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.int16()
+        cls.parser = h.h_int16()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\xfe\x00"), -0x200)
-        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+        self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200)
+        self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\xfe"), None)
-        self.assertEqual(self.parser.parse("\x02"), None)
+        self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None)
+        self.assertEqual(h.h_parse(self.parser, "\x02", 1), None)
 
 class TestInt8(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.int8()
+        cls.parser = h.h_int8()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x88"), -0x78)
+        self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78)
     def test_failure(self):
-        self.assertEqual(self.parser.parse(""), None)
+        self.assertEqual(h.h_parse(self.parser, "", 0), None)
 
 class TestUint64(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.uint64()
+        cls.parser = h.h_uint64()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None)
 
 class TestUint32(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.uint32()
+        cls.parser = h.h_uint32()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+        self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None)
 
 class TestUint16(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.uint16()
+        cls.parser = h.h_uint16()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+        self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\x02"), None)
+        self.assertEqual(h.h_parse(self.parser, "\x02", 1), None)
 
 class TestUint8(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.uint8()
+        cls.parser = h.h_uint8()
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x78"), 0x78)
+        self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78)
     def test_failure(self):
-        self.assertEqual(self.parser.parse(""), None)
+        self.assertEqual(h.h_parse(self.parser, "", 0), None)
         
 class TestIntRange(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.int_range(h.uint8(), 3, 10)
+        cls.parser = h.h_int_range(h.h_uint8(), 3, 10)
     def test_success(self):
-        self.assertEqual(self.parser.parse("\x05"), 5)
+        self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5)
     def test_failure(self):
-        self.assertEqual(self.parser.parse("\x0b"), None)
+        self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None)
 
 class TestWhitespace(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.whitespace(h.ch("a"))
+        cls.parser = h.h_whitespace(h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), "a")
-        self.assertEqual(self.parser.parse(" a"), "a")
-        self.assertEqual(self.parser.parse("  a"), "a")
-        self.assertEqual(self.parser.parse("\ta"), "a")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "  a", 3).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("_a"), None)
+        self.assertEqual(h.h_parse(self.parser, "_a", 2), None)
 
 class TestWhitespaceEnd(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.whitespace(h.end_p())
-### this segfaults
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse(""), "")
-#        self.assertEqual(self.parser.parse("  "), "")
+        cls.parser = h.h_whitespace(h.h_end_p())
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string
+        self.assertEqual(h.h_parse(self.parser, "  ", 2).ast, None) # empty string
     def test_failure(self):
-        self.assertEqual(self.parser.parse("  x"), None)
+        self.assertEqual(h.h_parse(self.parser, "  x", 3), None)
 
 class TestLeft(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.left(h.ch("a"), h.ch(" "))
+        cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" "))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a "), "a")
+        self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
-        self.assertEqual(self.parser.parse(" "), None)
-        self.assertEqual(self.parser.parse("ab"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
+        self.assertEqual(h.h_parse(self.parser, " ", 1), None)
+        self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
 
 class TestRight(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.right(h.ch(" "), h.ch("a"))
+        cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(self.parser.parse(" a"), "a")
+        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
-        self.assertEqual(self.parser.parse(" "), None)
-        self.assertEqual(self.parser.parse("ba"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
+        self.assertEqual(h.h_parse(self.parser, " ", 1), None)
+        self.assertEqual(h.h_parse(self.parser, "ba", 2), None)
 
 class TestMiddle(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" "))
+        cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" "))
     def test_success(self):
-        self.assertEqual(self.parser.parse(" a "), "a")
+        self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
-        self.assertEqual(self.parser.parse(" "), None)
-        self.assertEqual(self.parser.parse(" a"), None)
-        self.assertEqual(self.parser.parse("a "), None)
-        self.assertEqual(self.parser.parse(" b "), None)
-        self.assertEqual(self.parser.parse("ba "), None)
-        self.assertEqual(self.parser.parse(" ab"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
+        self.assertEqual(h.h_parse(self.parser, " ", 1), None)
+        self.assertEqual(h.h_parse(self.parser, " a", 2), None)
+        self.assertEqual(h.h_parse(self.parser, "a ", 2), None)
+        self.assertEqual(h.h_parse(self.parser, " b ", 3), None)
+        self.assertEqual(h.h_parse(self.parser, "ba ", 3), None)
+        self.assertEqual(h.h_parse(self.parser, " ab", 3), None)
 
 class TestAction(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x])
-### fails with "corrupted double-linked list"
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("ab"), ["A", "B"])
-#        self.assertEqual(self.parser.parse("AB"), ["A", "B"])
+        cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x])
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"])
+        self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("XX"), None)
+        self.assertEqual(h.h_parse(self.parser, "XX", 2), None)
 
 class TestIn(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.in_("abc")
+        cls.parser = h.h_in("abc", 3)
     def test_success(self):
-        self.assertEqual(self.parser.parse("b"), "b")
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes!
     def test_failure(self):
-        self.assertEqual(self.parser.parse("d"), None)
+        self.assertEqual(h.h_parse(self.parser, "d", 1), None)
 
 class TestNotIn(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.not_in("abc")
+        cls.parser = h.h_not_in("abc", 3)
     def test_success(self):
-        self.assertEqual(self.parser.parse("d"), "d")
+        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes!
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
 
 class TestEndP(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.end_p())
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p())
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("aa"), None)
+        self.assertEqual(h.h_parse(self.parser, "aa", 2), None)
 
 class TestNothingP(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.nothing_p()
+        cls.parser = h.h_nothing_p()
     def test_success(self):
         pass
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
 
 class TestSequence(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
-        self.assertEqual(self.parser.parse("b"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
+        self.assertEqual(h.h_parse(self.parser, "b", 1), None)
 
 class TestSequenceWhitespace(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b")))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b")))
     def test_success(self):
-        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
-        self.assertEqual(self.parser.parse("a b"), ["a", "b"])
-        self.assertEqual(self.parser.parse("a  b"), ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "a  b", 4).ast.token_data.seq, ["a", "b"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a  c"), None)
+        self.assertEqual(h.h_parse(self.parser, "a  c", 4), None)
 
 class TestChoice(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.choice(h.ch("a"), h.ch("b"))
+        cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), "a")
-        self.assertEqual(self.parser.parse("b"), "b")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a")
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("c"), None)
+        self.assertEqual(h.h_parse(self.parser, "c", 1), None)
 
 class TestButNot(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.butnot(h.ch("a"), h.token("ab"))
+        cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), "a")
-        self.assertEqual(self.parser.parse("aa"), "a")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("ab"), None)
+        self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
 
-### fails with malloc() memory corruption
-#class TestButNotRange(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6"))
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("4"), "4")
-### this segfaults
-#    def test_failure(self):
-#        self.assertEqual(self.parser.parse("6"), None)
+class TestButNotRange(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6"))
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4")
+    def test_failure(self):
+        self.assertEqual(h.h_parse(self.parser, "6", 1), None)
 
 class TestDifference(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.difference(h.token("ab"), h.ch("a"))
+        cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("ab"), "ab")
+        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab")
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
 
-#class TestXor(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9"))
-### this segfaults
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("0"), "0")
-#        self.assertEqual(self.parser.parse("9"), "9")
-### fails with "malloc(): smallbin double linked list corrupted"
-#    def test_failure(self):
-#        self.assertEqual(self.parser.parse("5"), None)
-#        self.assertEqual(self.parser.parse("a"), None)
+class TestXor(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9"))
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0")
+        self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9")
+    def test_failure(self):
+        self.assertEqual(h.h_parse(self.parser, "5", 1), None)
+        self.assertEqual(h.h_parse(self.parser, "a", 1), None)
 
 class TestMany(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.many(h.choice(h.ch("a"), h.ch("b")))
+        cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b")))
     def test_success(self):
-        self.assertEqual(self.parser.parse(""), [])
-        self.assertEqual(self.parser.parse("a"), ["a"])
-        self.assertEqual(self.parser.parse("b"), ["b"])
-        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+        self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"])
+        self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"])
     def test_failure(self):
         pass
 
 class TestMany1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b")))
+        cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b")))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), ["a"])
-        self.assertEqual(self.parser.parse("b"), ["b"])
-        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"])
+        self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse(""), None)
-        self.assertEqual(self.parser.parse("daabbabadef"), None)
+        self.assertEqual(h.h_parse(self.parser, "", 0), None)
+        self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None)
 
 class TestRepeatN(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2)
+        cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2)
     def test_success(self):
-        self.assertEqual(self.parser.parse("abdef"), ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("adef"), None)
-        self.assertEqual(self.parser.parse("dabdef"), None)
+        self.assertEqual(h.h_parse(self.parser, "adef", 4), None)
+        self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None)
 
 class TestOptional(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"])
-        self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"])
-        self.assertEqual(self.parser.parse("ad"), ["a", None, "d"])
+        self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"])
+        self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"])
+        self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("aed"), None)
-        self.assertEqual(self.parser.parse("ab"), None)
-        self.assertEqual(self.parser.parse("ac"), None)
+        self.assertEqual(h.h_parse(self.parser, "aed", 3), None)
+        self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
+        self.assertEqual(h.h_parse(self.parser, "ac", 2), None)
 
 class TestIgnore(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("abc"), ["a", "c"])
+        self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("ac"), None)
+        self.assertEqual(h.h_parse(self.parser, "ac", 2), None)
 
 class TestSepBy(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+        cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
     def test_success(self):
-        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
-        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
-        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
-        self.assertEqual(self.parser.parse("3"), ["3"])
-        self.assertEqual(self.parser.parse(""), [])
+        self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"])
+        self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"])
+        self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"])
+        self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"])
+        self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
     def test_failure(self):
         pass
 
 class TestSepBy1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+        cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
     def test_success(self):
-        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
-        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
-        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
-        self.assertEqual(self.parser.parse("3"), ["3"])
+        self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"])
+        self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"])
+        self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"])
+        self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse(""), None)
+        self.assertEqual(h.h_parse(self.parser, "", 0), None)
 
 class TestEpsilonP1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
     def test_failure(self):
         pass
 
 class TestEpsilonP2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.epsilon_p(), h.ch("a"))
+        cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
     def test_failure(self):
         pass
 
 class TestEpsilonP3(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.epsilon_p())
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p())
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
     def test_failure(self):
         pass
 
-# this has a double-free problem
-#class TestAttrBool(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1])
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
-#        self.assertEqual(self.parser.parse("bb"), ["b", "b"])
-#    def test_failure(self):
-#        self.assertEqual(self.parser.parse("ab"), None)
+class TestAttrBool(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1])
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
+        self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"])
+    def test_failure(self):
+        self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
 
 class TestAnd1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0"))
+        cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("0"), ["0"])
+        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"])
     def test_failure(self):
         pass
 
 class TestAnd2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1"))
+        cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1"))
     def test_success(self):
         pass
     def test_failure(self):
-        self.assertEqual(self.parser.parse("0"), None)
+        self.assertEqual(h.h_parse(self.parser, "0", 1), None)
 
 class TestAnd3(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2")))
+        cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2")))
     def test_success(self):
-        self.assertEqual(self.parser.parse("12"), ["1"])
+        self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"])
     def test_failure(self):
         pass
 
 class TestNot1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"])
+        self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"])
     def test_failure(self):
-        self.assertEqual(self.parser.parse("a++b"), None)
+        self.assertEqual(h.h_parse(self.parser, "a++b", 4), None)
 
 class TestNot2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"])
-        self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"])
+        self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"])
+        self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"])
     def test_failure(self):
         pass
 
 class TestLeftrec(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.indirect()
-        a = h.ch("a")
-        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a))
+        cls.parser = h.h_indirect()
+        a = h.h_ch("a")
+        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), "a")
-        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
-        self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
+        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
     def test_failure(self):
         pass
 
 class TestRightrec(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.indirect()
-        a = h.ch("a")
-        h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p()))
+        cls.parser = h.h_indirect()
+        a = h.h_ch("a")
+        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p()))
     def test_success(self):
-        self.assertEqual(self.parser.parse("a"), ["a"])
-        self.assertEqual(self.parser.parse("aa"), ["a", ["a"]])
-        self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]])
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]])
+        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]])
     def test_failure(self):
         pass
 
-#class TestAmbiguous(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.indirect()
-#        d = h.ch("d")
-#        p = h.ch("+")
-#        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d))
-#        # this is supposed to be flattened
-#    def test_success(self):
-#        self.assertEqual(self.parser.parse("d"), ["d"])
-#        self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"])
-#        self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"])
-#    def test_failure(self):
-#        self.assertEqual(self.parser.parse("d+"), None)
+class TestAmbiguous(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.h_indirect()
+        d = h.h_ch("d")
+        p = h.h_ch("+")
+        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
+        # this is supposed to be flattened
+    def test_success(self):
+        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
+        self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
+        self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
+    def test_failure(self):
+        self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
 
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index bef82b9..ad61e49 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -8,8 +8,11 @@
 %typemap(in) uint8_t* {
   $1 = (uint8_t*)PyString_AsString($input);
  }
+%typemap(out) uint8_t* {
+  $result = PyString_FromString((char*)$1);
+ }
 #else
-  #warning no "in" typemap defined
+  #warning no uint8_t* typemaps defined
 #endif
 
  // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind.

From d1b71779e130b4d1dfeab98f2958fa300cc589b5 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Mon, 18 Nov 2013 21:50:28 -0600
Subject: [PATCH 18/23] there's the uint8_t problem (mostly) sorted

---
 src/bindings/python/hammer_tests.py | 10 +++++-----
 src/bindings/swig/hammer.i          | 16 ++++++++++++++--
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index 3f0596c..d1c18db 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_token("95\xa2", 3)
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2")
+        self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2")
     def test_partial_fails(self):
         self.assertEqual(h.h_parse(self.parser, "95", 2), None)
 
@@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"])
+        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
         self.assertEqual(h.h_parse(self.parser, "b", 1), None)
@@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase):
 class TestButNot(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab"))
+        cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2))
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
         self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a")
@@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase):
 class TestNot1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b"))
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"])
     def test_failure(self):
@@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase):
 class TestNot2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b"))
+        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b"))
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"])
         self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"])
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index ad61e49..412743a 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -11,6 +11,20 @@
 %typemap(out) uint8_t* {
   $result = PyString_FromString((char*)$1);
  }
+%typemap(in) uint8_t {
+  if (PyInt_Check($input)) {
+    $1 = PyInt_AsLong($input);
+  }
+  else if (!PyString_Check($input)) {
+    PyErr_SetString(PyExc_ValueError, "Expecting a string");
+    return NULL;
+  } else {
+    $1 = *(uint8_t*)PyString_AsString($input);
+  }
+ }
+%typemap(out) HBytes* {
+  $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
+ }
 #else
   #warning no uint8_t* typemaps defined
 #endif
@@ -24,5 +38,3 @@
 %include "allocator.h"
 %include "hammer.h"
 
-
-

From f3ce2fc7de48a7be27f7c9cabfa710ec56a9fe41 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Tue, 19 Nov 2013 00:18:25 -0600
Subject: [PATCH 19/23] h_sequence, h_choice and HCountedSequence are all
 having weird issues, but tests are in a workable format now

---
 src/bindings/python/hammer_tests.py | 157 ++++++++++++++--------------
 src/bindings/swig/hammer.i          |  19 +++-
 2 files changed, 98 insertions(+), 78 deletions(-)

diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index d1c18db..ee797d9 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase):
         cls.parser_chr = h.h_ch("\xa2")
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2)
-        self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2")
+        self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None)
         self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None)
@@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_ch_range("a", "c")
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b")
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "d", 1), None)
 
@@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_whitespace(h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
-        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a")
-        self.assertEqual(h.h_parse(self.parser, "  a", 3).ast.token_data.bytes, "a")
-        self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a"))
+        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a"))
+        self.assertEqual(h.h_parse(self.parser, "  a", 3).ast.token_data.uint, ord("a"))
+        self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "_a", 2), None)
 
@@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" "))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
         self.assertEqual(h.h_parse(self.parser, " ", 1), None)
@@ -154,7 +154,7 @@ class TestRight(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
         self.assertEqual(h.h_parse(self.parser, " ", 1), None)
@@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" "))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
         self.assertEqual(h.h_parse(self.parser, " ", 1), None)
@@ -190,7 +190,7 @@ class TestIn(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_in("abc", 3)
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes!
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) 
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "d", 1), None)
 
@@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p())
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
+        ### failing: parses a single 'a', dunno why
         self.assertEqual(h.h_parse(self.parser, "aa", 2), None)
 
 class TestNothingP(unittest.TestCase):
@@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
         self.assertEqual(h.h_parse(self.parser, "b", 1), None)
@@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b")))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
-        self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"])
-        self.assertEqual(h.h_parse(self.parser, "a  b", 4).ast.token_data.seq, ["a", "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a  b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a  c", 4), None)
 
@@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a")
-        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a"))
+        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "c", 1), None)
 
@@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
-        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a")
+        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a"))
+        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
 
@@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4")
+        self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "6", 1), None)
 
@@ -285,8 +286,8 @@ class TestXor(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0")
-        self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9")
+        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0"))
+        self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "5", 1), None)
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
@@ -297,9 +298,9 @@ class TestMany(unittest.TestCase):
         cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b")))
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
-        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"])
-        self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]])
     def test_failure(self):
         pass
 
@@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b")))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
-        self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"])
-        self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "", 0), None)
         self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None)
@@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"])
-        self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"])
-        self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "aed", 3), None)
         self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
@@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "ac", 2), None)
 
@@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"])
-        self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"])
-        self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"])
-        self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]])
         self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
     def test_failure(self):
         pass
@@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"])
-        self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"])
-        self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"])
-        self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "", 0), None)
 
@@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
         pass
 
@@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
         pass
 
@@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p())
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
         pass
 
@@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0"))
     def test_success(self):
+        ### failing: [] != ["0"]. Token type is sequence.
         self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"])
     def test_failure(self):
         pass
@@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase):
     def test_success(self):
         pass
     def test_failure(self):
-        self.assertEqual(h.h_parse(self.parser, "0", 1), None)
+        ### failing: [] is not None, parse should have failed
+        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None)
 
 class TestAnd3(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2")))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]])
     def test_failure(self):
         pass
 
@@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a++b", 4), None)
 
@@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b"))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"])
         self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"])
     def test_failure(self):
         pass
 
-class TestLeftrec(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.parser = h.h_indirect()
-        a = h.h_ch("a")
-        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
-    def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
-        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
-        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
-    def test_failure(self):
-        pass
+### this is commented out for packrat in C ...
+#class TestLeftrec(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.h_indirect()
+#        a = h.h_ch("a")
+#        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
+#    def test_success(self):
+#        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
+#        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
+#        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
+#    def test_failure(self):
+#        pass
 
 class TestRightrec(unittest.TestCase):
     @classmethod
@@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase):
         a = h.h_ch("a")
         h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p()))
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"])
-        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]])
-        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
     def test_failure(self):
         pass
 
-class TestAmbiguous(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.parser = h.h_indirect()
-        d = h.h_ch("d")
-        p = h.h_ch("+")
-        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
-        # this is supposed to be flattened
-    def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
-        self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
-        self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
-    def test_failure(self):
-        self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
+### this is just for GLR
+#class TestAmbiguous(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.h_indirect()
+#        d = h.h_ch("d")
+#        p = h.h_ch("+")
+#        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
+#        # this is supposed to be flattened
+#    def test_success(self):
+#        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
+#        self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
+#        self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
+#    def test_failure(self):
+#        self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
 
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index 412743a..f97985d 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,10 +1,11 @@
 %module hammer
 
 %include "stdint.i"
-%include "typemaps.i"
-%apply char [ANY] { uint8_t [ANY] };
+ //%include "typemaps.i"
+ //%apply char [ANY] { uint8_t [ANY] };
 
 #if defined(SWIGPYTHON)
+%ignore HCountedArray_;
 %typemap(in) uint8_t* {
   $1 = (uint8_t*)PyString_AsString($input);
  }
@@ -22,9 +23,23 @@
     $1 = *(uint8_t*)PyString_AsString($input);
   }
  }
+/*
+%typemap(out) uint8_t {
+  $result = PyString_FromString(&$1);
+ }
+*/
 %typemap(out) HBytes* {
   $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
  }
+%typemap(out) struct HCountedArray_* {
+  int i;
+  $result = PyList_New($1->used);
+  for (i=0; i<$1->used; i++) {
+    HParsedToken *t = $1->elements[i];
+    PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0);
+    PyList_SetItem($result, i, o);
+  }
+ }
 #else
   #warning no uint8_t* typemaps defined
 #endif

From cec4659baa148de18403db5c305655453b1b019b Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Tue, 19 Nov 2013 02:41:45 -0600
Subject: [PATCH 20/23] kindasorta working. all tests (except h_action and
 h_attr_bool, need typemaps) have passed at one time or another, but some
 segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not,
 maybe h_choice seem culprity.

---
 src/bindings/python/SConscript      |   4 +-
 src/bindings/python/hammer_tests.py | 179 ++++++++++++++--------------
 src/bindings/swig/hammer.i          |  26 +++-
 3 files changed, 116 insertions(+), 93 deletions(-)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index 318103e..718f8b9 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -16,4 +16,6 @@ swig = ['hammer.i']
 libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
 
 pytestenv = pythonenv.Clone()
-pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE")
+pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE")
+
+Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) 
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index ee797d9..c776520 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase):
         self.assertEqual(h.h_parse(self.parser, "ba ", 3), None)
         self.assertEqual(h.h_parse(self.parser, " ab", 3), None)
 
-class TestAction(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x])
-    def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"])
-        self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"])
-    def test_failure(self):
-        self.assertEqual(h.h_parse(self.parser, "XX", 2), None)
+# class TestAction(unittest.TestCase):
+#     @classmethod
+#     def setUpClass(cls):
+#         cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x])
+#     def test_success(self):
+#         self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"])
+#         self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"])
+#     def test_failure(self):
+#         self.assertEqual(h.h_parse(self.parser, "XX", 2), None)
 
 class TestIn(unittest.TestCase):
     @classmethod
@@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase):
     def setUpClass(cls):
         cls.parser = h.h_not_in("abc", 3)
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes!
+        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d"))
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1), None)
 
 class TestEndP(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p())
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
-        ### failing: parses a single 'a', dunno why
         self.assertEqual(h.h_parse(self.parser, "aa", 2), None)
 
 class TestNothingP(unittest.TestCase):
@@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase):
 class TestSequence(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
@@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase):
 class TestSequenceWhitespace(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b")))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
@@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase):
 class TestChoice(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b"))
+        cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a"))
         self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b"))
@@ -295,7 +294,7 @@ class TestXor(unittest.TestCase):
 class TestMany(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b")))
+        cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]))
     def test_success(self):
         self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
@@ -307,7 +306,7 @@ class TestMany(unittest.TestCase):
 class TestMany1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b")))
+        cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]))
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]])
@@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase):
 class TestRepeatN(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2)
+        cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2)
     def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"])
+        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "adef", 4), None)
         self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None)
@@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase):
 class TestOptional(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]])
-        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]])
+        ### FIXME check this out in repl, what does tree look like
+        #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]])
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "aed", 3), None)
         self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
@@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase):
 class TestIgnore(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]])
     def test_failure(self):
@@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase):
 class TestSepBy(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
+        cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(","))
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
@@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase):
 class TestSepBy1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
+        cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(","))
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
@@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase):
     def test_failure(self):
         self.assertEqual(h.h_parse(self.parser, "", 0), None)
 
+### segfaults
 class TestEpsilonP1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
     def test_failure(self):
@@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase):
 class TestEpsilonP2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a"))
+        cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
@@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase):
 class TestEpsilonP3(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p())
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
     def test_failure(self):
         pass
 
-class TestAttrBool(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1])
-    def test_success(self):
-        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
-        self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"])
-    def test_failure(self):
-        self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
+# class TestAttrBool(unittest.TestCase):
+#     @classmethod
+#     def setUpClass(cls):
+#         cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1])
+#     def test_success(self):
+#         self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
+#         self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"])
+#     def test_failure(self):
+#         self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
 
 class TestAnd1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0"))
+        cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None])
     def test_success(self):
-        ### failing: [] != ["0"]. Token type is sequence.
-        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"])
+         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]])
     def test_failure(self):
         pass
 
 class TestAnd2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1"))
+        cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None])
     def test_success(self):
         pass
     def test_failure(self):
-        ### failing: [] is not None, parse should have failed
-        self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None)
+        self.assertEqual(h.h_parse(self.parser, "0", 1), None)
 
 class TestAnd3(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2")))
+        cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]])
     def test_failure(self):
@@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase):
 class TestNot1(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None])
     def test_success(self):
         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]])
     def test_failure(self):
@@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase):
 class TestNot2(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b"))
+        cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None])
     def test_success(self):
-        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"])
-        self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"])
+        tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq
+        tree[1] = tree[1].token_data.seq[0]
+        self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]])
+        tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq
+        tree[0] = chr(tree[0].token_data.uint)
+        tree[1] = tree[1].token_data.bytes
+        tree[2] = chr(tree[2].token_data.uint)
+        self.assertEqual(tree, ["a", "++", "b"])
     def test_failure(self):
         pass
 
-### this is commented out for packrat in C ...
-#class TestLeftrec(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.h_indirect()
-#        a = h.h_ch("a")
-#        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
-#    def test_success(self):
-#        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
-#        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
-#        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
-#    def test_failure(self):
-#        pass
+# ### this is commented out for packrat in C ...
+# #class TestLeftrec(unittest.TestCase):
+# #    @classmethod
+# #    def setUpClass(cls):
+# #        cls.parser = h.h_indirect()
+# #        a = h.h_ch("a")
+# #        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
+# #    def test_success(self):
+# #        self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
+# #        self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
+# #        self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
+# #    def test_failure(self):
+# #        pass
 
-class TestRightrec(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.parser = h.h_indirect()
-        a = h.h_ch("a")
-        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p()))
-    def test_success(self):
-        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
-        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
-        self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
-    def test_failure(self):
-        pass
+# class TestRightrec(unittest.TestCase):
+#     @classmethod
+#     def setUpClass(cls):
+#         cls.parser = h.h_indirect()
+#         a = h.h_ch("a")
+#         h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()]))
+#     def test_success(self):
+#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
+#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
+#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
+#     def test_failure(self):
+#         pass
 
-### this is just for GLR
-#class TestAmbiguous(unittest.TestCase):
-#    @classmethod
-#    def setUpClass(cls):
-#        cls.parser = h.h_indirect()
-#        d = h.h_ch("d")
-#        p = h.h_ch("+")
-#        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
-#        # this is supposed to be flattened
-#    def test_success(self):
-#        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
-#        self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
-#        self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
-#    def test_failure(self):
-#        self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
+# ### this is just for GLR
+# #class TestAmbiguous(unittest.TestCase):
+# #    @classmethod
+# #    def setUpClass(cls):
+# #        cls.parser = h.h_indirect()
+# #        d = h.h_ch("d")
+# #        p = h.h_ch("+")
+# #        h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
+# #        # this is supposed to be flattened
+# #    def test_success(self):
+# #        self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
+# #        self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
+# #        self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
+# #    def test_failure(self):
+# #        self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
 
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
index f97985d..81a9dd1 100644
--- a/src/bindings/swig/hammer.i
+++ b/src/bindings/swig/hammer.i
@@ -1,4 +1,5 @@
 %module hammer
+%nodefaultctor;
 
 %include "stdint.i"
  //%include "typemaps.i"
@@ -7,11 +8,31 @@
 #if defined(SWIGPYTHON)
 %ignore HCountedArray_;
 %typemap(in) uint8_t* {
+  Py_INCREF($input);
   $1 = (uint8_t*)PyString_AsString($input);
  }
 %typemap(out) uint8_t* {
   $result = PyString_FromString((char*)$1);
  }
+%typemap(in) void*[] {
+  if (PyList_Check($input)) {
+    Py_INCREF($input);
+    int size = PyList_Size($input);
+    int i = 0;
+    int res = 0;
+    $1 = (void**)malloc(size*sizeof(HParser*));
+    for (i=0; i<size; i++) {
+      PyObject *o = PyList_GetItem($input, i);
+      res = SWIG_ConvertPtr(o, &($1[i]), SWIGTYPE_p_HParser_, 0 | 0);
+      if (!SWIG_IsOK(res)) {
+	SWIG_exception_fail(SWIG_ArgError(res), "that wasn't an HParser" );
+      }
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError, "__a functions take lists of parsers as their argument");
+    return NULL;
+  }
+ }
 %typemap(in) uint8_t {
   if (PyInt_Check($input)) {
     $1 = PyInt_AsLong($input);
@@ -23,11 +44,6 @@
     $1 = *(uint8_t*)PyString_AsString($input);
   }
  }
-/*
-%typemap(out) uint8_t {
-  $result = PyString_FromString(&$1);
- }
-*/
 %typemap(out) HBytes* {
   $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
  }

From 656f4546ddbdfb405ae66845484d1b43cd5fe250 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Tue, 19 Nov 2013 03:45:49 -0600
Subject: [PATCH 21/23] travis needs python-nose to run nosetests

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index dd66860..2276989 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ compiler:
   - clang
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install -qq swig python-dev
+  - sudo apt-get install -qq swig python-dev python-nose
 script: 
   - scons
 notifications:

From 62af9f622233af22c0ac58b94e58157bfd5fabd5 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Tue, 19 Nov 2013 17:26:01 -0600
Subject: [PATCH 22/23] more verbose tests; should run w/o hammer installed
 system-wide

---
 src/bindings/python/SConscript      | 12 +++++++-----
 src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index 718f8b9..30d387c 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -1,21 +1,23 @@
 # -*- python -*-
-Import('env')
+import os.path
+Import('env libhammer_shared')
 
 pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 
 pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7'])
 pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
-pythonenv.Append(LIBS = ['hammer'])
-pythonenv.Append(LIBPATH = ['../../'])
+#pythonenv.Append(LIBS = ['hammer'])
+#pythonenv.Append(LIBPATH = ['../../'])
 pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python'])
 
 pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 
 swig = ['hammer.i']
 
-libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
+libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_')
 
 pytestenv = pythonenv.Clone()
-pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE")
+pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0]))
+pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE")
 
 Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) 
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index c776520..9d0c487 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase):
 # #    def test_failure(self):
 # #        pass
 
-# class TestRightrec(unittest.TestCase):
-#     @classmethod
-#     def setUpClass(cls):
-#         cls.parser = h.h_indirect()
-#         a = h.h_ch("a")
-#         h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()]))
-#     def test_success(self):
-#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
-#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
-#         self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
-#     def test_failure(self):
-#         pass
+class TestARightrec(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.h_indirect()
+        a = h.h_ch("a")
+        h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None]))
+    def test_success(self):
+        tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq
+        self.assertEqual(tree[0].token_data.uint, ord("a"))
+        tree = h_parse(self.parser, "aa", 2).ast.token_data.seq
+        self.assertEqual(tree[0].token_data.uint, ord("a"))
+        self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
+        tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq
+        self.assertEqual(tree[0].token_data.uint, ord("a"))
+        self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
+        self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a"))
+    def test_failure(self):
+        pass
 
 # ### this is just for GLR
 # #class TestAmbiguous(unittest.TestCase):

From 10c8b0bd22bd1f172fb04aa2434e40035accd626 Mon Sep 17 00:00:00 2001
From: "Meredith L. Patterson" <mlp@thesmartpolitenerd.com>
Date: Tue, 19 Nov 2013 19:00:58 -0600
Subject: [PATCH 23/23] fixed the segfault! hand-initialized HParser needed
 PB_MIN set.

---
 src/bindings/python/SConscript      | 6 +++---
 src/bindings/python/hammer_tests.py | 8 ++++----
 src/parsers/choice.c                | 4 +++-
 src/parsers/sequence.c              | 4 +++-
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
index 30d387c..0bde353 100644
--- a/src/bindings/python/SConscript
+++ b/src/bindings/python/SConscript
@@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 
 pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7'])
 pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
-#pythonenv.Append(LIBS = ['hammer'])
-#pythonenv.Append(LIBPATH = ['../../'])
+pythonenv.Append(LIBS = ['hammer'])
+pythonenv.Append(LIBPATH = ['../../'])
 pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python'])
 
 pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 
 swig = ['hammer.i']
 
-libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_')
+libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
 
 pytestenv = pythonenv.Clone()
 pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0]))
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
index 9d0c487..41cd13f 100644
--- a/src/bindings/python/hammer_tests.py
+++ b/src/bindings/python/hammer_tests.py
@@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase):
 # #    def test_failure(self):
 # #        pass
 
-class TestARightrec(unittest.TestCase):
+class TestRightrec(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.parser = h.h_indirect()
@@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase):
     def test_success(self):
         tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq
         self.assertEqual(tree[0].token_data.uint, ord("a"))
-        tree = h_parse(self.parser, "aa", 2).ast.token_data.seq
+        tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq
         self.assertEqual(tree[0].token_data.uint, ord("a"))
         self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
-        tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq
+        tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq
         self.assertEqual(tree[0].token_data.uint, ord("a"))
         self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
-        self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a"))
+        self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a"))
     def test_failure(self):
         pass
 
diff --git a/src/parsers/choice.c b/src/parsers/choice.c
index 6db1378..bfc3f90 100644
--- a/src/parsers/choice.c
+++ b/src/parsers/choice.c
@@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) {
 
   s->len = len;
   HParser *ret = h_new(HParser, 1);
-  ret->vtable = &choice_vt; ret->env = (void*)s;
+  ret->vtable = &choice_vt; 
+  ret->env = (void*)s;
+  ret->backend = PB_MIN;
   return ret;
 }
diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c
index eff4610..281b78f 100644
--- a/src/parsers/sequence.c
+++ b/src/parsers/sequence.c
@@ -159,6 +159,8 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
   
   s->len = len;
   HParser *ret = h_new(HParser, 1);
-  ret->vtable = &sequence_vt; ret->env = (void*)s;
+  ret->vtable = &sequence_vt; 
+  ret->env = (void*)s; 
+  ret->backend = PB_MIN;
   return ret;
 }