merging in aegis' JNI bindings

src/Makefile

@@ -42,6 +42,8 @@ HAMMER_PARTS := \
 	benchmark.o \
 	cfgrammar.o \
 	actions.o \
+	compile.o \
+	glue.o \
 	$(PARSERS:%=parsers/%.o) \
 	$(BACKENDS:%=backends/%.o)
 
@@ -50,6 +52,7 @@ TESTS := t_benchmark.o \
 	 t_bitwriter.o \
 	 t_parser.o \
 	 t_grammar.o \
+	 t_misc.o \
 	 test_suite.o
 
 OUTPUTS := libhammer.a \
@@ -65,12 +68,15 @@ include ../common.mk
 $(TESTS): CFLAGS += $(TEST_CFLAGS)
 $(TESTS): LDFLAGS += $(TEST_LDFLAGS)
 
+CFLAGS += -fPIC
+
 all: libhammer.a
 
 libhammer.a: $(HAMMER_PARTS)
 
 bitreader.o: test_suite.h
 hammer.o: hammer.h
+glue.o: hammer.h glue.h
 
 all: libhammer.a
 

src/backends/packrat.c

@@ -86,14 +86,18 @@ HParserCacheValue* recall(HParserCacheKey *k, HParseState *state) {
 void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) {
   if (!rec_detect->head) {
     HRecursionHead *some = a_new(HRecursionHead, 1);
-    some->head_parser = p; some->involved_set = NULL; some->eval_set = NULL;
+    some->head_parser = p;
+    some->involved_set = h_slist_new(state->arena);
+    some->eval_set = NULL;
     rec_detect->head = some;
   }
   assert(state->lr_stack->head != NULL);
-  HLeftRec *lr = state->lr_stack->head->elem;
-  while (lr && lr->rule != p) {
+  HSlistNode *head = state->lr_stack->head;
+  HLeftRec *lr;
+  while (head && (lr = head->elem)->rule != p) {
     lr->head = rec_detect->head;
     h_slist_push(lr->head->involved_set, (void*)lr->rule);
+    head = head->next;
   }
 }
 
@@ -110,7 +114,7 @@ HParseResult* grow(HParserCacheKey *k, HParseState *state, HRecursionHead *head)
   HParseResult *old_res = old_cached->right->result;
   
   // reset the eval_set of the head of the recursion at each beginning of growth
-  head->eval_set = head->involved_set;
+  head->eval_set = h_slist_copy(head->involved_set);
   HParseResult *tmp_res = perform_lowlevel_parse(state, k->parser);
 
   if (tmp_res) {

src/datastructures.c

@@ -41,6 +41,26 @@ HSlist* h_slist_new(HArena *arena) {
   return ret;
 }
 
+HSlist* h_slist_copy(HSlist *slist) {
+  HSlist *ret = h_slist_new(slist->arena);
+  HSlistNode *head = slist->head;
+  HSlistNode *tail;
+  if (head != NULL) {
+    h_slist_push(ret, head->elem);
+    tail = ret->head;
+    head = head->next;
+  }
+  while (head != NULL) {
+    // append head item to tail in a new node
+    HSlistNode *node = h_arena_malloc(slist->arena, sizeof(HSlistNode));
+    node->elem = head->elem;
+    node->next = NULL;
+    tail = tail->next = node;
+    head = head->next;
+  }
+  return ret;
+}
+
 void* h_slist_pop(HSlist *slist) {
   HSlistNode *head = slist->head;
   if (!head)

src/glue.c

@@ -0,0 +1,177 @@
+#include "glue.h"
+#include "../src/internal.h"  // for h_carray_*
+
+
+// The action equivalent of h_ignore.
+const HParsedToken *h_act_ignore(const HParseResult *p)
+{
+  return NULL;
+}
+
+// Helper to build HAction's that pick one index out of a sequence.
+const HParsedToken *h_act_index(int i, const HParseResult *p)
+{
+    if(!p) return NULL;
+
+    const HParsedToken *tok = p->ast;
+
+    if(!tok || tok->token_type != TT_SEQUENCE)
+        return NULL;
+
+    const HCountedArray *seq = tok->seq;
+    size_t n = seq->used;
+
+    if(i<0 || (size_t)i>=n)
+        return NULL;
+    else
+        return tok->seq->elements[i];
+}
+
+// Action version of h_seq_flatten.
+const HParsedToken *h_act_flatten(const HParseResult *p) {
+  return h_seq_flatten(p->arena, p->ast);
+}
+
+// Low-level helper for the h_make family.
+HParsedToken *h_make_(HArena *arena, HTokenType type)
+{
+  HParsedToken *ret = h_arena_malloc(arena, sizeof(HParsedToken));
+  ret->token_type = type;
+  return ret;
+}
+
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value)
+{
+  assert(type >= TT_USER);
+  HParsedToken *ret = h_make_(arena, type);
+  ret->user = value;
+  return ret;
+}
+
+HParsedToken *h_make_seq(HArena *arena)
+{
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new(arena);
+  return ret;
+}
+
+HParsedToken *h_make_seqn(HArena *arena, size_t n)
+{
+  HParsedToken *ret = h_make_(arena, TT_SEQUENCE);
+  ret->seq = h_carray_new_sized(arena, n);
+  return ret;
+}
+
+HParsedToken *h_make_bytes(HArena *arena, size_t len)
+{
+  HParsedToken *ret = h_make_(arena, TT_BYTES);
+  ret->bytes.len = len;
+  ret->bytes.token = h_arena_malloc(arena, len);
+  return ret;
+}
+
+HParsedToken *h_make_sint(HArena *arena, int64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_SINT);
+  ret->sint = val;
+  return ret;
+}
+
+HParsedToken *h_make_uint(HArena *arena, uint64_t val)
+{
+  HParsedToken *ret = h_make_(arena, TT_UINT);
+  ret->uint = val;
+  return ret;
+}
+
+// XXX -> internal
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
+{
+  assert(i < a->used);
+  return a->elements[i];
+}
+
+size_t h_seq_len(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->used;
+}
+
+HParsedToken **h_seq_elements(const HParsedToken *p)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return p->seq->elements;
+}
+
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i)
+{
+  assert(p != NULL);
+  assert(p->token_type == TT_SEQUENCE);
+  return h_carray_index(p->seq, i);
+}
+
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...)
+{
+  va_list va;
+
+  va_start(va, i);
+  HParsedToken *ret = h_seq_index_vpath(p, i, va);
+  va_end(va);
+
+  return ret;
+}
+
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va)
+{
+  HParsedToken *ret = h_seq_index(p, i);
+  int j;
+
+  while((j = va_arg(va, int)) >= 0)
+    ret = h_seq_index(p, j);
+
+  return ret;
+}
+
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+
+  h_carray_append(xs->seq, (HParsedToken *)x);
+}
+
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys)
+{
+  assert(xs != NULL);
+  assert(xs->token_type == TT_SEQUENCE);
+  assert(ys != NULL);
+  assert(ys->token_type == TT_SEQUENCE);
+
+  for(size_t i=0; i<ys->seq->used; i++)
+    h_carray_append(xs->seq, ys->seq->elements[i]);
+}
+
+// Flatten nested sequences. Always returns a sequence.
+// If input element is not a sequence, returns it as a singleton sequence.
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
+{
+  assert(p != NULL);
+
+  HParsedToken *ret = h_make_seq(arena);
+  switch(p->token_type) {
+  case TT_SEQUENCE:
+    // Flatten and append all.
+    for(size_t i; i<p->seq->used; i++) {
+      h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
+    }
+    break;
+  default:
+    // Make singleton sequence.
+    h_seq_snoc(ret, p);
+    break;
+  }
+
+  return ret;
+}

src/glue.h

@@ -0,0 +1,253 @@
+//
+// API additions for writing grammar and semantic actions more concisely
+//
+//
+// Quick Overview:
+//
+// Grammars can be succinctly specified with the family of H_RULE macros.
+// H_RULE defines a plain parser variable. H_ARULE additionally attaches a
+// semantic action; H_VRULE attaches a validation. H_AVRULE and H_VARULE
+// combine both.
+//
+// A few standard semantic actions are defined below. The H_ACT_APPLY macro
+// allows semantic actions to be defined by "partial application" of
+// a generic action to fixed paramters.
+//
+// The definition of more complex semantic actions will usually consist of
+// extracting data from the given parse tree and constructing a token of custom
+// type to represent the result. A number of functions and convenience macros
+// are provided to capture the most common cases and idioms.
+//
+// See the leading comment blocks on the sections below for more details.
+//
+
+#ifndef HAMMER_GLUE__H
+#define HAMMER_GLUE__H
+
+#include <assert.h>
+#include "hammer.h"
+
+
+//
+// Grammar specification
+//
+// H_RULE is simply a short-hand for the typical declaration and definition of
+// a parser variable. See its plain definition below. The goal is to save
+// horizontal space as well as to provide a clear and unified look together with
+// the other macro variants that stays close to an abstract PEG or BNF grammar.
+// The latter goal is more specifically enabled by H_ARULE, H_VRULE, and their
+// combinations as they allow the definition of syntax to be given without
+// intermingling it with the semantic specifications.
+//
+// H_ARULE defines a variable just like H_RULE but attaches a semantic action
+// to the result of the parser via h_action. The action is expected to be
+// named act_<rulename>.
+//
+// H_VRULE is analogous to H_ARULE but attaches a validation via h_attr_bool.
+// The validation is expected to be named validate_<rulename>.
+//
+// H_VARULE combines H_RULE with both an action and a validation. The action is
+// attached before the validation, i.e. the validation receives as input the
+// result of the action.
+//
+// H_AVRULE is like H_VARULE but the action is attached outside the validation,
+// i.e. the validation receives the uninterpreted AST as input.
+//
+
+
+#define H_RULE(rule, def) const HParser *rule = def
+#define H_ARULE(rule, def) const HParser *rule = h_action(def, act_ ## rule)
+#define H_VRULE(rule, def) const HParser *rule = \
+  h_attr_bool(def, validate_ ## rule)
+#define H_VARULE(rule, def) const HParser *rule = \
+  h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule)
+#define H_AVRULE(rule, def) const HParser *rule = \
+  h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
+
+
+//
+// Pre-fab semantic actions
+//
+// A collection of generally useful semantic actions is provided.
+//
+// h_act_ignore is the action equivalent of the parser combinator h_ignore. It
+// simply causes the AST it is applied to to be replaced with NULL. This most
+// importantly causes it to be elided from the result of a surrounding
+// h_sequence.
+//
+// h_act_index is of note as it is not itself suitable to be passed to
+// h_action. It is parameterized by an index to be picked from a sequence
+// token. It must be wrapped in a proper HAction to be used. The H_ACT_APPLY
+// macro provides a concise way to define such a parameter-application wrapper.
+//
+// h_act_flatten acts on a token of possibly nested sequences by recursively
+// flattening it into a single sequence. Cf. h_seq_flatten below.
+//
+// H_ACT_APPLY implements "partial application" for semantic actions. It
+// defines a new action that supplies given parameters to a parameterized
+// action such as h_act_index.
+//
+
+const HParsedToken *h_act_ignore(const HParseResult *p);
+const HParsedToken *h_act_index(int i, const HParseResult *p);
+const HParsedToken *h_act_flatten(const HParseResult *p);
+
+// Define 'myaction' as a specialization of 'paction' by supplying the leading
+// parameters.
+#define H_ACT_APPLY(myaction, paction, ...) \
+  const HParsedToken *myaction(const HParseResult *p) { \
+    return paction(__VA_ARGS__, p); \
+  }
+
+
+//
+// Working with HParsedTokens
+//
+// The type HParsedToken represents a dynamically-typed universe of values.
+// Declared below are constructors to turn ordinary values into their
+// HParsedToken equivalents, extractors to retrieve the original values from
+// inside an HParsedToken, and functions that inspect and modify tokens of
+// sequence type directly.
+//
+// In addition, there are a number of short-hand macros that work with some
+// conventions to eliminate common boilerplate. These conventions are listed
+// below. Be sure to follow them if you want to use the respective macros.
+//
+//  * The single argument to semantic actions should be called 'p'.
+//
+//    The H_MAKE macros suppy 'p->arena' to their underlying h_make
+//    counterparts. The H_FIELD macros supply 'p->ast' to their underlying
+//    H_INDEX counterparts.
+//
+//  * For each custom token type, there should be a typedef for the
+//    corresponding value type.
+//
+//    H_CAST, H_INDEX and H_FIELD cast the void * user field of such a token to
+//    a pointer to the given type. 
+//
+//  * For each custom token type, say 'foo_t', there must be an integer
+//    constant 'TT_foo_t' to identify the token type. This constant must have a
+//    value greater or equal than TT_USER.
+//
+//    One idiom is to define an enum for all custom token types and to assign a
+//    value of TT_USER to the first element. This can be viewed as extending
+//    the HTokenType enum.
+//
+//    The H_MAKE and H_ASSERT macros derive the name of the token type constant
+//    from the given type name.
+//
+//
+// The H_ALLOC macro is useful for allocating values of custom token types.
+//
+// The H_MAKE family of macros construct tokens of a given type. The native
+// token types are indicated by a corresponding suffix such as in H_MAKE_SEQ.
+// The form with no suffix is used for custom token types. This convention is
+// also used for other macro and function families.
+//
+// The H_ASSERT family simply asserts that a given token has the expected type.
+// It mainly serves as an implementation aid for H_CAST. Of note in that regard
+// is that, unlike the standard 'assert' macro, these form _expressions_ that
+// return the value of their token argument; thus they can be used in a
+// "pass-through" fashion inside other expressions.
+//
+// The H_CAST family combines a type assertion with access to the
+// statically-typed value inside a token.
+//
+// A number of functions h_seq_* operate on and inspect sequence tokens.
+// Note that H_MAKE_SEQ takes no arguments and constructs an empty sequence.
+// Therefore there are h_seq_snoc and h_seq_append to build up sequences.
+//
+// The macro families H_FIELD and H_INDEX combine index access on a sequence
+// with a cast to the appropriate result type. H_FIELD is used to access the
+// elements of the argument token 'p' in an action. H_INDEX allows any sequence
+// token to be specified. Both macro families take an arbitrary number of index
+// arguments, giving access to elements in nested sequences by path.
+// These macros are very useful to avoid spaghetti chains of unchecked pointer
+// dereferences.
+//
+
+// Standard short-hand for arena-allocating a variable in a semantic action.
+#define H_ALLOC(TYP)  ((TYP *) h_arena_malloc(p->arena, sizeof(TYP)))
+
+// Token constructors...
+
+HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
+HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
+HParsedToken *h_make_seqn(HArena *arena, size_t n);  // Makes empty sequence of expected size n.
+HParsedToken *h_make_bytes(HArena *arena, size_t len);
+HParsedToken *h_make_sint(HArena *arena, int64_t val);
+HParsedToken *h_make_uint(HArena *arena, uint64_t val);
+
+// Standard short-hands to make tokens in an action.
+#define H_MAKE(TYP, VAL)  h_make(p->arena, TT_ ## TYP, VAL)
+#define H_MAKE_SEQ()      h_make_seq(p->arena)
+#define H_MAKE_SEQN(N)    h_make_seqn(p->arena, N)
+#define H_MAKE_BYTES(LEN) h_make_bytes(p->arena, LEN)
+#define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
+#define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
+
+// Extract (cast) type-specific value back from HParsedTokens...
+
+// Pass-through assertion that a given token has the expected type.
+#define h_assert_type(T,P)  (assert(P->token_type == (HTokenType)T), P)
+
+// Convenience short-hand forms of h_assert_type.
+#define H_ASSERT(TYP, TOK)   h_assert_type(TT_ ## TYP, TOK)
+#define H_ASSERT_SEQ(TOK)    h_assert_type(TT_SEQUENCE, TOK)
+#define H_ASSERT_BYTES(TOK)  h_assert_type(TT_BYTES, TOK)
+#define H_ASSERT_SINT(TOK)   h_assert_type(TT_SINT, TOK)
+#define H_ASSERT_UINT(TOK)   h_assert_type(TT_UINT, TOK)
+
+// Assert expected type and return contained value.
+#define H_CAST(TYP, TOK)   ((TYP *) H_ASSERT(TYP, TOK)->user)
+#define H_CAST_SEQ(TOK)    (H_ASSERT_SEQ(TOK)->seq)
+#define H_CAST_BYTES(TOK)  (H_ASSERT_BYTES(TOK)->bytes)
+#define H_CAST_SINT(TOK)   (H_ASSERT_SINT(TOK)->sint)
+#define H_CAST_UINT(TOK)   (H_ASSERT_UINT(TOK)->uint)
+
+// Sequence access...
+
+// Return the length of a sequence.
+size_t h_seq_len(const HParsedToken *p);
+
+// Access a sequence's element array.
+HParsedToken **h_seq_elements(const HParsedToken *p);
+
+// Access a sequence element by index.
+HParsedToken *h_seq_index(const HParsedToken *p, size_t i);
+
+// Access an element in a nested sequence by a path of indices.
+HParsedToken *h_seq_index_path(const HParsedToken *p, size_t i, ...);
+HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
+
+// Convenience macros combining (nested) index access and h_cast.
+#define H_INDEX(TYP, SEQ, ...)   H_CAST(TYP, H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SEQ(SEQ, ...)    H_CAST_SEQ(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_BYTES(SEQ, ...)  H_CAST_BYTES(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_SINT(SEQ, ...)   H_CAST_SINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_UINT(SEQ, ...)   H_CAST_UINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
+
+// Standard short-hand to access and cast elements on a sequence token.
+#define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
+#define H_FIELD_SEQ(...)   H_INDEX_SEQ(p->ast, __VA_ARGS__)
+#define H_FIELD_BYTES(...) H_INDEX_BYTES(p->ast, __VA_ARGS__)
+#define H_FIELD_SINT(...)  H_INDEX_SINT(p->ast, __VA_ARGS__)
+#define H_FIELD_UINT(...)  H_INDEX_UINT(p->ast, __VA_ARGS__)
+
+// Lower-level helper for h_seq_index.
+HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal
+
+// Sequence modification...
+
+// Add elements to a sequence.
+void h_seq_snoc(HParsedToken *xs, const HParsedToken *x);     // append one
+void h_seq_append(HParsedToken *xs, const HParsedToken *ys);  // append many
+
+// XXX TODO: Remove elements from a sequence.
+
+// Flatten nested sequences into one.
+const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p);
+
+
+#endif

src/hammer.h

@@ -42,14 +42,15 @@ typedef enum HParserBackend_ {
 } HParserBackend;
 
 typedef enum HTokenType_ {
-  TT_NONE,
-  TT_BYTES,
-  TT_SINT,
-  TT_UINT,
-  TT_SEQUENCE,
+  // Before you change the explicit values of these, think of the poor bindings ;_;
+  TT_NONE = 1,
+  TT_BYTES = 2,
+  TT_SINT = 4,
+  TT_UINT = 8,
+  TT_SEQUENCE = 16,
   TT_RESERVED_1, // reserved for backend-specific internal use
+  TT_ERR = 32,
   TT_USER = 64,
-  TT_ERR,
   TT_MAX
 } HTokenType;
 
@@ -60,13 +61,15 @@ typedef struct HCountedArray_ {
   struct HParsedToken_ **elements;
 } HCountedArray;
 
+typedef struct HBytes_ {
+  const uint8_t *token;
+  size_t len;
+} HBytes;
+
 typedef struct HParsedToken_ {
   HTokenType token_type;
   union {
-    struct {
-      const uint8_t *token;
-      size_t len;
-    } bytes;
+    HBytes bytes;
     int64_t sint;
     uint64_t uint;
     double dbl;
@@ -175,14 +178,18 @@ typedef struct HBenchmarkResults_ {
   rtype_t name(__VA_ARGS__, ...);					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
   rtype_t name##__mv(HAllocator* mm__, __VA_ARGS__, va_list ap);	\
-  rtype_t name##__v(__VA_ARGS__, va_list ap)
+  rtype_t name##__v(__VA_ARGS__, va_list ap);   \
+  rtype_t name##__a(void *args[]);  \
+  rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // Note: this drops the attributes on the floor for the __v versions
 #define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...)		\
   rtype_t name(__VA_ARGS__, ...) attr;					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr;		\
   rtype_t name##__mv(HAllocator* mm__, __VA_ARGS__, va_list ap);	\
-  rtype_t name##__v(__VA_ARGS__, va_list ap)
+  rtype_t name##__v(__VA_ARGS__, va_list ap); \
+  rtype_t name##__a(void *args[]);  \
+  rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // }}}
 

src/internal.h

@@ -243,6 +243,7 @@ HCountedArray *h_carray_new(HArena * arena);
 void h_carray_append(HCountedArray *array, void* item);
 
 HSlist* h_slist_new(HArena *arena);
+HSlist* h_slist_copy(HSlist *slist);
 void* h_slist_pop(HSlist *slist);
 void h_slist_push(HSlist *slist, void* item);
 bool h_slist_find(HSlist *slist, const void* item);

src/parsers/choice.c

@@ -126,3 +126,27 @@ HParser* h_choice__mv(HAllocator* mm__, const HParser* p, va_list ap_) {
   return h_new_parser(mm__, &choice_vt, s);
 }
 
+HParser* h_choice__a(void *args[]) {
+  return h_choice__ma(&system_allocator, args);
+}
+
+HParser* h_choice__ma(HAllocator* mm__, void *args[]) {
+  size_t len = -1; // because do...while
+  const HParser *arg;
+
+  do {
+    arg=((HParser **)args)[++len];
+  } while(arg);
+
+  HSequence *s = h_new(HSequence, 1);
+  s->p_array = h_new(const HParser *, len);
+
+  for (size_t i = 0; i < len; i++) {
+    s->p_array[i] = ((HParser **)args)[i];
+  }
+
+  s->len = len;
+  HParser *ret = h_new(HParser, 1);
+  ret->vtable = &choice_vt; ret->env = (void*)s;
+  return ret;
+}

src/parsers/sequence.c

@@ -144,3 +144,28 @@ HParser* h_sequence__mv(HAllocator* mm__, const HParser *p, va_list ap_) {
   s->len = len;
   return h_new_parser(mm__, &sequence_vt, s);
 }
+
+HParser* h_sequence__a(void *args[]) {
+  return h_sequence__ma(&system_allocator, args);
+}
+
+HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
+  size_t len = -1; // because do...while
+  const HParser *arg;
+  
+  do {
+    arg=((HParser **)args)[++len];
+  } while(arg);
+  
+  HSequence *s = h_new(HSequence, 1);
+  s->p_array = h_new(const HParser *, len);
+
+  for (size_t i = 0; i < len; i++) {
+    s->p_array[i] = ((HParser **)args)[i];
+  }
+  
+  s->len = len;
+  HParser *ret = h_new(HParser, 1);
+  ret->vtable = &sequence_vt; ret->env = (void*)s;
+  return ret;
+}

src/parsers/whitespace.c

@@ -9,7 +9,7 @@ static HParseResult* parse_whitespace(void* env, HParseState *state) {
     bak = state->input_stream;
     c = h_read_bits(&state->input_stream, 8, false);
     if (state->input_stream.overrun)
-      return NULL;
+      break;
   } while (isspace(c));
   state->input_stream = bak;
   return h_do_parse((HParser*)env, state);

src/pprint.c

@@ -69,7 +69,11 @@ void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta) {
     fprintf(stream, "%*sUSER\n", indent, "");
     break;
   default:
-    assert_message(0, "Should not reach here.");
+    if(tok->token_type > TT_USER) {
+      fprintf(stream, "%*sUSER %d\n", indent, "", tok->token_type-TT_USER);
+    } else {
+      assert_message(0, "Should not reach here.");
+    }
   }
 }
 

src/t_misc.c

@@ -0,0 +1,16 @@
+#include <glib.h>
+#include "test_suite.h"
+#include "hammer.h"
+
+static void test_tt_user(void) {
+  g_check_cmpint(TT_USER, >, TT_NONE);
+  g_check_cmpint(TT_USER, >, TT_BYTES);
+  g_check_cmpint(TT_USER, >, TT_SINT);
+  g_check_cmpint(TT_USER, >, TT_UINT);
+  g_check_cmpint(TT_USER, >, TT_SEQUENCE);
+  g_check_cmpint(TT_USER, >, TT_ERR);
+}
+
+void register_misc_tests(void) {
+  g_test_add_func("/core/misc/tt_user", test_tt_user);
+}

src/t_parser.c

@@ -116,12 +116,17 @@ static void test_float32(gconstpointer backend) {
 
 static void test_whitespace(gconstpointer backend) {
   const HParser *whitespace_ = h_whitespace(h_ch('a'));
+  const HParser *whitespace_end = h_whitespace(h_end_p());
 
   g_check_parse_ok(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "u0x61");
   g_check_parse_ok(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", 2, "u0x61");
   g_check_parse_ok(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "  a", 3, "u0x61");
   g_check_parse_ok(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "\ta", 2, "u0x61");
   g_check_parse_failed(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "_a", 2);
+
+  g_check_parse_ok(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend), "", 0, "NULL");
+  g_check_parse_ok(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend),"  ", 2, "NULL");
+  g_check_parse_failed(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend),"  x", 3);
 }
 
 static void test_left(gconstpointer backend) {
@@ -395,6 +400,17 @@ static void test_not(gconstpointer backend) {
   g_check_parse_ok(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", 4, "(u0x61 <2b.2b> u0x62)");
 }
 
+static void test_leftrec(gconstpointer backend) {
+  const HParser *a_ = h_ch('a');
+
+  HParser *lr_ = h_indirect();
+  h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL));
+
+  g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "u0x61");
+  g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)");
+  g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "((u0x61 u0x61) u0x61)");
+}
+
 void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token);
   g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch);
@@ -437,6 +453,7 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and);
   g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not);
   g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore);
+  g_test_add_data_func("/core/parser/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec);
 
   g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token);
   g_test_add_data_func("/core/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch);
@@ -473,6 +490,7 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p);
   g_test_add_data_func("/core/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool);
   g_test_add_data_func("/core/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore);
+  g_test_add_data_func("/core/parser/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec);
 
   g_test_add_data_func("/core/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token);
   g_test_add_data_func("/core/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch);

src/test_suite.c

@@ -23,6 +23,7 @@ extern void register_bitreader_tests();
 extern void register_bitwriter_tests();
 extern void register_parser_tests();
 extern void register_grammar_tests();
+extern void register_misc_tests();
 extern void register_benchmark_tests();
 
 int main(int argc, char** argv) {
@@ -33,6 +34,7 @@ int main(int argc, char** argv) {
   register_bitwriter_tests();
   register_parser_tests();
   register_grammar_tests();
+  register_misc_tests();
   register_benchmark_tests();
 
   g_test_run();