Merge remote-tracking branch 'upstream/master' into fix-wrong_bit_length

.gitignore

@@ -20,6 +20,7 @@ Session.vim
 *.gcov
 cscope.out
 build/
+libhammer.pc
 .sconsign.dblite
 *.os
 *.pyc

.travis.yml

@@ -92,6 +92,8 @@ matrix:
       env: BINDINGS=cpp CC=clang
 before_install:
   - sudo apt-get update -qq
+  - sudo apt-get install lcov
+  - gem install coveralls-lcov
   - if [ "$BINDINGS" != "none" ]; then sudo apt-get install -qq swig; fi
   - if [ "$BINDINGS" == "perl" ]; then sudo add-apt-repository ppa:dns/irc -y; sudo apt-get update -qq; sudo apt-get install -qq swig=2.0.8-1irc1~12.04; fi
   - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq python-dev; fi
@@ -100,11 +102,13 @@ install: true
 before_script:
   - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi
 script:
-  - scons bindings=$BINDINGS test
+  - if [ "$BINDINGS" == "none" ]; then scons test --variant=debug --coverage; else scons bindings=$BINDINGS test; fi
+after_success:
+  - if [ "$BINDINGS" == "none" ]; then if [ "$CC" == "clang" ]; then llvm-cov gcov -o coverage.info build/debug/src/test_suite.gcda; else lcov --capture --directory build/debug/src --output-file coverage.info; fi; fi
+  - coveralls-lcov coverage.info
 notifications:
   irc: 
     channels: 
       - "irc.upstandinghackers.com#hammer"
     use_notice: true
     skip_join: true
-        

SConstruct

@@ -90,15 +90,18 @@ if GetOption("variant") == 'debug':
 else:
     env = opt
 
-if GetOption("coverage"):
-    env.Append(CFLAGS=["-fprofile-arcs", "-ftest-coverage"],
-               CXXFLAGS=["-fprofile-arcs", "-ftest-coverage"],
-               LDFLAGS=["-fprofile-arcs", "-ftest-coverage"],
-               LIBS=['gcov'])
-
 env["CC"] = os.getenv("CC") or env["CC"]
 env["CXX"] = os.getenv("CXX") or env["CXX"]
 
+if GetOption("coverage"):
+    env.Append(CFLAGS=["--coverage"],
+               CXXFLAGS=["--coverage"],
+               LDFLAGS=["--coverage"])
+    if env["CC"] == "gcc":
+        env.Append(LIBS=['gcov'])
+    else:
+        env.ParseConfig('llvm-config --ldflags')
+
 if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin':
     env.Replace(CC="clang",
                 CXX="clang++")

src/SConscript

@@ -7,7 +7,8 @@ dist_headers = [
     "allocator.h",
     "compiler_specifics.h",
     "glue.h",
-    "internal.h"
+    "internal.h",
+    "platform.h"
 ]
 
 parsers_headers = [
@@ -87,8 +88,9 @@ env.Install("$pkgconfigpath", "../../../libhammer.pc")
 
 testenv = env.Clone()
 testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
-testenv.Append(LIBS=['hammer'], LIBPATH=['.'])
+testenv.Append(LIBS=['hammer'])
-ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'])
+testenv.Prepend(LIBPATH=['.'])
+ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS="--coverage" if testenv.GetOption("coverage") else None)
 ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path]))
 AlwaysBuild(ctest)
 testruns.append(ctest)

src/allocator.c

@@ -18,6 +18,7 @@
 #include <string.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <setjmp.h>
 
 #include "hammer.h"
 #include "internal.h"
@@ -42,17 +43,24 @@ struct HArena_ {
   size_t block_size;
   size_t used;
   size_t wasted;
+
+  jmp_buf *except;
 };
 
+void* h_alloc(HAllocator* mm__, size_t size) {
+  void *p = mm__->alloc(mm__, size);
+  if(!p)
+    h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size);
+  return p;
+}
+
 HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   if (block_size == 0)
     block_size = 4096;
   struct HArena_ *ret = h_new(struct HArena_, 1);
-  struct arena_link *link = (struct arena_link*)mm__->alloc(mm__, sizeof(struct arena_link) + block_size);
+  struct arena_link *link = (struct arena_link*)h_alloc(mm__, sizeof(struct arena_link) + block_size);
-  if (!link) {
+  assert(ret != NULL);
-    // TODO: error-reporting -- let user know that arena link couldn't be allocated
+  assert(link != NULL);
-    return NULL;
-  }
   memset(link, 0, sizeof(struct arena_link) + block_size);
   link->free = block_size;
   link->used = 0;
@@ -62,9 +70,26 @@ HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   ret->used = 0;
   ret->mm__ = mm__;
   ret->wasted = sizeof(struct arena_link) + sizeof(struct HArena_) + block_size;
+  ret->except = NULL;
   return ret;
 }
 
+void h_arena_set_except(HArena *arena, jmp_buf *except)
+{
+  arena->except = except;
+}
+
+static void *alloc_block(HArena *arena, size_t size)
+{
+  void *block = arena->mm__->alloc(arena->mm__, size);
+  if (!block) {
+    if (arena->except)
+      longjmp(*arena->except, 1);
+    h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size);
+  }
+  return block;
+}
+
 void* h_arena_malloc(HArena *arena, size_t size) {
   if (size <= arena->head->free) {
     // fast path..
@@ -79,22 +104,16 @@ void* h_arena_malloc(HArena *arena, size_t size) {
     // This involves some annoying casting...
     arena->used += size;
     arena->wasted += sizeof(struct arena_link*);
-    void* link = arena->mm__->alloc(arena->mm__, size + sizeof(struct arena_link*));
+    void* link = alloc_block(arena, size + sizeof(struct arena_link*));
-    if (!link) {
+    assert(link != NULL);
-      // TODO: error-reporting -- let user know that arena link couldn't be allocated
-      return NULL;
-    }
     memset(link, 0, size + sizeof(struct arena_link*));
     *(struct arena_link**)link = arena->head->next;
     arena->head->next = (struct arena_link*)link;
     return (void*)(((uint8_t*)link) + sizeof(struct arena_link*));
   } else {
     // we just need to allocate an ordinary new block.
-    struct arena_link *link = (struct arena_link*)arena->mm__->alloc(arena->mm__, sizeof(struct arena_link) + arena->block_size);
+    struct arena_link *link = alloc_block(arena, sizeof(struct arena_link) + arena->block_size);
-    if (!link) {
+    assert(link != NULL);
-      // TODO: error-reporting -- let user know that arena link couldn't be allocated
-      return NULL;
-    }
     memset(link, 0, sizeof(struct arena_link) + arena->block_size);
     link->free = arena->block_size - size;
     link->used = size;

src/allocator.h

@@ -18,22 +18,12 @@
 #ifndef HAMMER_ALLOCATOR__H__
 #define HAMMER_ALLOCATOR__H__
 #include <sys/types.h>
+#include <setjmp.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer.
-typedef struct HAllocator_ {
-  void* (*alloc)(struct HAllocator_* allocator, size_t size);
-  void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size);
-  void (*free)(struct HAllocator_* allocator, void* ptr);
-} HAllocator;
-
-typedef struct HArena_ HArena ; // hidden implementation
-
-HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
-
 #if defined __llvm__
 # if __has_attribute(malloc)
 #   define ATTR_MALLOC(n) __attribute__((malloc))
@@ -48,9 +38,23 @@ HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for def
 # define ATTR_MALLOC(n)
 #endif
 
+// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer.
+typedef struct HAllocator_ {
+  void* (*alloc)(struct HAllocator_* allocator, size_t size);
+  void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size);
+  void (*free)(struct HAllocator_* allocator, void* ptr);
+} HAllocator;
+
+void* h_alloc(HAllocator* allocator, size_t size) ATTR_MALLOC(2);
+
+typedef struct HArena_ HArena ; // hidden implementation
+
+HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
+
 void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2);
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
+void h_arena_set_except(HArena *arena, jmp_buf *except);
 
 typedef struct {
   size_t used;

src/backends/glr.c

@@ -198,6 +198,16 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
   HArena *arena  = h_new_arena(mm__, 0);    // will hold the results
   HArena *tarena = h_new_arena(mm__, 0);    // tmp, deleted after parse
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    h_delete_arena(tarena);
+    return NULL;
+  }
+
   // allocate engine lists (will hold one engine per state)
   // these are swapped each iteration
   HSlist *engines = h_slist_new(tarena);

src/backends/llk.c

@@ -383,12 +383,20 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   HArena *arena = s->arena;
   HArena *tarena = s->tarena;
   HSlist *stack = s->stack;
-  HCountedArray *seq = s->seq;
   size_t kmax = table->kmax;
 
-  if(!seq)
+  if(!s->seq)
     return NULL;  // parse already failed
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except))
+    goto no_parse;
+
+  HCountedArray *seq = s->seq;
+
   if(s->win.length > 0) {
     append_win(kmax, s, chunk);
     stream = &s->win;
@@ -527,12 +535,15 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   // since we started with a single nonterminal on the stack, seq should
   // contain exactly the parse result.
   assert(seq->used == 1);
+
+ end:
+  h_arena_set_except(arena, NULL);
+  h_arena_set_except(tarena, NULL);
   return seq;
 
  no_parse:
-  h_delete_arena(arena);
+  seq = NULL;
-  s->arena = NULL;
+  goto end;
-  return NULL;
 
  need_input:
   if(stream->last_chunk)
@@ -540,7 +551,7 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   if(tok)
     h_arena_free(arena, tok);   // no result, yet
   h_slist_push(stack, x);       // try this symbol again next time
-  return seq;
+  goto end;
 }
 
 static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s)
@@ -550,6 +561,8 @@ static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s)
   if(s->seq) {
     assert(s->seq->used == 1);
     res = make_result(s->arena, s->seq->elements[0]);
+  } else {
+    h_delete_arena(s->arena);
   }
 
   h_delete_arena(s->tarena);
@@ -582,6 +595,9 @@ bool h_llk_parse_chunk(HSuspendedParser *s, HInputStream *input)
 
   state->seq = llk_parse_chunk_(state, s->parser, input);
 
+  h_arena_set_except(state->arena, NULL);
+  h_arena_set_except(state->tarena, NULL);
+
   return (state->seq == NULL || h_slist_empty(state->stack));
 }
 

src/backends/lr.c

@@ -388,6 +388,16 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
   HArena *tarena = h_new_arena(mm__, 0);    // tmp, deleted after parse
   HLREngine *engine = h_lrengine_new(arena, tarena, table, stream);
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    h_delete_arena(tarena);
+    return NULL;
+  }
+
   // iterate engine to completion
   while(h_lrengine_step(engine, h_lrengine_action(engine)));
 
@@ -416,6 +426,16 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
   engine->input = *stream;
 
   bool run = true;
+
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(engine->arena, &except);
+  h_arena_set_except(engine->tarena, &except);
+  if(setjmp(except)) {
+    run = false;                            // done immediately
+    assert(engine->state != HLR_SUCCESS);   // h_parse_finish will return NULL
+  }
+
   while(run) {
     // check input against table to determine which action to take
     const HLRAction *action = h_lrengine_action(engine);
@@ -431,6 +451,9 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
       break;
   }
 
+  h_arena_set_except(engine->arena, NULL);
+  h_arena_set_except(engine->tarena, NULL);
+
   *stream = engine->input;
   return !run;  // done if engine no longer running
 }

src/backends/packrat.c

@@ -3,7 +3,7 @@
 #include "../internal.h"
 #include "../parsers/parser_internal.h"
 
-// short-hand for creating cache values (regular case)
+// short-hand for creating lowlevel parse cache values (parse result case)
 static
 HParserCacheValue * cached_result(HParseState *state, HParseResult *result) {
   HParserCacheValue *ret = a_new(HParserCacheValue, 1);
@@ -13,7 +13,7 @@ HParserCacheValue * cached_result(HParseState *state, HParseResult *result) {
   return ret;
 }
 
-// short-hand for caching parse results (left recursion case)
+// short-hand for creating lowlevel parse cache values (left recursion case)
 static
 HParserCacheValue *cached_lr(HParseState *state, HLeftRec *lr) {
   HParserCacheValue *ret = a_new(HParserCacheValue, 1);
@@ -181,23 +181,31 @@ HParseResult* lr_answer(HParserCacheKey *k, HParseState *state, HLeftRec *growab
 HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
   HParserCacheKey *key = a_new(HParserCacheKey, 1);
   key->input_pos = state->input_stream; key->parser = parser;
-  HParserCacheValue *m = recall(key, state);
+  HParserCacheValue *m = NULL;
+  if (parser->vtable->higher) {
+    m = recall(key, state);
+  }
   // check to see if there is already a result for this object...
   if (!m) {
     // It doesn't exist, so create a dummy result to cache
     HLeftRec *base = a_new(HLeftRec, 1);
+    // But only cache it now if there's some chance it could grow; primitive parsers can't
+    if (parser->vtable->higher) {
       base->seed = NULL; base->rule = parser; base->head = NULL;
       h_slist_push(state->lr_stack, base);
       // cache it
       h_hashtable_put(state->cache, key, cached_lr(state, base));
       // parse the input
+    }
     HParseResult *tmp_res = perform_lowlevel_parse(state, parser);
+    if (parser->vtable->higher) {
       // the base variable has passed equality tests with the cache
       h_slist_pop(state->lr_stack);
       // update the cached value to our new position
       HParserCacheValue *cached = h_hashtable_get(state->cache, key);
       assert(cached != NULL);
       cached->input_stream = state->input_stream;
+    }
     // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
     if (NULL == base->head) {
       h_hashtable_put(state->cache, key, cached_result(state, tmp_res));
@@ -246,6 +254,15 @@ static bool pos_equal(const void* key1, const void* key2) {
 
 HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStream *input_stream) {
   HArena * arena = h_new_arena(mm__, 0);
+
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    return NULL;
+  }
+
   HParseState *parse_state = a_new_(arena, HParseState, 1);
   parse_state->cache = h_hashtable_new(arena, cache_key_equal, // key_equal_func
 				       cache_key_hash); // hash_func

src/backends/regex.c

@@ -52,11 +52,21 @@ HRVMTrace *invert_trace(HRVMTrace *trace) {
 
 void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_t len) {
   HArena *arena = h_new_arena(mm__, 0);
-  HSArray *heads_n = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s
+  HSArray *heads_a = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s
-    *heads_p = h_sarray_new(mm__, prog->length);
+          *heads_b = h_sarray_new(mm__, prog->length);
 
   HRVMTrace *ret_trace = NULL;
+  HParseResult *ret = NULL;
   
+  // out of memory handling
+  if(!arena || !heads_a || !heads_b)
+    goto end;
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except))
+    goto end;
+
+  HSArray *heads_n = heads_a, *heads_p = heads_b;
   uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
   HRVMThread *ip_queue = a_new(HRVMThread, prog->length);
   size_t ipq_top;
@@ -164,18 +174,19 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
   }
   // No accept was reached.
  match_fail:
-  if (ret_trace == NULL) {
+
-    // No match found; definite failure.
+  h_arena_set_except(arena, NULL);  // there should be no more allocs from this
-    h_delete_arena(arena);
+  if (ret_trace) {
-    return NULL;
+    // Invert the direction of the trace linked list.
+    ret_trace = invert_trace(ret_trace);
+    ret = run_trace(mm__, prog, ret_trace, input, len);
+    // NB: ret is in its own arena
   }
   
-  // Invert the direction of the trace linked list.
+ end:
-
+  if (arena)   h_delete_arena(arena);
-  ret_trace = invert_trace(ret_trace);
+  if (heads_a) h_sarray_free(heads_a);
-  HParseResult *ret = run_trace(mm__, prog, ret_trace, input, len);
+  if (heads_b) h_sarray_free(heads_b);
-  // ret is in its own arena
-  h_delete_arena(arena);
   return ret;
 }
 #undef PUSH_SVM
@@ -203,6 +214,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
   ctx.stack_capacity = 16;
   ctx.stack = h_new(HParsedToken*, ctx.stack_capacity);
 
+  // out of memory handling
+  if(!arena || !ctx.stack)
+    goto fail;
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except))
+    goto fail;
+
   HParsedToken *tmp_res;
   HRVMTrace *cur;
   for (cur = trace; cur; cur = cur->next) {
@@ -250,11 +269,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       }
       res->bit_length = cur->input_pos * 8;
       res->arena = arena;
+      h_arena_set_except(arena, NULL);
+      h_free(ctx.stack);
       return res;
     }
   }
  fail:
-  h_delete_arena(arena);
+  if (arena) h_delete_arena(arena);
+  if (ctx.stack) h_free(ctx.stack);
   return NULL;
 }
 

src/bitwriter.c

@@ -12,10 +12,8 @@
 HBitWriter *h_bit_writer_new(HAllocator* mm__) {
   HBitWriter *writer = h_new(HBitWriter, 1);
   memset(writer, 0, sizeof(*writer));
-  writer->buf = mm__->alloc(mm__, writer->capacity = 8);
+  writer->buf = h_alloc(mm__, writer->capacity = 8);
-  if (!writer) {
+  assert(writer != NULL);
-    return NULL;
-  }
   memset(writer->buf, 0, writer->capacity);
   writer->mm__ = mm__;
   writer->flags = BYTE_BIG_ENDIAN | BIT_BIG_ENDIAN;

src/datastructures.c

@@ -25,11 +25,12 @@ HCountedArray *h_carray_new(HArena * arena) {
 
 void h_carray_append(HCountedArray *array, void* item) {
   if (array->used >= array->capacity) {
-    HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(HCountedArray*));
+    HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(void*));
     for (size_t i = 0; i < array->used; i++)
       elements[i] = array->elements[i];
     for (size_t i = array->used; i < array->capacity; i++)
       elements[i] = 0;
+    h_arena_free(array->arena, array->elements);
     array->elements = elements;
   }
   array->elements[array->used++] = item;

src/internal.h

@@ -49,7 +49,7 @@
   rtype_t name##__m(HAllocator* mm__)
 // Functions with arguments are difficult to forward cleanly. Alas, we will need to forward them manually.
 
-#define h_new(type, count) ((type*)(mm__->alloc(mm__, sizeof(type)*(count))))
+#define h_new(type, count) ((type*)(h_alloc(mm__, sizeof(type)*(count))))
 #define h_free(addr) (mm__->free(mm__, (addr)))
 
 #ifndef __cplusplus
@@ -278,9 +278,9 @@ typedef struct HRecursionHead_ {
 /* A left recursion.
  *
  * Members:
- *   seed -
+ *   seed - the HResult yielded by rule
- *   rule -
+ *   rule - the HParser that produces seed
- *   head -
+ *   head - the 
  */
 typedef struct HLeftRec_ {
   HParseResult *seed;
@@ -419,6 +419,7 @@ struct HParserVtable_ {
   bool (*isValidCF)(void *env);
   bool (*compile_to_rvm)(HRVMProg *prog, void* env); // FIXME: forgot what the bool return value was supposed to mean.
   void (*desugar)(HAllocator *mm__, HCFStack *stk__, void *env);
+  bool higher; // false if primitive
 };
 
 bool h_false(void*);

src/parsers/action.c

@@ -81,6 +81,7 @@ static const HParserVtable action_vt = {
   .isValidCF = action_isValidCF,
   .desugar = desugar_action,
   .compile_to_rvm = action_ctrvm,
+  .higher = true,
 };
 
 HParser* h_action(const HParser* p, const HAction a, void* user_data) {

src/parsers/and.c

@@ -17,6 +17,7 @@ static const HParserVtable and_vt = {
 				revision. --mlp, 18/12/12 */
   .isValidCF = h_false,      /* despite TODO above, this remains false. */
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 

src/parsers/attr_bool.c

@@ -79,6 +79,7 @@ static const HParserVtable attr_bool_vt = {
   .isValidCF = ab_isValidCF,
   .desugar = desugar_ab,
   .compile_to_rvm = ab_ctrvm,
+  .higher = true,
 };
 
 

src/parsers/bind.c

@@ -4,7 +4,6 @@ typedef struct {
     const HParser *p;
     HContinuation k;
     void *env;
-    HAllocator *mm__;
 } BindEnv;
 
 // an HAllocator backed by an HArena
@@ -39,13 +38,11 @@ static HParseResult *parse_bind(void *be_, HParseState *state) {
     if(!res)
         return NULL;
 
-    // create a temporary arena allocator for the continuation
+    // create a wrapper arena allocator for the continuation
-    HArena *arena = h_new_arena(be->mm__, 0);
+    ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, state->arena};
-    ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, arena};
 
     HParser *kx = be->k((HAllocator *)&aa, res->ast, be->env);
     if(!kx) {
-        h_delete_arena(arena);
         return NULL;
     }
 
@@ -53,7 +50,6 @@ static HParseResult *parse_bind(void *be_, HParseState *state) {
     if(res2)
         res2->bit_length = 0;   // recalculate
 
-    h_delete_arena(arena);
     return res2;
 }
 
@@ -62,6 +58,7 @@ static const HParserVtable bind_vt = {
     .isValidRegular = h_false,
     .isValidCF = h_false,
     .compile_to_rvm = h_not_regular,
+    .higher = true,
 };
 
 HParser *h_bind(const HParser *p, HContinuation k, void *env)
@@ -77,7 +74,6 @@ HParser *h_bind__m(HAllocator *mm__,
     be->p = p;
     be->k = k;
     be->env = env;
-    be->mm__ = mm__;
 
     return h_new_parser(mm__, &bind_vt, be);
 }

src/parsers/bits.c

@@ -102,6 +102,7 @@ static const HParserVtable bits_vt = {
   .isValidCF = h_true,
   .desugar = desugar_bits,
   .compile_to_rvm = bits_ctrvm,
+  .higher = false,
 };
 
 HParser* h_bits(size_t len, bool sign) {

src/parsers/butnot.c

@@ -40,6 +40,7 @@ static const HParserVtable butnot_vt = {
   .isValidRegular = h_false,
   .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF?
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_butnot(const HParser* p1, const HParser* p2) {

src/parsers/ch.c

@@ -47,6 +47,7 @@ static const HParserVtable ch_vt = {
   .isValidCF = h_true,
   .desugar = desugar_ch,
   .compile_to_rvm = ch_ctrvm,
+  .higher = false,
 };
 
 HParser* h_ch(const uint8_t c) {

src/parsers/charset.c

@@ -76,6 +76,7 @@ static const HParserVtable charset_vt = {
   .isValidCF = h_true,
   .desugar = desugar_charset,
   .compile_to_rvm = cs_ctrvm,
+  .higher = false,
 };
 
 HParser* h_ch_range(const uint8_t lower, const uint8_t upper) {

src/parsers/choice.c

@@ -75,6 +75,7 @@ static const HParserVtable choice_vt = {
   .isValidCF = choice_isValidCF,
   .desugar = desugar_choice,
   .compile_to_rvm = choice_ctrvm,
+  .higher = true,
 };
 
 HParser* h_choice(HParser* p, ...) {

src/parsers/difference.c

@@ -39,6 +39,7 @@ static HParserVtable difference_vt = {
   .isValidRegular = h_false,
   .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF?
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_difference(const HParser* p1, const HParser* p2) {

src/parsers/end.c

@@ -25,6 +25,7 @@ static const HParserVtable end_vt = {
   .isValidCF = h_true,
   .desugar = desugar_end,
   .compile_to_rvm = end_ctrvm,
+  .higher = false,
 };
 
 HParser* h_end_p() {

src/parsers/endianness.c

@@ -46,6 +46,7 @@ static const HParserVtable endianness_vt = {
     .isValidCF = h_false,
     .desugar = NULL,
     .compile_to_rvm = h_not_regular,
+    .higher = true,
 };
 
 HParser* h_with_endianness(char endianness, const HParser *p)

src/parsers/epsilon.c

@@ -18,6 +18,7 @@ static const HParserVtable epsilon_vt = {
   .isValidCF = h_true,
   .desugar = desugar_epsilon,
   .compile_to_rvm = epsilon_ctrvm,
+  .higher = false,
 };
 
 HParser* h_epsilon_p() {

src/parsers/ignore.c

@@ -49,6 +49,7 @@ static const HParserVtable ignore_vt = {
   .isValidCF = ignore_isValidCF,
   .desugar = desugar_ignore,
   .compile_to_rvm = ignore_ctrvm,
+  .higher = true,
 };
 
 HParser* h_ignore(const HParser* p) {

src/parsers/ignoreseq.c

@@ -105,6 +105,7 @@ static const HParserVtable ignoreseq_vt = {
   .isValidCF = is_isValidCF,
   .desugar = desugar_ignoreseq,
   .compile_to_rvm = is_ctrvm,
+  .higher = true,
 };
 
 

src/parsers/indirect.c

@@ -31,6 +31,7 @@ static const HParserVtable indirect_vt = {
   .isValidCF = indirect_isValidCF,
   .desugar = desugar_indirect,
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) {

src/parsers/int_range.c

@@ -117,6 +117,7 @@ static const HParserVtable int_range_vt = {
   .isValidCF = h_true,
   .desugar = desugar_int_range,
   .compile_to_rvm = ir_ctrvm,
+  .higher = false,
 };
 
 HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper) {

src/parsers/many.c

@@ -10,7 +10,10 @@ typedef struct {
 
 static HParseResult *parse_many(void* env, HParseState *state) {
   HRepeat *env_ = (HRepeat*) env;
-  HCountedArray *seq = h_carray_new_sized(state->arena, (env_->count > 0 ? env_->count : 4));
+  size_t size = env_->count;
+  if(size <= 0) size = 4;
+  if(size > 1024) size = 1024;  // let's try parsing some elements first...
+  HCountedArray *seq = h_carray_new_sized(state->arena, size);
   size_t count = 0;
   HInputStream bak;
   while (env_->min_p || env_->count > count) {
@@ -199,6 +202,7 @@ static const HParserVtable many_vt = {
   .isValidCF = many_isValidCF,
   .desugar = desugar_many,
   .compile_to_rvm = many_ctrvm,
+  .higher = true,
 };
 
 HParser* h_many(const HParser* p) {

src/parsers/not.c

@@ -15,6 +15,7 @@ static const HParserVtable not_vt = {
   .isValidRegular = h_false,  /* see and.c for why */
   .isValidCF = h_false,
   .compile_to_rvm = h_not_regular, // Is actually regular, but the generation step is currently unable to handle it. TODO: fix this.
+  .higher = true,
 };
 
 HParser* h_not(const HParser* p) {

src/parsers/nothing.c

@@ -22,6 +22,7 @@ static const HParserVtable nothing_vt = {
   .isValidCF = h_true,
   .desugar = desugar_nothing,
   .compile_to_rvm = nothing_ctrvm,
+  .higher = false,
 };
 
 HParser* h_nothing_p() {

src/parsers/optional.c

@@ -84,6 +84,7 @@ static const HParserVtable optional_vt = {
   .isValidCF = opt_isValidCF,
   .desugar = desugar_optional,
   .compile_to_rvm = opt_ctrvm,
+  .higher = true,
 };
 
 HParser* h_optional(const HParser* p) {

src/parsers/permutation.c

@@ -104,6 +104,7 @@ static const HParserVtable permutation_vt = {
   .isValidCF = h_false,
   .desugar = NULL,
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_permutation(HParser* p, ...) {

src/parsers/sequence.c

@@ -93,6 +93,7 @@ static const HParserVtable sequence_vt = {
   .isValidCF = sequence_isValidCF,
   .desugar = desugar_sequence,
   .compile_to_rvm = sequence_ctrvm,
+  .higher = true,
 };
 
 HParser* h_sequence(HParser* p, ...) {
@@ -116,16 +117,21 @@ HParser* h_sequence__v(HParser* p, va_list ap) {
 }
 
 HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) {
-  va_list ap;
+  HSequence *s = h_new(HSequence, 1);
-  size_t len = 0;
+  s->len = 0;
+
+  if(p) {
+    // non-empty sequence
     const HParser *arg;
+    size_t len = 0;
+    va_list ap;
+
     va_copy(ap, ap_);
     do {
       len++;
       arg = va_arg(ap, HParser *);
     } while (arg);
     va_end(ap);
-  HSequence *s = h_new(HSequence, 1);
     s->p_array = h_new(HParser *, len);
 
     va_copy(ap, ap_);
@@ -136,6 +142,8 @@ HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) {
     va_end(ap);
 
     s->len = len;
+  }
+
   return h_new_parser(mm__, &sequence_vt, s);
 }
 

src/parsers/token.c

@@ -73,6 +73,7 @@ const HParserVtable token_vt = {
   .isValidCF = h_true,
   .desugar = desugar_token,
   .compile_to_rvm = token_ctrvm,
+  .higher = false,
 };
 
 HParser* h_token(const uint8_t *str, const size_t len) {

src/parsers/unimplemented.c

@@ -18,6 +18,7 @@ static const HParserVtable unimplemented_vt = {
   .isValidCF = h_false,
   .desugar = NULL,
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 static HParser unimplemented = {

src/parsers/value.c

@@ -26,6 +26,7 @@ static const HParserVtable put_vt = {
   .isValidRegular = h_false,
   .isValidCF = h_false,
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_put_value(const HParser* p, const char* name) {
@@ -55,6 +56,7 @@ static const HParserVtable get_vt = {
   .isValidRegular = h_false,
   .isValidCF = h_false,
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_get_value(const char* name) {

src/parsers/whitespace.c

@@ -75,6 +75,7 @@ static const HParserVtable whitespace_vt = {
   .isValidCF = ws_isValidCF,
   .desugar = desugar_whitespace,
   .compile_to_rvm = ws_ctrvm,
+  .higher = false,
 };
 
 HParser* h_whitespace(const HParser* p) {

src/parsers/xor.c

@@ -36,6 +36,7 @@ static const HParserVtable xor_vt = {
   .isValidRegular = h_false,
   .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF?
   .compile_to_rvm = h_not_regular,
+  .higher = true,
 };
 
 HParser* h_xor(const HParser* p1, const HParser* p2) {

src/pprint.c

@@ -186,13 +186,11 @@ static void unamb_sub(const HParsedToken* tok, struct result_buf *buf) {
 
 char* h_write_result_unamb(const HParsedToken* tok) {
   struct result_buf buf = {
-    .output = (&system_allocator)->alloc(&system_allocator, 16),
+    .output = h_alloc(&system_allocator, 16),
     .len = 0,
     .capacity = 16
   };
-  if (!buf.output) {
+  assert(buf.output != NULL);
-    return NULL;
-  }
   unamb_sub(tok, &buf);
   append_buf_c(&buf, 0);
   return buf.output;

src/registry.c

@@ -46,10 +46,8 @@ static int compare_entries(const void* v1, const void* v2) {
 }
 
 HTokenType h_allocate_token_type(const char* name) {
-  Entry* new_entry = (&system_allocator)->alloc(&system_allocator, sizeof(*new_entry));
+  Entry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
-  if (!new_entry) {
+  assert(new_entry != NULL);
-    return TT_INVALID;
-  }
   new_entry->name = name;
   new_entry->value = 0;
   Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries);

src/t_misc.c

@@ -1,5 +1,6 @@
 #include <glib.h>
 #include <string.h>
+#include <sys/resource.h>
 #include "test_suite.h"
 #include "hammer.h"
 
@@ -29,7 +30,54 @@ static void test_tt_registry(void) {
   g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, 0);
 }
 
+// test out-of-memory handling with a selectively failing allocator
+static void *fail_alloc(HAllocator *mm__, size_t size) {
+  if(size - 0xdead <= 0x30) // allow for overhead of arena link structure
+    return NULL;
+  return system_allocator.alloc(&system_allocator, size);
+}
+static void *fail_realloc(HAllocator *mm__, void *ptr, size_t size) {
+  return system_allocator.realloc(&system_allocator, ptr, size);
+}
+static void fail_free(HAllocator *mm__, void *ptr) {
+  return system_allocator.free(&system_allocator, ptr);
+}
+static HAllocator fail_allocator = {fail_alloc, fail_realloc, fail_free};
+static HParsedToken *act_oom(const HParseResult *r, void *user) {
+  void *buf = h_arena_malloc(r->arena, 0xdead);
+  assert(buf != NULL);
+  return NULL;  // succeed with null token
+}
+static void test_oom(void) {
+  HParser *p = h_action(h_ch('x'), act_oom, NULL);
+    // this should always fail, but never crash
+
+  // sanity-check: parses should succeed with the normal allocator...
+  g_check_parse_ok(p, PB_PACKRAT, "x",1);
+  g_check_parse_ok(p, PB_REGULAR, "x",1);
+  g_check_parse_ok(p, PB_LLk, "x",1);
+  g_check_parse_ok(p, PB_LALR, "x",1);
+  g_check_parse_ok(p, PB_GLR, "x",1);
+  //XXX g_check_parse_chunks_ok(p, PB_REGULAR, "",0, "x",1);
+  g_check_parse_chunks_ok(p, PB_LLk, "",0, "x",1);
+  g_check_parse_chunks_ok(p, PB_LALR, "",0, "x",1);
+  //XXX g_check_parse_chunks_ok(p, PB_GLR, "",0, "x",1);
+
+  // ...and fail gracefully with the broken one
+  HAllocator *mm__ = &fail_allocator;
+  g_check_parse_failed__m(mm__, p, PB_PACKRAT, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_REGULAR, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_LLk, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_LALR, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_GLR, "x",1);
+  //XXX g_check_parse_chunks_failed__m(mm__, p, PB_REGULAR, "",0, "x",1);
+  g_check_parse_chunks_failed__m(mm__, p, PB_LLk, "",0, "x",1);
+  g_check_parse_chunks_failed__m(mm__, p, PB_LALR, "",0, "x",1);
+  //XXX g_check_parse_chunks_failed__m(mm__, p, PB_GLR, "",0, "x",1);
+}
+
 void register_misc_tests(void) {
   g_test_add_func("/core/misc/tt_user", test_tt_user);
   g_test_add_func("/core/misc/tt_registry", test_tt_registry);
+  g_test_add_func("/core/misc/oom", test_oom);
 }

src/t_parser.c

@@ -241,6 +241,7 @@ static void test_nothing_p(gconstpointer backend) {
 static void test_sequence(gconstpointer backend) {
   const HParser *sequence_1 = h_sequence(h_ch('a'), h_ch('b'), NULL);
   const HParser *sequence_2 = h_sequence(h_ch('a'), h_whitespace(h_ch('b')), NULL);
+  const HParser *sequence_3 = h_sequence(NULL, NULL);   // second NULL is to silence GCC
 
   g_check_parse_match(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)");
   g_check_parse_failed(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1);
@@ -248,6 +249,7 @@ static void test_sequence(gconstpointer backend) {
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)");
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", 3, "(u0x61 u0x62)");
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a  b", 4, "(u0x61 u0x62)");  
+  g_check_parse_match(sequence_3, (HParserBackend)GPOINTER_TO_INT(backend), "", 0, "()");
 }
 
 static void test_choice(gconstpointer backend) {

src/test_suite.h

@@ -78,29 +78,24 @@
   } while(0)
 
   
-// TODO: replace uses of this with g_check_parse_failed
+#define g_check_parse_failed__m(mm__, parser, backend, input, inp_len) do { \
-#define g_check_failed(res) do {					\
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
-    const HParseResult *result = (res);					\
-    if (NULL != result) {						\
-      g_test_message("Check failed: shouldn't have succeeded, but did"); \
-      g_test_fail();							\
-    }									\
-  } while(0)
-
-#define g_check_parse_failed(parser, backend, input, inp_len) do {	\
-    int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip != 0) {	\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;	\
     }	\
-    const HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \
+    HParseResult *result = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
     if (NULL != result) {						\
+      h_parse_result_free(result);					\
       g_test_message("Check failed: shouldn't have succeeded, but did"); \
       g_test_fail();							\
     }									\
   } while(0)
 
+#define g_check_parse_failed(p, be, input, len)				\
+    g_check_parse_failed__m(&system_allocator, p, be, input, len)
+
 #define g_check_parse_ok(parser, backend, input, inp_len) do {		\
     int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
     if(skip) {								\
@@ -119,7 +114,7 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 
@@ -144,21 +139,54 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 
-#define g_check_parse_chunks_failed(parser, backend, chunk1, c1_len, chunk2, c2_len) do {	\
+#define g_check_parse_chunks_failed__m(mm__, parser, backend, chunk1, c1_len, chunk2, c2_len) do { \
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
+    if(skip) {								\
+      g_test_message("Compile failed");					\
+      g_test_fail();							\
+      break;								\
+    }									\
+    g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len); \
+  } while(0)
+
+#define g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len) do { \
+    HSuspendedParser *s = h_parse_start__m(mm__, parser);		\
+    if(!s) {								\
+      g_test_message("Chunk-wise parsing not available");		\
+      g_test_fail();							\
+      break;								\
+    }									\
+    h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
+    h_parse_chunk(s, (const uint8_t*)chunk2, c2_len);			\
+    HParseResult *res = h_parse_finish(s);				\
+    if (NULL != res) {							\
+      h_parse_result_free(res);						\
+      g_test_message("Check failed: shouldn't have succeeded, but did"); \
+      g_test_fail();							\
+    }									\
+  } while(0)
+
+#define g_check_parse_chunks_failed(p, be, c1, c1_len, c2, c2_len) \
+  g_check_parse_chunks_failed__m(&system_allocator, p, be, c1, c1_len, c2, c2_len)
+
+#define g_check_parse_chunks_failed_(p, c1, c1_len, c2, c2_len) \
+  g_check_parse_chunks_failed___m(&system_allocator, p, c1, c1_len, c2, c2_len)
+
+#define g_check_parse_chunks_ok(parser, backend, chunk1, c1_len, chunk2, c2_len) do {	\
     int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip) {								\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;								\
     }									\
-    g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len); \
+    g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len); \
   } while(0)
 
-#define g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len) do {	\
+#define g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len) do {	\
     HSuspendedParser *s = h_parse_start(parser);			\
     if(!s) {								\
       g_test_message("Chunk-wise parsing not available");		\
@@ -167,10 +195,18 @@
     }									\
     h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
     h_parse_chunk(s, (const uint8_t*)chunk2, c2_len);			\
-    const HParseResult *res = h_parse_finish(s);			\
+    HParseResult *res = h_parse_finish(s);				\
-    if (NULL != res) {							\
+    if (!res) {								\
-      g_test_message("Check failed: shouldn't have succeeded, but did"); \
+      g_test_message("Parse failed on line %d", __LINE__);		\
       g_test_fail();							\
+    } else {								\
+      HArenaStats stats;						\
+      h_allocator_stats(res->arena, &stats);				\
+      g_test_message("Parse used %zd bytes, wasted %zd bytes. "		\
+                     "Inefficiency: %5f%%",				\
+		     stats.used, stats.wasted,				\
+		     stats.wasted * 100. / (stats.used+stats.wasted));	\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 
@@ -207,7 +243,7 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)