From d2ade1f5b4f8c438353d79085c55b1c06436b372 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 29 Oct 2015 13:11:32 +0100
Subject: [PATCH 01/21] call h_arena_free when resizing in h_carray_append

---
 src/datastructures.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/datastructures.c b/src/datastructures.c
index 0feeb21..af8477b 100644
--- a/src/datastructures.c
+++ b/src/datastructures.c
@@ -25,11 +25,12 @@ HCountedArray *h_carray_new(HArena * arena) {
 
 void h_carray_append(HCountedArray *array, void* item) {
   if (array->used >= array->capacity) {
-    HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(HCountedArray*));
+    HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(void*));
     for (size_t i = 0; i < array->used; i++)
       elements[i] = array->elements[i];
     for (size_t i = array->used; i < array->capacity; i++)
       elements[i] = 0;
+    h_arena_free(array->arena, array->elements);
     array->elements = elements;
   }
   array->elements[array->used++] = item;

From 3765fd64e1b2d0f15ac689f5ee3af9939b26ada5 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 29 Oct 2015 13:12:16 +0100
Subject: [PATCH 02/21] don't pre-allocate all space when parsing an
 h_length_value

---
 src/parsers/many.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/parsers/many.c b/src/parsers/many.c
index 6496bbe..071e3fc 100644
--- a/src/parsers/many.c
+++ b/src/parsers/many.c
@@ -10,7 +10,10 @@ typedef struct {
 
 static HParseResult *parse_many(void* env, HParseState *state) {
   HRepeat *env_ = (HRepeat*) env;
-  HCountedArray *seq = h_carray_new_sized(state->arena, (env_->count > 0 ? env_->count : 4));
+  size_t size = env_->count;
+  if(size <= 0) size = 4;
+  if(size > 1024) size = 1024;  // let's try parsing some elements first...
+  HCountedArray *seq = h_carray_new_sized(state->arena, size);
   size_t count = 0;
   HInputStream bak;
   while (env_->min_p || env_->count > count) {

From d5e79aa4cb0f932a1f4b4b913d0b8f55678316f3 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 29 Oct 2015 22:03:05 +0100
Subject: [PATCH 03/21] fail an assert on h_sequence(NULL) instead of
 segfaulting later

---
 src/parsers/sequence.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c
index 30de34a..6377589 100644
--- a/src/parsers/sequence.c
+++ b/src/parsers/sequence.c
@@ -120,6 +120,8 @@ HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) {
   va_list ap;
   size_t len = 0;
   const HParser *arg;
+
+  assert(p != NULL);
   va_copy(ap, ap_);
   do {
     len++;

From 9ef70f2f2d40b973c81cce4ca9e65bafa6c1ba4a Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 30 Oct 2015 21:23:45 +0100
Subject: [PATCH 04/21] never return if h_arena_malloc fails, call errx() or
 longjmp()

---
 src/allocator.c | 34 ++++++++++++++++++++++++----------
 src/allocator.h |  2 ++
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/allocator.c b/src/allocator.c
index 258edfa..4646dd8 100644
--- a/src/allocator.c
+++ b/src/allocator.c
@@ -18,6 +18,7 @@
 #include <string.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <setjmp.h>
 
 #include "hammer.h"
 #include "internal.h"
@@ -42,6 +43,8 @@ struct HArena_ {
   size_t block_size;
   size_t used;
   size_t wasted;
+
+  jmp_buf *except;
 };
 
 HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
@@ -62,9 +65,26 @@ HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   ret->used = 0;
   ret->mm__ = mm__;
   ret->wasted = sizeof(struct arena_link) + sizeof(struct HArena_) + block_size;
+  ret->except = NULL;
   return ret;
 }
 
+void h_arena_set_except(HArena *arena, jmp_buf *except)
+{
+  arena->except = except;
+}
+
+static void *alloc_block(HArena *arena, size_t size)
+{
+  void *block = arena->mm__->alloc(arena->mm__, size);
+  if (!block) {
+    if (arena->except)
+      longjmp(*arena->except, 1);
+    h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size);
+  }
+  return block;
+}
+
 void* h_arena_malloc(HArena *arena, size_t size) {
   if (size <= arena->head->free) {
     // fast path..
@@ -79,22 +99,16 @@ void* h_arena_malloc(HArena *arena, size_t size) {
     // This involves some annoying casting...
     arena->used += size;
     arena->wasted += sizeof(struct arena_link*);
-    void* link = arena->mm__->alloc(arena->mm__, size + sizeof(struct arena_link*));
-    if (!link) {
-      // TODO: error-reporting -- let user know that arena link couldn't be allocated
-      return NULL;
-    }
+    void* link = alloc_block(arena, size + sizeof(struct arena_link*));
+    assert(link != NULL);
     memset(link, 0, size + sizeof(struct arena_link*));
     *(struct arena_link**)link = arena->head->next;
     arena->head->next = (struct arena_link*)link;
     return (void*)(((uint8_t*)link) + sizeof(struct arena_link*));
   } else {
     // we just need to allocate an ordinary new block.
-    struct arena_link *link = (struct arena_link*)arena->mm__->alloc(arena->mm__, sizeof(struct arena_link) + arena->block_size);
-    if (!link) {
-      // TODO: error-reporting -- let user know that arena link couldn't be allocated
-      return NULL;
-    }
+    struct arena_link *link = alloc_block(arena, sizeof(struct arena_link) + arena->block_size);
+    assert(link != NULL);
     memset(link, 0, sizeof(struct arena_link) + arena->block_size);
     link->free = arena->block_size - size;
     link->used = size;
diff --git a/src/allocator.h b/src/allocator.h
index 4a48693..d80209d 100644
--- a/src/allocator.h
+++ b/src/allocator.h
@@ -18,6 +18,7 @@
 #ifndef HAMMER_ALLOCATOR__H__
 #define HAMMER_ALLOCATOR__H__
 #include <sys/types.h>
+#include <setjmp.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -51,6 +52,7 @@ HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for def
 void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2);
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
+void h_arena_set_except(HArena *arena, jmp_buf *except);
 
 typedef struct {
   size_t used;

From e8b1962005e32156a722c9189e2431c7d3a50a2b Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 30 Oct 2015 21:26:08 +0100
Subject: [PATCH 05/21] out of memory handling and leak fixes in regex backend

---
 src/backends/regex.c | 45 ++++++++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 12 deletions(-)

diff --git a/src/backends/regex.c b/src/backends/regex.c
index f6494fa..3c2e803 100644
--- a/src/backends/regex.c
+++ b/src/backends/regex.c
@@ -56,7 +56,16 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
     *heads_p = h_sarray_new(mm__, prog->length);
 
   HRVMTrace *ret_trace = NULL;
+  HParseResult *ret = NULL;
   
+  // out of memory handling
+  if(!arena || !heads_n || !heads_p)
+    goto end;
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except))
+    goto end;
+
   uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
   HRVMThread *ip_queue = a_new(HRVMThread, prog->length);
   size_t ipq_top;
@@ -164,18 +173,19 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
   }
   // No accept was reached.
  match_fail:
-  if (ret_trace == NULL) {
-    // No match found; definite failure.
-    h_delete_arena(arena);
-    return NULL;
+
+  h_arena_set_except(arena, NULL);  // there should be no more allocs from this
+  if (ret_trace) {
+    // Invert the direction of the trace linked list.
+    ret_trace = invert_trace(ret_trace);
+    ret = run_trace(mm__, prog, ret_trace, input, len);
+    // NB: ret is in its own arena
   }
   
-  // Invert the direction of the trace linked list.
-
-  ret_trace = invert_trace(ret_trace);
-  HParseResult *ret = run_trace(mm__, prog, ret_trace, input, len);
-  // ret is in its own arena
-  h_delete_arena(arena);
+ end:
+  if (arena)   h_delete_arena(arena);
+  if (heads_n) h_sarray_free(heads_n);
+  if (heads_p) h_sarray_free(heads_p);
   return ret;
 }
 #undef PUSH_SVM
@@ -203,6 +213,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
   ctx.stack_capacity = 16;
   ctx.stack = h_new(HParsedToken*, ctx.stack_capacity);
 
+  // out of memory handling
+  if(!arena || !ctx.stack)
+    goto fail;
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except))
+    goto fail;
+
   HParsedToken *tmp_res;
   HRVMTrace *cur;
   for (cur = trace; cur; cur = cur->next) {
@@ -242,7 +260,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       break;
     case SVM_ACCEPT:
       assert(ctx.stack_count <= 1);
-	HParseResult *res = a_new(HParseResult, 1);
+      HParseResult *res = a_new(HParseResult, 1);
       if (ctx.stack_count == 1) {
 	res->ast = ctx.stack[0];
       } else {
@@ -250,11 +268,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       }
       res->bit_length = cur->input_pos * 8;
       res->arena = arena;
+      h_arena_set_except(arena, NULL);
+      h_free(ctx.stack);
       return res;
     }
   }
  fail:
-  h_delete_arena(arena);
+  if (arena) h_delete_arena(arena);
+  if (ctx.stack) h_free(ctx.stack);
   return NULL;
 }
 

From 25acd90a481357942cd3a7f3add6b8b996345305 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sun, 1 Nov 2015 17:49:53 +0100
Subject: [PATCH 06/21] allow h_sequence(NULL) as parser for the empty sequence

---
 src/parsers/sequence.c | 43 +++++++++++++++++++++++-------------------
 src/t_parser.c         |  2 ++
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c
index 6377589..55c0c88 100644
--- a/src/parsers/sequence.c
+++ b/src/parsers/sequence.c
@@ -117,28 +117,33 @@ HParser* h_sequence__v(HParser* p, va_list ap) {
 }
 
 HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) {
-  va_list ap;
-  size_t len = 0;
-  const HParser *arg;
-
-  assert(p != NULL);
-  va_copy(ap, ap_);
-  do {
-    len++;
-    arg = va_arg(ap, HParser *);
-  } while (arg);
-  va_end(ap);
   HSequence *s = h_new(HSequence, 1);
-  s->p_array = h_new(HParser *, len);
+  s->len = 0;
 
-  va_copy(ap, ap_);
-  s->p_array[0] = p;
-  for (size_t i = 1; i < len; i++) {
-    s->p_array[i] = va_arg(ap, HParser *);
-  } while (arg);
-  va_end(ap);
+  if(p) {
+    // non-empty sequence
+    const HParser *arg;
+    size_t len = 0;
+    va_list ap;
+
+    va_copy(ap, ap_);
+    do {
+      len++;
+      arg = va_arg(ap, HParser *);
+    } while (arg);
+    va_end(ap);
+    s->p_array = h_new(HParser *, len);
+
+    va_copy(ap, ap_);
+    s->p_array[0] = p;
+    for (size_t i = 1; i < len; i++) {
+      s->p_array[i] = va_arg(ap, HParser *);
+    } while (arg);
+    va_end(ap);
+
+    s->len = len;
+  }
 
-  s->len = len;
   return h_new_parser(mm__, &sequence_vt, s);
 }
 
diff --git a/src/t_parser.c b/src/t_parser.c
index c42eca9..331d262 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -241,6 +241,7 @@ static void test_nothing_p(gconstpointer backend) {
 static void test_sequence(gconstpointer backend) {
   const HParser *sequence_1 = h_sequence(h_ch('a'), h_ch('b'), NULL);
   const HParser *sequence_2 = h_sequence(h_ch('a'), h_whitespace(h_ch('b')), NULL);
+  const HParser *sequence_3 = h_sequence(NULL, NULL);   // second NULL is to silence GCC
 
   g_check_parse_match(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)");
   g_check_parse_failed(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1);
@@ -248,6 +249,7 @@ static void test_sequence(gconstpointer backend) {
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)");
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", 3, "(u0x61 u0x62)");
   g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a  b", 4, "(u0x61 u0x62)");  
+  g_check_parse_match(sequence_3, (HParserBackend)GPOINTER_TO_INT(backend), "", 0, "()");
 }
 
 static void test_choice(gconstpointer backend) {

From 5b3cb46c968a5960378b6d31412943422e3e207d Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Sat, 14 Nov 2015 23:26:04 +0100
Subject: [PATCH 07/21] avoid GCC warning about potential longjmp clobbers

---
 src/backends/regex.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/backends/regex.c b/src/backends/regex.c
index 3c2e803..f7dd98a 100644
--- a/src/backends/regex.c
+++ b/src/backends/regex.c
@@ -52,20 +52,21 @@ HRVMTrace *invert_trace(HRVMTrace *trace) {
 
 void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_t len) {
   HArena *arena = h_new_arena(mm__, 0);
-  HSArray *heads_n = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s
-    *heads_p = h_sarray_new(mm__, prog->length);
+  HSArray *heads_a = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s
+          *heads_b = h_sarray_new(mm__, prog->length);
 
   HRVMTrace *ret_trace = NULL;
   HParseResult *ret = NULL;
   
   // out of memory handling
-  if(!arena || !heads_n || !heads_p)
+  if(!arena || !heads_a || !heads_b)
     goto end;
   jmp_buf except;
   h_arena_set_except(arena, &except);
   if(setjmp(except))
     goto end;
 
+  HSArray *heads_n = heads_a, *heads_p = heads_b;
   uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
   HRVMThread *ip_queue = a_new(HRVMThread, prog->length);
   size_t ipq_top;
@@ -184,8 +185,8 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
   
  end:
   if (arena)   h_delete_arena(arena);
-  if (heads_n) h_sarray_free(heads_n);
-  if (heads_p) h_sarray_free(heads_p);
+  if (heads_a) h_sarray_free(heads_a);
+  if (heads_b) h_sarray_free(heads_b);
   return ret;
 }
 #undef PUSH_SVM

From e26a8ff572e71b0e2690e7497f015aad09cf0711 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 27 Nov 2015 17:55:18 +0100
Subject: [PATCH 08/21] add libhammer.pc to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 721dcf9..570fbf8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ Session.vim
 *.gcov
 cscope.out
 build/
+libhammer.pc
 .sconsign.dblite
 *.os
 *.pyc

From ca1d8df06cdead8f9de1ba81d1de8cda363c16b4 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 30 Nov 2015 14:19:40 +0100
Subject: [PATCH 09/21] don't allocate a new arena in h_bind, use the existing
 one

Rationale: If memory allocation fails in the inner parse and we
longjump up the stack, the temporary arena will be missed and leak.

NB: This change means that any allocations done by the continuation
(in the form of new parsers, probably) will persist for the
lifetime of the parse result. Beware of wasting too much memory
this way! The bind continuation should generally keep dynamic
allocations to a minimum.
---
 src/parsers/bind.c | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/parsers/bind.c b/src/parsers/bind.c
index 7fa821d..808df97 100644
--- a/src/parsers/bind.c
+++ b/src/parsers/bind.c
@@ -4,7 +4,6 @@ typedef struct {
     const HParser *p;
     HContinuation k;
     void *env;
-    HAllocator *mm__;
 } BindEnv;
 
 // an HAllocator backed by an HArena
@@ -39,20 +38,15 @@ static HParseResult *parse_bind(void *be_, HParseState *state) {
     if(!res)
         return NULL;
 
-    // create a temporary arena allocator for the continuation
-    HArena *arena = h_new_arena(be->mm__, 0);
-    ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, arena};
+    // create a wrapper arena allocator for the continuation
+    ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, state->arena};
 
     HParser *kx = be->k((HAllocator *)&aa, res->ast, be->env);
     if(!kx) {
-        h_delete_arena(arena);
         return NULL;
     }
 
-    res = h_do_parse(kx, state);
-
-    h_delete_arena(arena);
-    return res;
+    return h_do_parse(kx, state);
 }
 
 static const HParserVtable bind_vt = {
@@ -76,7 +70,6 @@ HParser *h_bind__m(HAllocator *mm__,
     be->p = p;
     be->k = k;
     be->env = env;
-    be->mm__ = mm__;
 
     return h_new_parser(mm__, &bind_vt, be);
 }

From 3fc56a0dc3390e5a5ac94bacb2110ba4dcf673ee Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 30 Nov 2015 16:37:00 +0100
Subject: [PATCH 10/21] add h_alloc() which calls errx() on failure and use it
 for all basic allocation

Rationale: "Basic allocation" refers to things outside of parsing proper,
mostly initialization. If such allocations fail, the system is globally
emory-starved from which it will likely not recover by returning failure.
In this case, terminating the process is in fact the most robust strategy as
it may mean the difference between a permanent hang and a temporary crash.
---
 src/allocator.c | 15 ++++++++++-----
 src/allocator.h | 24 +++++++++++++-----------
 src/bitwriter.c |  6 ++----
 src/internal.h  |  2 +-
 src/pprint.c    |  6 ++----
 src/registry.c  |  6 ++----
 6 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/src/allocator.c b/src/allocator.c
index 4646dd8..cc259e6 100644
--- a/src/allocator.c
+++ b/src/allocator.c
@@ -47,15 +47,20 @@ struct HArena_ {
   jmp_buf *except;
 };
 
+void* h_alloc(HAllocator* mm__, size_t size) {
+  void *p = mm__->alloc(mm__, size);
+  if(!p)
+    h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size);
+  return p;
+}
+
 HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   if (block_size == 0)
     block_size = 4096;
   struct HArena_ *ret = h_new(struct HArena_, 1);
-  struct arena_link *link = (struct arena_link*)mm__->alloc(mm__, sizeof(struct arena_link) + block_size);
-  if (!link) {
-    // TODO: error-reporting -- let user know that arena link couldn't be allocated
-    return NULL;
-  }
+  struct arena_link *link = (struct arena_link*)h_alloc(mm__, sizeof(struct arena_link) + block_size);
+  assert(ret != NULL);
+  assert(link != NULL);
   memset(link, 0, sizeof(struct arena_link) + block_size);
   link->free = block_size;
   link->used = 0;
diff --git a/src/allocator.h b/src/allocator.h
index d80209d..dc88af6 100644
--- a/src/allocator.h
+++ b/src/allocator.h
@@ -24,17 +24,6 @@
 extern "C" {
 #endif
 
-// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer.
-typedef struct HAllocator_ {
-  void* (*alloc)(struct HAllocator_* allocator, size_t size);
-  void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size);
-  void (*free)(struct HAllocator_* allocator, void* ptr);
-} HAllocator;
-
-typedef struct HArena_ HArena ; // hidden implementation
-
-HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
-
 #if defined __llvm__
 # if __has_attribute(malloc)
 #   define ATTR_MALLOC(n) __attribute__((malloc))
@@ -49,6 +38,19 @@ HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for def
 # define ATTR_MALLOC(n)
 #endif
 
+// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer.
+typedef struct HAllocator_ {
+  void* (*alloc)(struct HAllocator_* allocator, size_t size);
+  void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size);
+  void (*free)(struct HAllocator_* allocator, void* ptr);
+} HAllocator;
+
+void* h_alloc(HAllocator* allocator, size_t size) ATTR_MALLOC(2);
+
+typedef struct HArena_ HArena ; // hidden implementation
+
+HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
+
 void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2);
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
diff --git a/src/bitwriter.c b/src/bitwriter.c
index 74e2734..bcc21dd 100644
--- a/src/bitwriter.c
+++ b/src/bitwriter.c
@@ -12,10 +12,8 @@
 HBitWriter *h_bit_writer_new(HAllocator* mm__) {
   HBitWriter *writer = h_new(HBitWriter, 1);
   memset(writer, 0, sizeof(*writer));
-  writer->buf = mm__->alloc(mm__, writer->capacity = 8);
-  if (!writer) {
-    return NULL;
-  }
+  writer->buf = h_alloc(mm__, writer->capacity = 8);
+  assert(writer != NULL);
   memset(writer->buf, 0, writer->capacity);
   writer->mm__ = mm__;
   writer->flags = BYTE_BIG_ENDIAN | BIT_BIG_ENDIAN;
diff --git a/src/internal.h b/src/internal.h
index 776f636..10db4b2 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -49,7 +49,7 @@
   rtype_t name##__m(HAllocator* mm__)
 // Functions with arguments are difficult to forward cleanly. Alas, we will need to forward them manually.
 
-#define h_new(type, count) ((type*)(mm__->alloc(mm__, sizeof(type)*(count))))
+#define h_new(type, count) ((type*)(h_alloc(mm__, sizeof(type)*(count))))
 #define h_free(addr) (mm__->free(mm__, (addr)))
 
 #ifndef __cplusplus
diff --git a/src/pprint.c b/src/pprint.c
index 11ec3d6..9c7c652 100644
--- a/src/pprint.c
+++ b/src/pprint.c
@@ -186,13 +186,11 @@ static void unamb_sub(const HParsedToken* tok, struct result_buf *buf) {
 
 char* h_write_result_unamb(const HParsedToken* tok) {
   struct result_buf buf = {
-    .output = (&system_allocator)->alloc(&system_allocator, 16),
+    .output = h_alloc(&system_allocator, 16),
     .len = 0,
     .capacity = 16
   };
-  if (!buf.output) {
-    return NULL;
-  }
+  assert(buf.output != NULL);
   unamb_sub(tok, &buf);
   append_buf_c(&buf, 0);
   return buf.output;
diff --git a/src/registry.c b/src/registry.c
index 2ebac1a..d905320 100644
--- a/src/registry.c
+++ b/src/registry.c
@@ -46,10 +46,8 @@ static int compare_entries(const void* v1, const void* v2) {
 }
 
 HTokenType h_allocate_token_type(const char* name) {
-  Entry* new_entry = (&system_allocator)->alloc(&system_allocator, sizeof(*new_entry));
-  if (!new_entry) {
-    return TT_INVALID;
-  }
+  Entry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
+  assert(new_entry != NULL);
   new_entry->name = name;
   new_entry->value = 0;
   Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries);

From 5996477a5e96086501fd9140aa877c08d65edc3c Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 30 Nov 2015 17:50:29 +0100
Subject: [PATCH 11/21] out-of-memory handling in packrat backend

---
 src/backends/packrat.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/backends/packrat.c b/src/backends/packrat.c
index abb198b..b7e47ae 100644
--- a/src/backends/packrat.c
+++ b/src/backends/packrat.c
@@ -254,6 +254,15 @@ static bool pos_equal(const void* key1, const void* key2) {
 
 HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStream *input_stream) {
   HArena * arena = h_new_arena(mm__, 0);
+
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    return NULL;
+  }
+
   HParseState *parse_state = a_new_(arena, HParseState, 1);
   parse_state->cache = h_hashtable_new(arena, cache_key_equal, // key_equal_func
 				       cache_key_hash); // hash_func

From d5f3e133095bb93e5557c95976f9e48476dfcc88 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Mon, 30 Nov 2015 18:07:33 +0100
Subject: [PATCH 12/21] out-of-memory handling in LL(k), LR, and GLR backends

---
 src/backends/glr.c | 10 ++++++++++
 src/backends/llk.c | 11 +++++++++++
 src/backends/lr.c  | 10 ++++++++++
 3 files changed, 31 insertions(+)

diff --git a/src/backends/glr.c b/src/backends/glr.c
index e753ea5..535dc28 100644
--- a/src/backends/glr.c
+++ b/src/backends/glr.c
@@ -198,6 +198,16 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
   HArena *arena  = h_new_arena(mm__, 0);    // will hold the results
   HArena *tarena = h_new_arena(mm__, 0);    // tmp, deleted after parse
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    h_delete_arena(tarena);
+    return NULL;
+  }
+
   // allocate engine lists (will hold one engine per state)
   // these are swapped each iteration
   HSlist *engines = h_slist_new(tarena);
diff --git a/src/backends/llk.c b/src/backends/llk.c
index 0ab4610..2f41f91 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -561,6 +561,17 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream*
 {
   HLLkState *s = llk_parse_start_(mm__, parser);
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(s->arena, &except);
+  h_arena_set_except(s->tarena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(s->arena);
+    h_delete_arena(s->tarena);
+    h_free(s);
+    return NULL;
+  }
+
   assert(stream->last_chunk);
   s->seq = llk_parse_chunk_(s, parser, stream);
 
diff --git a/src/backends/lr.c b/src/backends/lr.c
index fb256c0..8f2a0ea 100644
--- a/src/backends/lr.c
+++ b/src/backends/lr.c
@@ -388,6 +388,16 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
   HArena *tarena = h_new_arena(mm__, 0);    // tmp, deleted after parse
   HLREngine *engine = h_lrengine_new(arena, tarena, table, stream);
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except)) {
+    h_delete_arena(arena);
+    h_delete_arena(tarena);
+    return NULL;
+  }
+
   // iterate engine to completion
   while(h_lrengine_step(engine, h_lrengine_action(engine)));
 

From 2309bd6da901c2a8287463e5d4e408aa6f456e77 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 16:48:01 +0100
Subject: [PATCH 13/21] out-of-memory support for iterative LL(k)

---
 src/backends/llk.c | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/backends/llk.c b/src/backends/llk.c
index 2f41f91..3780cf1 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -389,6 +389,13 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   if(!seq)
     return NULL;  // parse already failed
 
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(arena, &except);
+  h_arena_set_except(tarena, &except);
+  if(setjmp(except))
+    goto no_parse;
+
   if(s->win.length > 0) {
     append_win(kmax, s, chunk);
     stream = &s->win;
@@ -530,8 +537,6 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   return seq;
 
  no_parse:
-  h_delete_arena(arena);
-  s->arena = NULL;
   return NULL;
 
  need_input:
@@ -550,6 +555,8 @@ static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s)
   if(s->seq) {
     assert(s->seq->used == 1);
     res = make_result(s->arena, s->seq->elements[0]);
+  } else {
+    h_delete_arena(s->arena);
   }
 
   h_delete_arena(s->tarena);
@@ -561,17 +568,6 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream*
 {
   HLLkState *s = llk_parse_start_(mm__, parser);
 
-  // out-of-memory handling
-  jmp_buf except;
-  h_arena_set_except(s->arena, &except);
-  h_arena_set_except(s->tarena, &except);
-  if(setjmp(except)) {
-    h_delete_arena(s->arena);
-    h_delete_arena(s->tarena);
-    h_free(s);
-    return NULL;
-  }
-
   assert(stream->last_chunk);
   s->seq = llk_parse_chunk_(s, parser, stream);
 

From 3ce4f61cf9253c6f746626a36e66501f94557523 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 16:49:40 +0100
Subject: [PATCH 14/21] add test for out-of-memory handling

---
 src/t_misc.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/src/t_misc.c b/src/t_misc.c
index 92c2b32..47afbe4 100644
--- a/src/t_misc.c
+++ b/src/t_misc.c
@@ -1,5 +1,6 @@
 #include <glib.h>
 #include <string.h>
+#include <sys/resource.h>
 #include "test_suite.h"
 #include "hammer.h"
 
@@ -29,7 +30,43 @@ static void test_tt_registry(void) {
   g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, 0);
 }
 
+// perform a big allocation during parsing to trigger out-of-memory handling
+static HParsedToken *act_big_alloc(const HParseResult *r, void *user) {
+  void *buf = h_arena_malloc(r->arena, 1024*1024*1024);
+  assert(buf != NULL);
+  return NULL;
+}
+static void test_oom(void) {
+  HParser *p = h_action(h_ch('x'), act_big_alloc, NULL);
+    // this should always fail, but never crash
+
+  struct rlimit bak, lim;
+  int i;
+  i = getrlimit(RLIMIT_DATA, &bak);
+  assert(i == 0);
+  lim.rlim_cur = 1000*1024*1024;   // never enough
+  if(lim.rlim_cur > bak.rlim_max)
+    lim.rlim_cur = bak.rlim_max;
+  lim.rlim_max = bak.rlim_max;
+  i = setrlimit(RLIMIT_DATA, &lim);
+  assert(i == 0);
+
+  g_check_parse_failed(p, PB_PACKRAT, "x",1);
+  g_check_parse_failed(p, PB_REGULAR, "x",1);
+  g_check_parse_failed(p, PB_LLk, "x",1);
+  g_check_parse_failed(p, PB_LALR, "x",1);
+  g_check_parse_failed(p, PB_GLR, "x",1);
+
+  g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1);
+  //g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1);
+  //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1);
+
+  i = setrlimit(RLIMIT_DATA, &bak);
+  assert(i == 0);
+}
+
 void register_misc_tests(void) {
   g_test_add_func("/core/misc/tt_user", test_tt_user);
   g_test_add_func("/core/misc/tt_registry", test_tt_registry);
+  g_test_add_func("/core/misc/oom", test_oom);
 }

From 384a7b939039a8409849e41df1b5d17144152e5c Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 17:07:04 +0100
Subject: [PATCH 15/21] reset arena jmp_bufs at end of h_llk_parse_chunk

---
 src/backends/llk.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/backends/llk.c b/src/backends/llk.c
index 3780cf1..6954e60 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -589,6 +589,9 @@ bool h_llk_parse_chunk(HSuspendedParser *s, HInputStream *input)
 
   state->seq = llk_parse_chunk_(state, s->parser, input);
 
+  h_arena_set_except(state->arena, NULL);
+  h_arena_set_except(state->tarena, NULL);
+
   return (state->seq == NULL || h_slist_empty(state->stack));
 }
 

From 22b5611cdfab7b460d03abadb2951d1eeedc19dd Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 17:08:02 +0100
Subject: [PATCH 16/21] add oom handling to iterative LR engine

---
 src/backends/lr.c | 13 +++++++++++++
 src/t_misc.c      |  3 ++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/backends/lr.c b/src/backends/lr.c
index 8f2a0ea..f2ac495 100644
--- a/src/backends/lr.c
+++ b/src/backends/lr.c
@@ -426,6 +426,16 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
   engine->input = *stream;
 
   bool run = true;
+
+  // out-of-memory handling
+  jmp_buf except;
+  h_arena_set_except(engine->arena, &except);
+  h_arena_set_except(engine->tarena, &except);
+  if(setjmp(except)) {
+    run = false;                            // done immediately
+    assert(engine->state != HLR_SUCCESS);   // h_parse_finish will return NULL
+  }
+
   while(run) {
     // check input against table to determine which action to take
     const HLRAction *action = h_lrengine_action(engine);
@@ -441,6 +451,9 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
       break;
   }
 
+  h_arena_set_except(engine->arena, NULL);
+  h_arena_set_except(engine->tarena, NULL);
+
   *stream = engine->input;
   return !run;  // done if engine no longer running
 }
diff --git a/src/t_misc.c b/src/t_misc.c
index 47afbe4..8cd73dc 100644
--- a/src/t_misc.c
+++ b/src/t_misc.c
@@ -57,8 +57,9 @@ static void test_oom(void) {
   g_check_parse_failed(p, PB_LALR, "x",1);
   g_check_parse_failed(p, PB_GLR, "x",1);
 
+  //g_check_parse_chunks_failed(p, PB_REGULAR, "",0, "x",1);
   g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1);
-  //g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1);
+  g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1);
   //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1);
 
   i = setrlimit(RLIMIT_DATA, &bak);

From e89d9f9134efb82d524bdc6defd6d09b2c526260 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 17:47:37 +0100
Subject: [PATCH 17/21] delete results properly in parser test macros

---
 src/test_suite.h | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/test_suite.h b/src/test_suite.h
index 49f13cf..d36820f 100644
--- a/src/test_suite.h
+++ b/src/test_suite.h
@@ -78,15 +78,6 @@
   } while(0)
 
   
-// TODO: replace uses of this with g_check_parse_failed
-#define g_check_failed(res) do {					\
-    const HParseResult *result = (res);					\
-    if (NULL != result) {						\
-      g_test_message("Check failed: shouldn't have succeeded, but did"); \
-      g_test_fail();							\
-    }									\
-  } while(0)
-
 #define g_check_parse_failed(parser, backend, input, inp_len) do {	\
     int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip != 0) {	\
@@ -94,8 +85,9 @@
       g_test_fail();							\
       break;	\
     }	\
-    const HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \
+    HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \
     if (NULL != result) {						\
+      h_parse_result_free(result);					\
       g_test_message("Check failed: shouldn't have succeeded, but did"); \
       g_test_fail();							\
     }									\
@@ -119,7 +111,7 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 
@@ -144,7 +136,7 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 
@@ -167,8 +159,9 @@
     }									\
     h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
     h_parse_chunk(s, (const uint8_t*)chunk2, c2_len);			\
-    const HParseResult *res = h_parse_finish(s);			\
+    HParseResult *res = h_parse_finish(s);				\
     if (NULL != res) {							\
+      h_parse_result_free(res);						\
       g_test_message("Check failed: shouldn't have succeeded, but did"); \
       g_test_fail();							\
     }									\
@@ -207,7 +200,7 @@
                      "Inefficiency: %5f%%",				\
 		     stats.used, stats.wasted,				\
 		     stats.wasted * 100. / (stats.used+stats.wasted));	\
-      h_delete_arena(res->arena);					\
+      h_parse_result_free(res);						\
     }									\
   } while(0)
 

From db4fd66eaf6f56a0fc90734ede702c3c92ffda9f Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Tue, 1 Dec 2015 18:12:21 +0100
Subject: [PATCH 18/21] try it with a smaller alloc

---
 src/t_misc.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/t_misc.c b/src/t_misc.c
index 8cd73dc..59d6260 100644
--- a/src/t_misc.c
+++ b/src/t_misc.c
@@ -32,8 +32,9 @@ static void test_tt_registry(void) {
 
 // perform a big allocation during parsing to trigger out-of-memory handling
 static HParsedToken *act_big_alloc(const HParseResult *r, void *user) {
-  void *buf = h_arena_malloc(r->arena, 1024*1024*1024);
+  void *buf = h_arena_malloc(r->arena, 500*1024*1024);
   assert(buf != NULL);
+  g_test_message("Memory allocation was supposed to fail");
   return NULL;
 }
 static void test_oom(void) {
@@ -44,7 +45,7 @@ static void test_oom(void) {
   int i;
   i = getrlimit(RLIMIT_DATA, &bak);
   assert(i == 0);
-  lim.rlim_cur = 1000*1024*1024;   // never enough
+  lim.rlim_cur = 499*1024*1024;   // never enough
   if(lim.rlim_cur > bak.rlim_max)
     lim.rlim_cur = bak.rlim_max;
   lim.rlim_max = bak.rlim_max;

From 7b13a82851f78a118e3014612ea8640fa49c6670 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 2 Dec 2015 13:28:24 +0100
Subject: [PATCH 19/21] add __m variants to some check macros

---
 src/test_suite.h | 59 +++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 51 insertions(+), 8 deletions(-)

diff --git a/src/test_suite.h b/src/test_suite.h
index d36820f..83359f9 100644
--- a/src/test_suite.h
+++ b/src/test_suite.h
@@ -78,14 +78,14 @@
   } while(0)
 
   
-#define g_check_parse_failed(parser, backend, input, inp_len) do {	\
-    int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
+#define g_check_parse_failed__m(mm__, parser, backend, input, inp_len) do { \
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip != 0) {	\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;	\
     }	\
-    HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \
+    HParseResult *result = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
     if (NULL != result) {						\
       h_parse_result_free(result);					\
       g_test_message("Check failed: shouldn't have succeeded, but did"); \
@@ -93,6 +93,9 @@
     }									\
   } while(0)
 
+#define g_check_parse_failed(p, be, input, len)				\
+    g_check_parse_failed__m(&system_allocator, p, be, input, len)
+
 #define g_check_parse_ok(parser, backend, input, inp_len) do {		\
     int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
     if(skip) {								\
@@ -140,18 +143,18 @@
     }									\
   } while(0)
 
-#define g_check_parse_chunks_failed(parser, backend, chunk1, c1_len, chunk2, c2_len) do {	\
-    int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
+#define g_check_parse_chunks_failed__m(mm__, parser, backend, chunk1, c1_len, chunk2, c2_len) do { \
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip) {								\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;								\
     }									\
-    g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len); \
+    g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len); \
   } while(0)
 
-#define g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len) do {	\
-    HSuspendedParser *s = h_parse_start(parser);			\
+#define g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len) do { \
+    HSuspendedParser *s = h_parse_start__m(mm__, parser);		\
     if(!s) {								\
       g_test_message("Chunk-wise parsing not available");		\
       g_test_fail();							\
@@ -167,6 +170,46 @@
     }									\
   } while(0)
 
+#define g_check_parse_chunks_failed(p, be, c1, c1_len, c2, c2_len) \
+  g_check_parse_chunks_failed__m(&system_allocator, p, be, c1, c1_len, c2, c2_len)
+
+#define g_check_parse_chunks_failed_(p, c1, c1_len, c2, c2_len) \
+  g_check_parse_chunks_failed___m(&system_allocator, p, c1, c1_len, c2, c2_len)
+
+#define g_check_parse_chunks_ok(parser, backend, chunk1, c1_len, chunk2, c2_len) do {	\
+    int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
+    if(skip) {								\
+      g_test_message("Compile failed");					\
+      g_test_fail();							\
+      break;								\
+    }									\
+    g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len); \
+  } while(0)
+
+#define g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len) do {	\
+    HSuspendedParser *s = h_parse_start(parser);			\
+    if(!s) {								\
+      g_test_message("Chunk-wise parsing not available");		\
+      g_test_fail();							\
+      break;								\
+    }									\
+    h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
+    h_parse_chunk(s, (const uint8_t*)chunk2, c2_len);			\
+    HParseResult *res = h_parse_finish(s);				\
+    if (!res) {								\
+      g_test_message("Parse failed on line %d", __LINE__);		\
+      g_test_fail();							\
+    } else {								\
+      HArenaStats stats;						\
+      h_allocator_stats(res->arena, &stats);				\
+      g_test_message("Parse used %zd bytes, wasted %zd bytes. "		\
+                     "Inefficiency: %5f%%",				\
+		     stats.used, stats.wasted,				\
+		     stats.wasted * 100. / (stats.used+stats.wasted));	\
+      h_parse_result_free(res);						\
+    }									\
+  } while(0)
+
 #define g_check_parse_chunks_match(parser, backend, chunk1, c1_len, chunk2, c2_len, result) do { \
     int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
     if(skip) {								\

From 9602caf64f055e3ac312d9c8ab01c7d8dc573b97 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Wed, 2 Dec 2015 13:28:54 +0100
Subject: [PATCH 20/21] test out-of-memory handling with a mock allocator

---
 src/t_misc.c | 67 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 29 deletions(-)

diff --git a/src/t_misc.c b/src/t_misc.c
index 59d6260..04ce96c 100644
--- a/src/t_misc.c
+++ b/src/t_misc.c
@@ -30,41 +30,50 @@ static void test_tt_registry(void) {
   g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, 0);
 }
 
-// perform a big allocation during parsing to trigger out-of-memory handling
-static HParsedToken *act_big_alloc(const HParseResult *r, void *user) {
-  void *buf = h_arena_malloc(r->arena, 500*1024*1024);
+// test out-of-memory handling with a selectively failing allocator
+static void *fail_alloc(HAllocator *mm__, size_t size) {
+  if(size - 0xdead <= 0x30) // allow for overhead of arena link structure
+    return NULL;
+  return system_allocator.alloc(&system_allocator, size);
+}
+static void *fail_realloc(HAllocator *mm__, void *ptr, size_t size) {
+  return system_allocator.realloc(&system_allocator, ptr, size);
+}
+static void fail_free(HAllocator *mm__, void *ptr) {
+  return system_allocator.free(&system_allocator, ptr);
+}
+static HAllocator fail_allocator = {fail_alloc, fail_realloc, fail_free};
+static HParsedToken *act_oom(const HParseResult *r, void *user) {
+  void *buf = h_arena_malloc(r->arena, 0xdead);
   assert(buf != NULL);
-  g_test_message("Memory allocation was supposed to fail");
-  return NULL;
+  return NULL;  // succeed with null token
 }
 static void test_oom(void) {
-  HParser *p = h_action(h_ch('x'), act_big_alloc, NULL);
+  HParser *p = h_action(h_ch('x'), act_oom, NULL);
     // this should always fail, but never crash
 
-  struct rlimit bak, lim;
-  int i;
-  i = getrlimit(RLIMIT_DATA, &bak);
-  assert(i == 0);
-  lim.rlim_cur = 499*1024*1024;   // never enough
-  if(lim.rlim_cur > bak.rlim_max)
-    lim.rlim_cur = bak.rlim_max;
-  lim.rlim_max = bak.rlim_max;
-  i = setrlimit(RLIMIT_DATA, &lim);
-  assert(i == 0);
+  // sanity-check: parses should succeed with the normal allocator...
+  g_check_parse_ok(p, PB_PACKRAT, "x",1);
+  g_check_parse_ok(p, PB_REGULAR, "x",1);
+  g_check_parse_ok(p, PB_LLk, "x",1);
+  g_check_parse_ok(p, PB_LALR, "x",1);
+  g_check_parse_ok(p, PB_GLR, "x",1);
+  //XXX g_check_parse_chunks_ok(p, PB_REGULAR, "",0, "x",1);
+  g_check_parse_chunks_ok(p, PB_LLk, "",0, "x",1);
+  g_check_parse_chunks_ok(p, PB_LALR, "",0, "x",1);
+  //XXX g_check_parse_chunks_ok(p, PB_GLR, "",0, "x",1);
 
-  g_check_parse_failed(p, PB_PACKRAT, "x",1);
-  g_check_parse_failed(p, PB_REGULAR, "x",1);
-  g_check_parse_failed(p, PB_LLk, "x",1);
-  g_check_parse_failed(p, PB_LALR, "x",1);
-  g_check_parse_failed(p, PB_GLR, "x",1);
-
-  //g_check_parse_chunks_failed(p, PB_REGULAR, "",0, "x",1);
-  g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1);
-  g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1);
-  //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1);
-
-  i = setrlimit(RLIMIT_DATA, &bak);
-  assert(i == 0);
+  // ...and fail gracefully with the broken one
+  HAllocator *mm__ = &fail_allocator;
+  g_check_parse_failed__m(mm__, p, PB_PACKRAT, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_REGULAR, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_LLk, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_LALR, "x",1);
+  g_check_parse_failed__m(mm__, p, PB_GLR, "x",1);
+  //XXX g_check_parse_chunks_failed__m(mm__, p, PB_REGULAR, "",0, "x",1);
+  g_check_parse_chunks_failed__m(mm__, p, PB_LLk, "",0, "x",1);
+  g_check_parse_chunks_failed__m(mm__, p, PB_LALR, "",0, "x",1);
+  //XXX g_check_parse_chunks_failed__m(mm__, p, PB_GLR, "",0, "x",1);
 }
 
 void register_misc_tests(void) {

From d28f182c4f62990899609e674239c4f134e834b8 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Fri, 4 Dec 2015 13:13:06 +0100
Subject: [PATCH 21/21] clear arena exception handler when exiting
 llk_parse_chunk_

---
 src/backends/llk.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/backends/llk.c b/src/backends/llk.c
index 6954e60..4e8209b 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -383,10 +383,9 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   HArena *arena = s->arena;
   HArena *tarena = s->tarena;
   HSlist *stack = s->stack;
-  HCountedArray *seq = s->seq;
   size_t kmax = table->kmax;
 
-  if(!seq)
+  if(!s->seq)
     return NULL;  // parse already failed
 
   // out-of-memory handling
@@ -396,6 +395,8 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   if(setjmp(except))
     goto no_parse;
 
+  HCountedArray *seq = s->seq;
+
   if(s->win.length > 0) {
     append_win(kmax, s, chunk);
     stream = &s->win;
@@ -534,10 +535,15 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   // since we started with a single nonterminal on the stack, seq should
   // contain exactly the parse result.
   assert(seq->used == 1);
+
+ end:
+  h_arena_set_except(arena, NULL);
+  h_arena_set_except(tarena, NULL);
   return seq;
 
  no_parse:
-  return NULL;
+  seq = NULL;
+  goto end;
 
  need_input:
   if(stream->last_chunk)
@@ -545,7 +551,7 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
   if(tok)
     h_arena_free(arena, tok);   // no result, yet
   h_slist_push(stack, x);       // try this symbol again next time
-  return seq;
+  goto end;
 }
 
 static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s)