add API and backend hooks for iterative (chunked) parsing

src/hammer.c

@@ -43,6 +43,7 @@ typedef struct {
 
 
 
+#define DEFAULT_ENDIANNESS (BIT_BIG_ENDIAN | BYTE_BIG_ENDIAN)
 
 HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length) {
   return h_parse__m(&system_allocator, parser, input, length);
@@ -53,7 +54,7 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t*
     .index = 0,
     .bit_offset = 0,
     .overrun = 0,
-    .endianness = BIT_BIG_ENDIAN | BYTE_BIG_ENDIAN,
+    .endianness = DEFAULT_ENDIANNESS,
     .length = length,
     .input = input
   };
@@ -96,3 +97,59 @@ int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, cons
     parser->backend = backend;
   return ret;
 }
+
+
+HSuspendedParser* h_parse_start(const HParser* parser) {
+  return h_parse_start__m(&system_allocator, parser);
+}
+HSuspendedParser* h_parse_start__m(HAllocator* mm__, const HParser* parser) {
+  if(!backends[parser->backend]->parse_start)
+    return NULL;
+
+  // allocate and init suspended state
+  HSuspendedParser *s = h_new(HSuspendedParser, 1);
+  if(!s)
+    return NULL;
+  s->mm__ = mm__;
+  s->parser = parser;
+  s->backend_state = NULL;
+  s->endianness = DEFAULT_ENDIANNESS;
+
+  // backend-specific initialization
+  // should allocate s->backend_state
+  backends[parser->backend]->parse_start(s);
+
+  return s;
+}
+
+bool h_parse_chunk(HSuspendedParser* s, const uint8_t* input, size_t length) {
+  assert(backends[s->parser->backend]->parse_chunk != NULL);
+
+  // input 
+  HInputStream input_stream = {
+    .index = 0,
+    .bit_offset = 0,
+    .overrun = 0,
+    .endianness = s->endianness,
+    .length = length,
+    .input = input
+  };
+
+  // process chunk
+  backends[s->parser->backend]->parse_chunk(s, &input_stream);
+  s->endianness = input_stream.endianness;
+
+  return !input_stream.overrun; // parser wants no more input? done.
+}
+
+HParseResult* h_parse_finish(HSuspendedParser* s) {
+  assert(backends[s->parser->backend]->parse_finish != NULL);
+
+  HAllocator *mm__ = s->mm__;
+
+  HParseResult *r = backends[s->parser->backend]->parse_finish(s);
+    // NB: backend should have freed backend_state
+  h_free(s);
+
+  return r;
+}

src/hammer.h

@@ -140,6 +140,8 @@ typedef struct HParser_ {
   HCFChoice *desugared; /* if the parser can be desugared, its desugared form */
 } HParser;
 
+typedef struct HSuspendedParser_ HSuspendedParser;
+
 /**
  * Type of an action to apply to an AST, used in the action() parser. 
  * It can be any (user-defined) function that takes a HParseResult*
@@ -265,6 +267,27 @@ typedef struct HBenchmarkResults_ {
  */
 HAMMER_FN_DECL(HParseResult*, h_parse, const HParser* parser, const uint8_t* input, size_t length);
 
+/**
+ * Initialize a parser for iteratively consuming an input stream in chunks.
+ * This is only supported by some backends.
+ *
+ * Result is NULL if not supported by the backend.
+ */
+HAMMER_FN_DECL(HSuspendedParser*, h_parse_start, const HParser* parser);
+
+/**
+ * Run a suspended parser (as returned by h_parse_start) on a chunk of input.
+ *
+ * Returns true if the parser is done (needs no more input).
+ */
+bool h_parse_chunk(HSuspendedParser* s, const uint8_t* input, size_t length);
+
+/**
+ * Finish an iterative parse. Signals the end of input to the backend and
+ * returns the parse result.
+ */
+HParseResult* h_parse_finish(HSuspendedParser* s);
+
 /**
  * Given a string, returns a parser that parses that string value. 
  * 

src/internal.h

@@ -210,10 +210,28 @@ struct HParseState_ {
   HSlist *symbol_table; // its contents are HHashTables
 };
 
+struct HSuspendedParser_ {
+  HAllocator *mm__;
+  const HParser *parser;
+  void *backend_state;
+
+  // the only part of HInputStream that carries across chunks
+  uint8_t endianness;
+};
+
 typedef struct HParserBackendVTable_ {
   int (*compile)(HAllocator *mm__, HParser* parser, const void* params);
   HParseResult* (*parse)(HAllocator *mm__, const HParser* parser, HInputStream* stream);
   void (*free)(HParser* parser);
+
+  void (*parse_start)(HSuspendedParser *s);
+    // parse_start should allocate backend_state.
+  void (*parse_chunk)(HSuspendedParser *s, HInputStream *input);
+    // when parse_chunk leaves input.overrun unset, parse is done. else:
+    // parse_chunk MUST consume all input, integrating it into backend_state.
+    // calling parse_chunk again after parse is done should have no effect.
+  HParseResult *(*parse_finish)(HSuspendedParser *s);
+    // parse_finish must free backend_state.
 } HParserBackendVTable;