Benchmark is actually broken right now, for mysterious double-free related reasons. I will dig into this tomorrow.

src/benchmark.c

@@ -20,23 +20,39 @@
 
 */
 
+#define false 0
+#define true 1
+
+#include <stdlib.h>
 
 HBenchmarkResults *h_benchmark(const HParser* parser, HParserTestcase* testcases) {
   // For now, just output the results to stderr
   HParserTestcase* tc = testcases;
   HParserBackend backend = PB_MIN;
+  HBenchmarkResults *ret = (HBenchmarkResults*)malloc(sizeof(HBenchmarkResults*));
+  ret->len = PB_MAX-PB_MIN;
+  ret->results = (HBackendResults*)malloc(ret->len * sizeof(HBackendResults*));
 
   for (backend = PB_MIN; backend < PB_MAX; backend++) {
     fprintf(stderr, "Compiling for backend %d ... ", backend);
+    ret->results[backend].backend = backend;
     // Step 1: Compile grammar for given parser...
     if (h_compile(parser, PB_MIN, NULL) == -1) {
       // backend inappropriate for grammar...
       fprintf(stderr, "failed\n");
+      ret->results[backend].compile_success = false;
+      ret->results[backend].n_testcases = 0;
+      ret->results[backend].failed_testcases = 0;
+      ret->results[backend].cases = NULL;
       continue;
     }
+    ret->results[backend].compile_success = true;
     int tc_failed = 0;
     // Step 1: verify all test cases.
+    ret->results[backend].n_testcases = 0;
+    ret->results[backend].failed_testcases = 0;
     for (tc = testcases; tc->input != NULL; tc++) {
+      ret->results[backend].n_testcases++;
       HParseResult *res = h_parse(parser, tc->input, tc->length);
       char* res_unamb;
       if (res != NULL) {
@@ -51,6 +67,7 @@ HBenchmarkResults *h_benchmark(const HParser* parser, HParserTestcase* testcases
 	// report. (eg, if users are trying to fix a grammar for a
 	// faster backend)
 	tc_failed++;
+	ret->results[backend].failed_testcases++;
       }
       h_parse_result_free(res);
     }
@@ -61,6 +78,9 @@ HBenchmarkResults *h_benchmark(const HParser* parser, HParserTestcase* testcases
       continue;
     }
 
+    ret->results[backend].cases = (HCaseResult*)malloc(ret->results[backend].n_testcases * sizeof(HCaseResult*));
+    size_t cur_case = 0;
+
     for (tc = testcases; tc->input != NULL; tc++) {
       // The goal is to run each testcase for at least 50ms each
       // TODO: replace this with a posix timer-based benchmark. (cf. timerfd_create, timer_create, setitimer)
@@ -78,7 +98,9 @@ HBenchmarkResults *h_benchmark(const HParser* parser, HParserTestcase* testcases
 	// time_diff is in ns
 	time_diff = (ts_end.tv_sec - ts_start.tv_sec) * 1000000000 + (ts_end.tv_nsec - ts_start.tv_nsec);
       } while (time_diff < 100000000);
+      ret->results[backend].cases[cur_case].parse_time = (time_diff / count);
       fprintf(stderr, "Case %d: %lld ns/parse\n", (int)(tc - testcases),  time_diff / count);
+      cur_case++;
     }
   }
   return NULL;

src/hammer.h

@@ -125,7 +125,25 @@ typedef struct HParserTestcase_ {
   char* output_unambiguous;
 } HParserTestcase;
 
+typedef struct HCaseResult_ {
+  bool success;
+  union {
+    const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
+    size_t parse_time; // on success, filled in with time for a single parse, in nsec
+  };
+} HCaseResult;
+
+typedef struct HBackendResults_ {
+  HParserBackend backend;
+  bool compile_success;
+  size_t n_testcases;
+  size_t failed_testcases; // actually a count...
+  HCaseResult *cases;
+} HBackendResults;
+
 typedef struct HBenchmarkResults_ {
+  size_t len;
+  HBackendResults *results;
 } HBenchmarkResults;
 // }}}