Merge tq's regex backend fixes (branch 'master' of https://github.com/thequux/hammer)

Makefile

@@ -11,7 +11,10 @@ CONFIG_VARS= INCLUDE_TESTS
 
 .DEFAULT_GOAL := all
 
-%:
+nojni: all
+nojni: SUBDIRS:=$(filter-out jni,$(SUBDIRS))
+
+all clean:
 	+for dir in $(SUBDIRS); do $(MAKE) -C $${dir} $@; done
 
 test: src/test_suite

common.mk

@@ -6,7 +6,7 @@ endif
 include $(TOPLEVEL)/config.mk
 
 TEST_CFLAGS = $(shell pkg-config --cflags glib-2.0) -DINCLUDE_TESTS
-TEST_LDFLAGS = $(shell pkg-config --libs glib-2.0) -lrt
+TEST_LDFLAGS = $(shell pkg-config --libs glib-2.0) -lrt -ldl
 
 CFLAGS := -std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -g
 LDFLAGS :=

src/Makefile

@@ -83,3 +83,5 @@ test: test_suite
 
 test_suite: $(TESTS) libhammer.a
 	$(call hush, "Linking $@") $(CC) -o $@ $^ $(LDFLAGS) $(TEST_LDFLAGS)
+
+backends/regex.o: backends/regex_debug.c

src/backends/regex.c

@@ -1,3 +1,4 @@
+#define _GNU_SOURCE
 #include <string.h>
 #include <assert.h>
 #include "../internal.h"
@@ -13,6 +14,7 @@ typedef enum HSVMOp_ {
   SVM_ACTION, // Same meaning as RVM_ACTION
   SVM_CAPTURE, // Same meaning as RVM_CAPTURE
   SVM_ACCEPT,
+  SVM_OPCOUNT
 } HSVMOp;
 
 typedef struct HRVMTrace_ {
@@ -42,8 +44,8 @@ HRVMTrace *invert_trace(HRVMTrace *trace) {
     trace->next = last;
     last = trace;
     trace = next;
-  } while (trace->next);
-  return trace;
+  } while (trace);
+  return last;
 }
 
 void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_t len) {
@@ -96,7 +98,7 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
       if (!heads_p[ip_s])
 	continue;
       THREAD.ip = ip_s;
-
+      THREAD.trace = heads_p[ip_s];
       uint8_t hi, lo;
       uint16_t arg;
       while(ipq_top > 0) {
@@ -110,8 +112,6 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
 	  ret_trace = THREAD.trace;
 	  goto run_trace;
 	case RVM_MATCH:
-	  // Doesn't actually validate the "must be followed by MATCH
-	  // or STEP. It should. Preproc perhaps?
 	  hi = (arg >> 8) & 0xff;
 	  lo = arg & 0xff;
 	  THREAD.ip++;
@@ -151,7 +151,7 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
 	case RVM_STEP:
 	  // save thread
 	  live_threads++;
-	  heads_n[THREAD.ip++] = THREAD.trace;
+	  heads_n[++THREAD.ip] = THREAD.trace;
 	  ipq_top--;
 	  goto next_insn;
 	}
@@ -169,7 +169,6 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
  run_trace:
   // Invert the direction of the trace linked list.
 
-  
   ret_trace = invert_trace(ret_trace);
   HParseResult *ret = run_trace(mm__, prog, ret_trace, input, len);
   // ret is in its own arena
@@ -221,20 +220,24 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
     case SVM_CAPTURE: 
       // Top of stack must be a mark
       // This replaces said mark in-place with a TT_BYTES.
-      assert(ctx.stack[ctx.stack_count]->token_type == TT_MARK);
+      assert(ctx.stack[ctx.stack_count-1]->token_type == TT_MARK);
       
-      tmp_res = ctx.stack[ctx.stack_count];
+      tmp_res = ctx.stack[ctx.stack_count-1];
       tmp_res->token_type = TT_BYTES;
       // TODO: Will need to copy if bit_offset is nonzero
       assert(tmp_res->bit_offset == 0);
 	
       tmp_res->bytes.token = input + tmp_res->index;
-      tmp_res->bytes.len = cur->input_pos - tmp_res->index + 1; // inclusive
+      tmp_res->bytes.len = cur->input_pos - tmp_res->index;
       break;
     case SVM_ACCEPT:
-      assert(ctx.stack_count == 1);
+      assert(ctx.stack_count <= 1);
 	HParseResult *res = a_new(HParseResult, 1);
+      if (ctx.stack_count == 1) {
 	res->ast = ctx.stack[0];
+      } else {
+	res->ast = NULL;
+      }
       res->bit_length = cur->input_pos * 8;
       res->arena = arena;
       return res;
@@ -351,6 +354,7 @@ static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params
     h_free(prog);
     return 2;
   }
+  h_rvm_insert_insn(prog, RVM_ACCEPT, 0);
   parser->backend_data = prog;
   return 0;
 }
@@ -364,3 +368,7 @@ HParserBackendVTable h__regex_backend_vtable = {
   .parse = h_regex_parse,
   .free = h_regex_free
 };
+
+#ifndef NDEBUG
+#include "regex_debug.c"
+#endif

src/backends/regex_debug.c

@@ -0,0 +1,100 @@
+// Intended to be included from regex_debug.c
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <malloc.h>
+
+
+
+// This is some spectacularly non-portable code... but whee!
+#include <dlfcn.h>
+char* getsym(void* addr) {
+  Dl_info dli;
+  char* retstr;
+  if (dladdr(addr, &dli) != 0 && dli.dli_sname != NULL) {
+    if (dli.dli_saddr == addr)
+      return strdup(dli.dli_sname);
+    else
+      asprintf(&retstr, "%s+0x%lx", dli.dli_sname, addr - dli.dli_saddr);
+  } else
+    asprintf(&retstr, "%p", addr);
+
+  return retstr;
+}
+
+const char* rvm_op_names[RVM_OPCOUNT] = {
+  "ACCEPT",
+  "GOTO",
+  "FORK",
+  "PUSH",
+  "ACTION",
+  "CAPTURE",
+  "EOF",
+  "MATCH",
+  "STEP"
+};
+
+const char* svm_op_names[SVM_OPCOUNT] = {
+  "PUSH",
+  "NOP",
+  "ACTION",
+  "CAPTURE",
+  "ACCEPT"
+};
+
+void dump_rvm_prog(HRVMProg *prog) {
+  char* symref;
+  for (unsigned int i = 0; i < prog->length; i++) {
+    HRVMInsn *insn = &prog->insns[i];
+    printf("%4d %-10s", i, rvm_op_names[insn->op]);
+    switch (insn->op) {
+    case RVM_GOTO:
+    case RVM_FORK:
+      printf("%hd\n", insn->arg);
+      break;
+    case RVM_ACTION:
+      symref = getsym(prog->actions[insn->arg].action);
+      // TODO: somehow format the argument to action
+      printf("%s\n", symref);
+      free(symref);
+      break;
+    case RVM_MATCH: {
+      uint8_t low, high;
+      low = insn->arg & 0xff;
+      high = (insn->arg >> 8) & 0xff;
+      if (high < low)
+	printf("NONE\n");
+      else {
+	if (low >= 0x32 && low <= 0x7e)
+	  printf("%02hhx ('%c')", low, low);
+	else
+	  printf("%02hhx", low);
+	
+	if (high >= 0x32 && high <= 0x7e)
+	  printf(" - %02hhx ('%c')\n", high, high);
+	else
+	  printf(" - %02hhx\n", high);
+      }
+      break;
+    }
+    default:
+      printf("\n");
+    }
+  }
+}
+
+void dump_svm_prog(HRVMProg *prog, HRVMTrace *trace) {
+  char* symref;
+  for (; trace != NULL; trace = trace->next) {
+    printf("@%04zd %-10s", trace->input_pos, svm_op_names[trace->opcode]);
+    switch (trace->opcode) {
+    case SVM_ACTION:
+      symref = getsym(prog->actions[trace->arg].action);
+      // TODO: somehow format the argument to action
+      printf("%s\n", symref);
+      free(symref);
+      break;
+    default:
+      printf("\n");
+    }
+  }
+}

src/parsers/bits.c

@@ -97,6 +97,8 @@ static bool h_svm_action_bits(HArena *arena, HSVMContext *ctx, void* env) {
   uint64_t res = 0;
   for (size_t i = 0; i < top->bytes.len; i++)
     res = (res << 8) | top->bytes.token[i];   // TODO: Handle other endiannesses.
+  uint64_t msb = (env_->signedp ? 1LL:0) << (top->bytes.len * 8 - 1);
+  res = (res ^ msb) - msb;
   top->uint = res; // possibly cast to signed through union
   top->token_type = (env_->signedp ? TT_SINT : TT_UINT);
   return true;
@@ -105,7 +107,7 @@ static bool h_svm_action_bits(HArena *arena, HSVMContext *ctx, void* env) {
 static bool bits_ctrvm(HRVMProg *prog, void* env) {
   struct bits_env *env_ = (struct bits_env*)env;
   h_rvm_insert_insn(prog, RVM_PUSH, 0);
-  for (size_t i=0; (i < env_->length)/8; ++i) { // FUTURE: when we can handle non-byte-aligned, the env_->length/8 part will be different
+  for (size_t i=0; i < (env_->length/8); ++i) { // FUTURE: when we can handle non-byte-aligned, the env_->length/8 part will be different
     h_rvm_insert_insn(prog, RVM_MATCH, 0xFF00);
     h_rvm_insert_insn(prog, RVM_STEP, 0);
   }

src/parsers/ch.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include "parser_internal.h"
 
 static HParseResult* parse_ch(void* env, HParseState *state) {
@@ -20,11 +21,26 @@ static HCFChoice* desugar_ch(HAllocator *mm__, void *env) {
   return ret;
 }
 
+static bool h_svm_action_ch(HArena *arena, HSVMContext *ctx, void* env) {
+  // BUG: relies un undefined behaviour: int64_t is a signed uint64_t; not necessarily true on 32-bit
+  HParsedToken *top = ctx->stack[ctx->stack_count-1];
+  assert(top->token_type == TT_BYTES);
+  uint64_t res = 0;
+  for (size_t i = 0; i < top->bytes.len; i++)
+    res = (res << 8) | top->bytes.token[i];   // TODO: Handle other endiannesses.
+  top->uint = res; // possibly cast to signed through union
+  top->token_type = TT_UINT;
+  return true;
+}
+
 static bool ch_ctrvm(HRVMProg *prog, void* env) {
   uint8_t c = (uint8_t)(unsigned long)(env);
   // TODO: Does this capture anything?
-  h_rvm_insert_insn(prog, RVM_MATCH, c & c << 8);
+  h_rvm_insert_insn(prog, RVM_PUSH, 0);
+  h_rvm_insert_insn(prog, RVM_MATCH, c | c << 8);
   h_rvm_insert_insn(prog, RVM_STEP, 0);
+  h_rvm_insert_insn(prog, RVM_CAPTURE, 0);
+  h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_ch, env));
   return true;
 }
 

src/parsers/charset.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <string.h>
 #include "../internal.h"
 #include "parser_internal.h"
@@ -22,23 +23,54 @@ static HCFChoice* desugar_charset(HAllocator *mm__, void *env) {
   return ret;
 }
 
+static bool h_svm_action_ch(HArena *arena, HSVMContext *ctx, void* env) {
+  // BUG: relies un undefined behaviour: int64_t is a signed uint64_t; not necessarily true on 32-bit
+  HParsedToken *top = ctx->stack[ctx->stack_count-1];
+  assert(top->token_type == TT_BYTES);
+  uint64_t res = 0;
+  for (size_t i = 0; i < top->bytes.len; i++)
+    res = (res << 8) | top->bytes.token[i];   // TODO: Handle other endiannesses.
+  top->uint = res; // possibly cast to signed through union
+  top->token_type = TT_UINT;
+  return true;
+}
+
 // FUTURE: this is horribly inefficient
 static bool cs_ctrvm(HRVMProg *prog, void *env) {
   HCharset cs = (HCharset)env;
+  h_rvm_insert_insn(prog, RVM_PUSH, 0);
+
   uint16_t start = h_rvm_get_ip(prog);
-  for (size_t i=0; i<256; ++i) {
-    if (charset_isset(cs, i)) {
+
+  uint8_t range_start = 0;
+  bool collecting = false;
+  for (size_t i=0; i<257; ++i) {
+    // Position 256 is only there so that every included character has
+    // a non-included character after it.
+    if (i < 256 && charset_isset(cs, i)) {
+      if (!collecting) {
+	collecting = true;
+	range_start = i;
+      }
+    } else {
+      if (collecting) {
+	collecting = false;
 	uint16_t insn = h_rvm_insert_insn(prog, RVM_FORK, 0);
-      h_rvm_insert_insn(prog, RVM_MATCH, i & i << 8);
+	h_rvm_insert_insn(prog, RVM_MATCH, range_start | i << 8);
 	h_rvm_insert_insn(prog, RVM_GOTO, 0);
 	h_rvm_patch_arg(prog, insn, h_rvm_get_ip(prog));
       }
     }
+  }
+  h_rvm_insert_insn(prog, RVM_MATCH, 0x00FF);
   uint16_t jump = h_rvm_insert_insn(prog, RVM_STEP, 0);
   for (size_t i=start; i<jump; ++i) {
     if (RVM_GOTO == prog->insns[i].op)
       h_rvm_patch_arg(prog, i, jump);
   }
+
+  h_rvm_insert_insn(prog, RVM_CAPTURE, 0);
+  h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_ch, env));
   return true;
 }
 

src/parsers/ignoreseq.c

@@ -97,7 +97,7 @@ static bool is_ctrvm(HRVMProg *prog, void* env) {
   HIgnoreSeq *seq = (HIgnoreSeq*)env;
   for (size_t i=0; i<seq->len; ++i) {
     h_rvm_insert_insn(prog, RVM_PUSH, 0);
-    if (!h_compile_regex(prog, seq->parsers[i]->env))
+    if (!h_compile_regex(prog, seq->parsers[i]))
       return false;
   }
   h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_ignoreseq, env));

src/parsers/token.c

@@ -69,7 +69,7 @@ static bool token_ctrvm(HRVMProg *prog, void *env) {
   HToken *t = (HToken*)env;
   h_rvm_insert_insn(prog, RVM_PUSH, 0);
   for (int i=0; i<t->len; ++i) {
-    h_rvm_insert_insn(prog, RVM_MATCH, t->str[i] & t->str[i] << 8);
+    h_rvm_insert_insn(prog, RVM_MATCH, t->str[i] | t->str[i] << 8);
     h_rvm_insert_insn(prog, RVM_STEP, 0);
   }
   h_rvm_insert_insn(prog, RVM_CAPTURE, 0);

src/parsers/whitespace.c

@@ -67,13 +67,19 @@ static bool ws_ctrvm(HRVMProg *prog, void *env) {
   uint16_t start = h_rvm_get_ip(prog);
   uint16_t next;
 
-  for (int i = 0; i < 6; i++) {
+  uint16_t ranges[2] = {
+    0x0d09,
+    0x2020,
+  };
+  
+  for (int i = 0; i < 2; i++) {
     next = h_rvm_insert_insn(prog, RVM_FORK, 0);
-    h_rvm_insert_insn(prog, RVM_MATCH, (SPACE_CHRS[i] << 8) | (SPACE_CHRS[i]));
+    h_rvm_insert_insn(prog, RVM_MATCH, ranges[i]);
+    h_rvm_insert_insn(prog, RVM_STEP, 0);
     h_rvm_insert_insn(prog, RVM_GOTO, start);
     h_rvm_patch_arg(prog, next, h_rvm_get_ip(prog));
   }
-  return h_compile_regex(prog, p->env);
+  return h_compile_regex(prog, p);
 }
 
 static const HParserVtable whitespace_vt = {

src/system_allocator.c

@@ -21,6 +21,7 @@ static void* system_realloc(HAllocator *allocator, void* ptr, size_t size) {
 }
 
 static void system_free(HAllocator *allocator, void* ptr) {
+  if (ptr != NULL)
     free(ptr - sizeof(size_t));
 }