Still doesn't build, but desugaring is farther along

src/parsers/action.c

@@ -19,6 +19,21 @@ static HParseResult* parse_action(void *env, HParseState *state) {
     return NULL;
 }
 
+static HCFChoice* desugar_action(HAllocator *mm__, void *env) {
+  HParseAction *a = (HParseAction*)env;
+  HCFSequence *seq = h_new(HCFSequence, 1);
+  seq->items = h_new(HCFChoice*, 2);
+  seq->items[0] = a->p->vtable->desugar(mm__, a->p->env);
+  seq->items[1] = NULL;
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 2);
+  ret->seq[0] = seq;
+  ret->seq[1] = NULL;
+  ret->action = a->action;
+  return ret;
+}
+
 static bool action_isValidRegular(void *env) {
   HParseAction *a = (HParseAction*)env;
   return a->p->vtable->isValidRegular(a->p->env);
@@ -33,6 +48,7 @@ static const HParserVtable action_vt = {
   .parse = parse_action,
   .isValidRegular = action_isValidRegular,
   .isValidCF = action_isValidCF,
+  .desugar = desugar_action,
 };
 
 const HParser* h_action(const HParser* p, const HAction a) {

src/parsers/and.c

@@ -9,6 +9,11 @@ static HParseResult *parse_and(void* env, HParseState* state) {
   return NULL;
 }
 
+static const HCFChoice* desugar_and(HAllocator *mm__, void *env) {
+  assert_message(0, "Not context-free, can't be desugared");
+  return NULL;
+}
+
 static const HParserVtable and_vt = {
   .parse = parse_and,
   .isValidRegular = h_false, /* TODO: strictly speaking this should be regular,
@@ -16,6 +21,7 @@ static const HParserVtable and_vt = {
 				to get right, so we're leaving it for a future
 				revision. --mlp, 18/12/12 */
   .isValidCF = h_false,      /* despite TODO above, this remains false. */
+  .desugar = desugar_and,
 };
 
 

src/parsers/bits.c

@@ -16,10 +16,36 @@ static HParseResult* parse_bits(void* env, HParseState *state) {
   return make_result(state, result);
 }
 
+static HCFChoice* desugar_bits(HAllocator *mm__, void *env) {
+  struct bits_env *bits = (struct bits_env*)env;
+  if (0 != bits->length % 8)
+    return NULL; // can't handle non-byte-aligned for now
+  HCFSequence *seq = h_new(HCFSequence, 1);
+  seq->items = h_new(HCFChoice*, bits->length/8);
+  HCharset match_all = new_charset(mm__);
+  HCFChoice *match_all_choice = h_new(HCFChoice, 1);
+  match_all_choice->type = HCF_CHARSET;
+  match_all_choice->charset = match_all;
+  match_all_choice->action = NULL;
+  for (int i = 0; i < 256; i++)
+    charset_set(match_all, i, 1);
+  for (size_t i=0; i<bits->length/8; ++i) {
+    seq->items[i] = match_all_choice;
+  }
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 2);
+  ret->seq[0] = seq;
+  ret->seq[1] = NULL;
+  ret->action = NULL;
+  return ret;
+}
+
 static const HParserVtable bits_vt = {
   .parse = parse_bits,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_bits,
 };
 const HParser* h_bits(size_t len, bool sign) {
   return h_bits__m(&system_allocator, len, sign);

src/parsers/ch.c

@@ -12,10 +12,19 @@ static HParseResult* parse_ch(void* env, HParseState *state) {
   }
 }
 
+static HCFChoice* desugar_ch(HAllocator *mm__, void *env) {
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHAR;
+  ret->chr = (uint8_t)(unsigned long)(env);
+  ret->action = NULL;
+  return ret;
+}
+
 static const HParserVtable ch_vt = {
   .parse = parse_ch,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_ch,
 };
 
 const HParser* h_ch(const uint8_t c) {

src/parsers/charset.c

@@ -1,25 +1,7 @@
 #include <string.h>
+#include "../internal.h"
 #include "parser_internal.h"
 
-typedef unsigned int *HCharset;
-
-static inline HCharset new_charset(HAllocator* mm__) {
-  HCharset cs = h_new(unsigned int, 256 / sizeof(unsigned int));
-  memset(cs, 0, 256);
-  return cs;
-}
-
-static inline int charset_isset(HCharset cs, uint8_t pos) {
-  return !!(cs[pos / sizeof(*cs)] & (1 << (pos % sizeof(*cs))));
-}
-
-static inline void charset_set(HCharset cs, uint8_t pos, int val) {
-  cs[pos / sizeof(*cs)] =
-    val
-    ? cs[pos / sizeof(*cs)] |  (1 << (pos % sizeof(*cs)))
-    : cs[pos / sizeof(*cs)] & ~(1 << (pos % sizeof(*cs)));
-}
-
 static HParseResult* parse_charset(void *env, HParseState *state) {
   uint8_t in = h_read_bits(&state->input_stream, 8, false);
   HCharset cs = (HCharset)env;
@@ -32,10 +14,19 @@ static HParseResult* parse_charset(void *env, HParseState *state) {
     return NULL;
 }
 
+static HCFChoice* desugar_charset(HAllocator *mm__, void *env) {
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHARSET;
+  ret->charset = (HCharset)env;
+  ret->action = NULL;
+  return ret;
+}
+
 static const HParserVtable charset_vt = {
   .parse = parse_charset,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_charset,
 };
 
 const HParser* h_ch_range(const uint8_t lower, const uint8_t upper) {

src/parsers/choice.c

@@ -39,10 +39,26 @@ static bool choice_isValidCF(void *env) {
   return true;
 }
 
+static HCFChoice* desugar_choice(HAllocator *mm__, void *env) {
+  HSequence *s = (HSequence*)env;
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 1+s->len);
+  for (size_t i=0; i<s->len; ++i) {
+    ret->seq[i] = h_new(HCFSequence, 1);
+    ret->seq[i]->items = h_new(HCFChoice*, 2);
+    ret->seq[i]->items[0] = s->p_array[i]->vtable->desugar(mm__, s->p_array[i]->env);
+    ret->seq[i]->items[1] = NULL;
+  }
+  ret->seq[s->len] = NULL;
+  return ret;
+}
+
 static const HParserVtable choice_vt = {
   .parse = parse_choice,
   .isValidRegular = choice_isValidRegular,
   .isValidCF = choice_isValidCF,
+  .desugar = desugar_choice,
 };
 
 const HParser* h_choice(const HParser* p, ...) {

src/parsers/end.c

@@ -10,10 +10,18 @@ static HParseResult* parse_end(void *env, HParseState *state) {
   }
 }
 
+static const HCFChoice* desugar_end(HAllocator *mm__, void *env) {
+  static HCFChoice ret = {
+    .type = HCF_END
+  };
+  return &ret;
+}
+
 static const HParserVtable end_vt = {
   .parse = parse_end,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_end,
 };
 
 const HParser* h_end_p() {

src/parsers/epsilon.c

@@ -8,10 +8,21 @@ static HParseResult* parse_epsilon(void* env, HParseState* state) {
   return res;
 }
 
+static HCFChoice* desugar_epsilon(HAllocator *mm__, void *env) {
+  static HCFSequence res_seq = {NULL};
+  static HCFChoice res_ch = {
+    .type = HCF_CHOICE,
+    .seq = &res_seq
+  };
+
+  return &res_ch;
+}
+
 static const HParserVtable epsilon_vt = {
   .parse = parse_epsilon,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_epsilon,
 };
 
 static const HParser epsilon_p = {

src/parsers/ignoreseq.c

@@ -26,6 +26,23 @@ static HParseResult* parse_ignoreseq(void* env, HParseState *state) {
   return res;
 }
 
+static HCFChoice* desugar_ignoreseq(HAllocator *mm__, void *env) {
+  HIgnoreSeq *seq = (HIgnoreSeq*)env;
+  HCFSequence *hseq = h_new(HCFSequence, 1);
+  hseq->items = h_new(HCFChoice*, 1+seq->len);
+  for (size_t i=0; i<seq->len; ++i) {
+    hseq->items[i] = seq->parsers[i]->vtable->desugar(mm__, seq->parsers[i]->env);
+  }
+  hseq->items[seq->len] = NULL;
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 2);
+  ret->seq[0] = hseq;
+  ret->seq[1] = NULL;
+  ret->action = NULL;
+  return ret;
+}
+
 static bool is_isValidRegular(void *env) {
   HIgnoreSeq *seq = (HIgnoreSeq*)env;
   for (size_t i=0; i<seq->len; ++i) {
@@ -48,6 +65,7 @@ static const HParserVtable ignoreseq_vt = {
   .parse = parse_ignoreseq,
   .isValidRegular = is_isValidRegular,
   .isValidCF = is_isValidCF,
+  .desugar = desugar_ignoreseq,
 };
 
 

src/parsers/int_range.c

@@ -28,10 +28,57 @@ static HParseResult* parse_int_range(void *env, HParseState *state) {
   }
 }
 
+HCFChoice* gen_int_range(HAllocator *mm__, uint64_t low, uint64_t high, uint8_t bytes) {
+  if (1 == bytes) {
+    HCFChoice *cs = h_new(HCFChoice, 1);
+    cs->type = HCF_CHARSET;
+    cs->charset = new_charset(mm__);
+    for (uint64_t i=low; i<=high; ++i) {
+      charset_set(cs->charset, i, 1);
+    }
+    cs->action = NULL;
+    return cs;
+  }
+  else if (1 < bytes) {
+    HCFChoice *root = h_new(HCFChoice, 1);
+    root->type = HCF_CHOICE;
+    root->seq = h_new(HCFSequence*, 4);
+    root->seq[0] = h_new(HCFSequence, 1);
+    root->seq[0]->items = h_new(HCFChoice*, 2);
+    root->seq[0]->items[0] = gen_int_range(mm__, low, high, FIXME);
+    root->seq[0]->items[1] = NULL;
+    root->seq[1] = h_new(HCFSequence, 1);
+    root->seq[1]->items = h_new(HCFChoice*, 2);
+    root->seq[1]->items[0] = h_new(HCFChoice, 1);
+    /* do something with root->seq[1]->items[0] */
+    root->seq[1]->items[1] = NULL;
+    root->seq[2] = h_new(HCFSequence, 1);
+    root->seq[2]->items = h_new(HCFChoice*, 2);
+    root->seq[2]->items[0] = gen_int_range(mm__, low, high, FIXME);
+    root->seq[2]->items[1] = NULL;
+    root->seq[3] = NULL;
+    root->action = NULL;
+    return root;
+  }
+  else { // idk why this would ever be <1, but whatever
+    return NULL; 
+  }
+}
+
+static HCFChoice* desugar_int_range(HAllocator *mm__, void *env) {
+  HRange *r = (HRange*)env;
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  uint8_t bytes = r->p->env->length / 8;
+  HCFSequence *seq = h_new(HCFSequence, 1);
+  
+}
+
 static const HParserVtable int_range_vt = {
   .parse = parse_int_range,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_int_range,
 };
 
 const HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper) {

src/parsers/parser_internal.h

@@ -23,5 +23,7 @@ static inline size_t token_length(HParseResult *pr) {
   }
 }
 
+static inline bool h_true(void *env) { return true; }
+static inline bool h_false(void *env) { return false; }
 
 #endif // HAMMER_PARSE_INTERNAL__H

src/parsers/token.c

@@ -5,8 +5,6 @@ typedef struct {
   uint8_t len;
 } HToken;
 
-
-
 static HParseResult* parse_token(void *env, HParseState *state) {
   HToken *t = (HToken*)env;
   for (int i=0; i<t->len; ++i) {
@@ -20,10 +18,30 @@ static HParseResult* parse_token(void *env, HParseState *state) {
   return make_result(state, tok);
 }
 
+static HCFChoice* desugar_token(HAllocator *mm__, void *env) {
+  HToken *tok = (HToken*)env;
+  HCFSequence *seq = h_new(HCFSequence, 1);
+  seq->items = h_new(HCFChoice*, 1+tok->len);
+  for (size_t i=0; i<tok->len; ++i) {
+    seq->items[i] = h_new(HCFChoice, 1);
+    seq->items[i]->type = HCF_CHAR;
+    seq->items[i]->chr = tok->str[i];
+  }
+  seq->items[tok->len] = NULL;
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 2);
+  ret->seq[0] = seq;
+  ret->seq[1] = NULL;
+  ret->action = NULL;
+  return ret;
+}
+
 const HParserVtable token_vt = {
   .parse = parse_token,
   .isValidRegular = h_true,
   .isValidCF = h_true,
+  .desugar = desugar_token,
 };
 
 const HParser* h_token(const uint8_t *str, const size_t len) {

src/parsers/unimplemented.c

@@ -16,6 +16,7 @@ static const HParserVtable unimplemented_vt = {
   .parse = parse_unimplemented,
   .isValidRegular = h_false,
   .isValidCF = h_false,
+  .desugar = NULL,
 };
 
 static HParser unimplemented = {

src/parsers/whitespace.c

@@ -14,6 +14,31 @@ static HParseResult* parse_whitespace(void* env, HParseState *state) {
   return h_do_parse((HParser*)env, state);
 }
 
+static HCFChoice* desugar_whitespace(HAllocator *mm__, void *env) {
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->type = HCF_CHOICE;
+  ret->seq = h_new(HCFSequence*, 3);
+  HCFSequence *nonempty = h_new(HCFSequence, 1);
+  nonempty->items = h_new(HCFChoice*, 3);
+  nonempty->items[0] = h_new(HCFChoice, 1);
+  nonempty->items[0]->type = HCF_CHARSET;
+  nonempty->items[0]->charset = new_charset(mm__);
+  charset_set(nonempty->items[0]->charset, '\t', 1);
+  charset_set(nonempty->items[0]->charset, ' ', 1);
+  charset_set(nonempty->items[0]->charset, '\n', 1);
+  charset_set(nonempty->items[0]->charset, '\r', 1);
+  nonempty->items[1] = ret; // yay circular pointer!
+  nonempty->items[2] = NULL;
+  ret->seq[0] = nonempty;
+  HCFSequence *empty = h_new(HCFSequence, 1);
+  empty->items = h_new(HCFChoice*, 1);
+  empty->items[0] = NULL;
+  ret->seq[1] = empty;
+  ret->seq[2] = NULL;
+  ret->action = NULL;
+  return ret;
+}
+
 static bool ws_isValidRegular(void *env) {
   HParser *p = (HParser*)env;
   return p->vtable->isValidRegular(p->env);
@@ -28,6 +53,7 @@ static const HParserVtable whitespace_vt = {
   .parse = parse_whitespace,
   .isValidRegular = ws_isValidRegular,
   .isValidCF = ws_isValidCF,
+  .desugar = desugar_whitespace,
 };
 
 const HParser* h_whitespace(const HParser* p) {