From e0207b8d5fab6ec3cbb73b92eb14221db66f0a06 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 25 May 2013 14:49:59 +0200 Subject: [PATCH 01/39] properly initialize NULL fields in h_cfgrammar_new and h_stringmap_new --- src/cfgrammar.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cfgrammar.c b/src/cfgrammar.c index a691230..3d5e558 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -18,6 +18,7 @@ HCFGrammar *h_cfgrammar_new(HAllocator *mm__) g->mm__ = mm__; g->arena = h_new_arena(mm__, 0); // default blocksize g->nts = h_hashset_new(g->arena, h_eq_ptr, h_hash_ptr); + g->start = NULL; g->geneps = NULL; g->first = NULL; g->follow = NULL; @@ -222,6 +223,8 @@ static void collect_geneps(HCFGrammar *g) HCFStringMap *h_stringmap_new(HArena *a) { HCFStringMap *m = h_arena_malloc(a, sizeof(HCFStringMap)); + m->epsilon_branch = NULL; + m->end_branch = NULL; m->char_branches = h_hashtable_new(a, h_eq_ptr, h_hash_ptr); m->arena = a; return m; From dba9d41edaf0b0a99f530007e2cd9532b09648f7 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 4 Jun 2013 21:47:09 +0200 Subject: [PATCH 02/39] rename HCFStringMap to HStringMap --- src/backends/llk.c | 18 +++++----- src/cfgrammar.c | 88 +++++++++++++++++++++++----------------------- src/cfgrammar.h | 42 +++++++++++----------- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/src/backends/llk.c b/src/backends/llk.c index aeafd6a..79ab8f4 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -8,7 +8,7 @@ static const size_t DEFAULT_KMAX = 1; /* Generating the LL(k) parse table */ -/* Maps each nonterminal (HCFChoice) of the grammar to a HCFStringMap that +/* Maps each nonterminal (HCFChoice) of the grammar to a HStringMap that * maps lookahead strings to productions (HCFSequence). */ typedef struct HLLkTable_ { @@ -23,13 +23,13 @@ typedef struct HLLkTable_ { const HCFSequence *h_llk_lookup(const HLLkTable *table, const HCFChoice *x, HInputStream lookahead) { - const HCFStringMap *row = h_hashtable_get(table->rows, x); + const HStringMap *row = h_hashtable_get(table->rows, x); assert(row != NULL); // the table should have one row for each nonterminal assert(!row->epsilon_branch); // would match without looking at the input // XXX cases where this could be useful? - const HCFStringMap *m = row; + const HStringMap *m = row; while(m) { if(m->epsilon_branch) { // input matched // assert: another lookahead would not bring a more specific match. @@ -103,7 +103,7 @@ static void *combine_entries(HHashSet *workset, void *dst, const void *src) // add the mappings of src to dst, marking conflicts and adding the conflicting // values to workset. // note: reuses parts of src to build dst! -static void stringmap_merge(HHashSet *workset, HCFStringMap *dst, HCFStringMap *src) +static void stringmap_merge(HHashSet *workset, HStringMap *dst, HStringMap *src) { if(src->epsilon_branch) { if(dst->epsilon_branch) @@ -135,10 +135,10 @@ static void stringmap_merge(HHashSet *workset, HCFStringMap *dst, HCFStringMap * continue; HCharKey c = (HCharKey)hte->key; - HCFStringMap *src_ = hte->value; + HStringMap *src_ = hte->value; if(src_) { - HCFStringMap *dst_ = h_hashtable_get(dst->char_branches, (void *)c); + HStringMap *dst_ = h_hashtable_get(dst->char_branches, (void *)c); if(dst_) stringmap_merge(workset, dst_, src_); else @@ -149,7 +149,7 @@ static void stringmap_merge(HHashSet *workset, HCFStringMap *dst, HCFStringMap * } /* Generate entries for the productions of A in the given table row. */ -static int fill_table_row(size_t kmax, HCFGrammar *g, HCFStringMap *row, +static int fill_table_row(size_t kmax, HCFGrammar *g, HStringMap *row, const HCFChoice *A) { HHashSet *workset; @@ -177,7 +177,7 @@ static int fill_table_row(size_t kmax, HCFGrammar *g, HCFStringMap *row, assert(rhs != CONFLICT); // just to be sure there's no mixup // calculate predict set; let values map to rhs - HCFStringMap *pred = h_predict(k, g, A, rhs); + HStringMap *pred = h_predict(k, g, A, rhs); h_stringmap_replace(pred, NULL, rhs); // merge predict set into the row @@ -220,7 +220,7 @@ static int fill_table(size_t kmax, HCFGrammar *g, HLLkTable *table) assert(a->type == HCF_CHOICE); // create table row for this nonterminal - HCFStringMap *row = h_stringmap_new(table->arena); + HStringMap *row = h_stringmap_new(table->arena); h_hashtable_put(table->rows, a, row); if(fill_table_row(kmax, g, row, a) < 0) { diff --git a/src/cfgrammar.c b/src/cfgrammar.c index a691230..32cc9d6 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -23,7 +23,7 @@ HCFGrammar *h_cfgrammar_new(HAllocator *mm__) g->follow = NULL; g->kmax = 0; // will be increased as needed by ensure_k - HCFStringMap *eps = h_stringmap_new(g->arena); + HStringMap *eps = h_stringmap_new(g->arena); h_stringmap_put_epsilon(eps, INSET); g->singleton_epsilon = eps; @@ -219,32 +219,32 @@ static void collect_geneps(HCFGrammar *g) } -HCFStringMap *h_stringmap_new(HArena *a) +HStringMap *h_stringmap_new(HArena *a) { - HCFStringMap *m = h_arena_malloc(a, sizeof(HCFStringMap)); + HStringMap *m = h_arena_malloc(a, sizeof(HStringMap)); m->char_branches = h_hashtable_new(a, h_eq_ptr, h_hash_ptr); m->arena = a; return m; } -void h_stringmap_put_end(HCFStringMap *m, void *v) +void h_stringmap_put_end(HStringMap *m, void *v) { m->end_branch = v; } -void h_stringmap_put_epsilon(HCFStringMap *m, void *v) +void h_stringmap_put_epsilon(HStringMap *m, void *v) { m->epsilon_branch = v; } -void h_stringmap_put_after(HCFStringMap *m, uint8_t c, HCFStringMap *ends) +void h_stringmap_put_after(HStringMap *m, uint8_t c, HStringMap *ends) { h_hashtable_put(m->char_branches, (void *)char_key(c), ends); } -void h_stringmap_put_char(HCFStringMap *m, uint8_t c, void *v) +void h_stringmap_put_char(HStringMap *m, uint8_t c, void *v) { - HCFStringMap *node = h_stringmap_new(m->arena); + HStringMap *node = h_stringmap_new(m->arena); h_stringmap_put_epsilon(node, v); h_stringmap_put_after(m, c, node); } @@ -252,8 +252,8 @@ void h_stringmap_put_char(HCFStringMap *m, uint8_t c, void *v) // helper for h_stringmap_update static void *combine_stringmap(void *v1, const void *v2) { - HCFStringMap *m1 = v1; - const HCFStringMap *m2 = v2; + HStringMap *m1 = v1; + const HStringMap *m2 = v2; if(!m1) m1 = h_stringmap_new(m2->arena); h_stringmap_update(m1, m2); @@ -262,7 +262,7 @@ static void *combine_stringmap(void *v1, const void *v2) } /* Note: Does *not* reuse submaps from n in building m. */ -void h_stringmap_update(HCFStringMap *m, const HCFStringMap *n) +void h_stringmap_update(HStringMap *m, const HStringMap *n) { if(n->epsilon_branch) m->epsilon_branch = n->epsilon_branch; @@ -277,7 +277,7 @@ void h_stringmap_update(HCFStringMap *m, const HCFStringMap *n) * If old is NULL, replace all values in m with new. * If new is NULL, remove the respective values. */ -void h_stringmap_replace(HCFStringMap *m, void *old, void *new) +void h_stringmap_replace(HStringMap *m, void *old, void *new) { if(!old) { if(m->epsilon_branch) m->epsilon_branch = new; @@ -294,14 +294,14 @@ void h_stringmap_replace(HCFStringMap *m, void *old, void *new) if(hte->key == NULL) continue; - HCFStringMap *m_ = hte->value; + HStringMap *m_ = hte->value; if(m_) h_stringmap_replace(m_, old, new); } } } -void *h_stringmap_get(const HCFStringMap *m, const uint8_t *str, size_t n, bool end) +void *h_stringmap_get(const HStringMap *m, const uint8_t *str, size_t n, bool end) { for(size_t i=0; iend_branch) @@ -313,20 +313,20 @@ void *h_stringmap_get(const HCFStringMap *m, const uint8_t *str, size_t n, bool return m->epsilon_branch; } -bool h_stringmap_present(const HCFStringMap *m, const uint8_t *str, size_t n, bool end) +bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end) { return (h_stringmap_get(m, str, n, end) != NULL); } -bool h_stringmap_present_epsilon(const HCFStringMap *m) +bool h_stringmap_present_epsilon(const HStringMap *m) { return (m->epsilon_branch != NULL); } -const HCFStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x) +const HStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x) { - HCFStringMap *ret; + HStringMap *ret; HCFSequence **p; uint8_t c; @@ -372,18 +372,18 @@ const HCFStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x) } // helpers for h_first_seq, definitions below -static bool is_singleton_epsilon(const HCFStringMap *m); -static bool any_string_shorter(size_t k, const HCFStringMap *m); +static bool is_singleton_epsilon(const HStringMap *m); +static bool any_string_shorter(size_t k, const HStringMap *m); // pointer to functions like h_first_seq -typedef const HCFStringMap *(*StringSetFun)(size_t, HCFGrammar *, HCFChoice **); +typedef const HStringMap *(*StringSetFun)(size_t, HCFGrammar *, HCFChoice **); // helper for h_first_seq and h_follow -static void stringset_extend(HCFGrammar *g, HCFStringMap *ret, - size_t k, const HCFStringMap *as, +static void stringset_extend(HCFGrammar *g, HStringMap *ret, + size_t k, const HStringMap *as, StringSetFun f, HCFChoice **tail); -const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s) +const HStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s) { // shortcut: the first set of the empty sequence, for any k, is {""} if(*s == NULL) @@ -394,7 +394,7 @@ const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s) HCFChoice *x = s[0]; HCFChoice **tail = s+1; - const HCFStringMap *first_x = h_first(k, g, x); + const HStringMap *first_x = h_first(k, g, x); // shortcut: if first_k(X) = {""}, just return first_k(tail) if(is_singleton_epsilon(first_x)) @@ -405,7 +405,7 @@ const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s) return first_x; // create a new result set and build up the set described above - HCFStringMap *ret = h_stringmap_new(g->arena); + HStringMap *ret = h_stringmap_new(g->arena); // extend the elements of first_k(X) up to length k from tail stringset_extend(g, ret, k, first_x, h_first_seq, tail); @@ -413,14 +413,14 @@ const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s) return ret; } -static bool is_singleton_epsilon(const HCFStringMap *m) +static bool is_singleton_epsilon(const HStringMap *m) { return ( m->epsilon_branch && !m->end_branch && h_hashtable_empty(m->char_branches) ); } -static bool any_string_shorter(size_t k, const HCFStringMap *m) +static bool any_string_shorter(size_t k, const HStringMap *m) { if(k==0) return false; @@ -434,7 +434,7 @@ static bool any_string_shorter(size_t k, const HCFStringMap *m) for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { if(hte->key == NULL) continue; - HCFStringMap *m_ = hte->value; + HStringMap *m_ = hte->value; // check subtree for strings shorter than k-1 if(any_string_shorter(k-1, m_)) @@ -446,7 +446,7 @@ static bool any_string_shorter(size_t k, const HCFStringMap *m) } // helper for h_predict -static void remove_all_shorter(size_t k, HCFStringMap *m) +static void remove_all_shorter(size_t k, HStringMap *m) { if(k==0) return; m->epsilon_branch = NULL; @@ -465,12 +465,12 @@ static void remove_all_shorter(size_t k, HCFStringMap *m) // h_follow adapted to the signature of StringSetFun static inline -const HCFStringMap *h_follow_(size_t k, HCFGrammar *g, HCFChoice **s) +const HStringMap *h_follow_(size_t k, HCFGrammar *g, HCFChoice **s) { return h_follow(k, g, *s); } -const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x) +const HStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x) { // consider all occurances of X in g // the follow set of X is the union of: @@ -481,7 +481,7 @@ const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x) // first_k(tail follow_k(A)) = // { a b | a <- first_k(tail), b <- follow_l(A), l=k-|a| } - HCFStringMap *ret; + HStringMap *ret; // shortcut: follow_0(X) is always {""} if(k==0) @@ -519,7 +519,7 @@ const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x) if(*s == x) { // occurance found HCFChoice **tail = s+1; - const HCFStringMap *first_tail = h_first_seq(k, g, tail); + const HStringMap *first_tail = h_first_seq(k, g, tail); // extend the elems of first_k(tail) up to length k from follow(A) stringset_extend(g, ret, k, first_tail, h_follow_, &a); @@ -532,15 +532,15 @@ const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x) return ret; } -HCFStringMap *h_predict(size_t k, HCFGrammar *g, +HStringMap *h_predict(size_t k, HCFGrammar *g, const HCFChoice *A, const HCFSequence *rhs) { - HCFStringMap *ret = h_stringmap_new(g->arena); + HStringMap *ret = h_stringmap_new(g->arena); // predict_k(A -> rhs) = // { ab | a <- first_k(rhs), b <- follow_k(A), |ab|=k } - const HCFStringMap *first_rhs = h_first_seq(k, g, rhs->items); + const HStringMap *first_rhs = h_first_seq(k, g, rhs->items); // casting the const off of A below. note: stringset_extend does // not touch this argument, only passes it through to h_follow @@ -554,8 +554,8 @@ HCFStringMap *h_predict(size_t k, HCFGrammar *g, } // add the set { a b | a <- as, b <- f_l(S), l=k-|a| } to ret -static void stringset_extend(HCFGrammar *g, HCFStringMap *ret, - size_t k, const HCFStringMap *as, +static void stringset_extend(HCFGrammar *g, HStringMap *ret, + size_t k, const HStringMap *as, StringSetFun f, HCFChoice **tail) { if(as->epsilon_branch) { @@ -578,12 +578,12 @@ static void stringset_extend(HCFGrammar *g, HCFStringMap *ret, uint8_t c = key_char((HCharKey)hte->key); // follow the branch to find the set { a' | t a' <- as } - HCFStringMap *as_ = (HCFStringMap *)hte->value; + HStringMap *as_ = (HStringMap *)hte->value; // now the elements of ret that begin with t are given by // t { a b | a <- as_, b <- f_l(tail), l=k-|a|-1 } // so we can use recursion over k - HCFStringMap *ret_ = h_stringmap_new(g->arena); + HStringMap *ret_ = h_stringmap_new(g->arena); h_stringmap_put_after(ret, c, ret_); stringset_extend(g, ret_, k-1, as_, f, tail); @@ -800,7 +800,7 @@ void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, in static bool pprint_stringset_elems(FILE *file, bool first, char *prefix, size_t n, - const HCFStringMap *set) + const HStringMap *set) { assert(n < BUFSIZE-4); @@ -827,7 +827,7 @@ pprint_stringset_elems(FILE *file, bool first, char *prefix, size_t n, if(hte->key == NULL) continue; uint8_t c = key_char((HCharKey)hte->key); - HCFStringMap *ends = hte->value; + HStringMap *ends = hte->value; size_t n_ = n; switch(c) { @@ -852,7 +852,7 @@ pprint_stringset_elems(FILE *file, bool first, char *prefix, size_t n, return first; } -void h_pprint_stringset(FILE *file, const HCFStringMap *set, int indent) +void h_pprint_stringset(FILE *file, const HStringMap *set, int indent) { int j; for(j=0; jchar_branches, (void *)char_key(c)); } @@ -72,18 +72,18 @@ bool h_derives_epsilon(HCFGrammar *g, const HCFChoice *symbol); bool h_derives_epsilon_seq(HCFGrammar *g, HCFChoice **s); /* Compute first_k set of symbol x. Memoized. */ -const HCFStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x); +const HStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x); /* Compute first_k set of sentential form s. s NULL-terminated. */ -const HCFStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s); +const HStringMap *h_first_seq(size_t k, HCFGrammar *g, HCFChoice **s); /* Compute follow_k set of symbol x. Memoized. */ -const HCFStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x); +const HStringMap *h_follow(size_t k, HCFGrammar *g, const HCFChoice *x); /* Compute the predict_k set of production "A -> rhs". - * Always returns a newly-allocated HCFStringMap. + * Always returns a newly-allocated HStringMap. */ -HCFStringMap *h_predict(size_t k, HCFGrammar *g, +HStringMap *h_predict(size_t k, HCFGrammar *g, const HCFChoice *A, const HCFSequence *rhs); @@ -92,4 +92,4 @@ void h_pprint_grammar(FILE *file, const HCFGrammar *g, int indent); void h_pprint_sequence(FILE *f, const HCFGrammar *g, const HCFSequence *seq); void h_pprint_symbol(FILE *f, const HCFGrammar *g, const HCFChoice *x); void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, int indent); -void h_pprint_stringset(FILE *file, const HCFStringMap *set, int indent); +void h_pprint_stringset(FILE *file, const HStringMap *set, int indent); From 3be83c7fa2a9232fad3b1ea738a001df15a6d5b5 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 4 Jun 2013 21:59:39 +0200 Subject: [PATCH 03/39] set index and offset of result tokens in llk driver --- src/backends/llk.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/backends/llk.c b/src/backends/llk.c index 79ab8f4..4f73c46 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -339,10 +339,12 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* // the top of stack is such that there will be a result... HParsedToken *tok; // will hold result token + tok = h_arena_malloc(arena, sizeof(HParsedToken)); + tok->index = stream->index; + tok->bit_offset = stream->bit_offset; if(x == mark) { // hit stack frame boundary... // wrap the accumulated parse result, this sequence is finished - tok = h_arena_malloc(arena, sizeof(HParsedToken)); tok->token_type = TT_SEQUENCE; tok->seq = seq; @@ -361,13 +363,13 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* case HCF_END: if(!stream->overrun) goto no_parse; + h_arena_free(arena, tok); tok = NULL; break; case HCF_CHAR: if(input != x->chr) goto no_parse; - tok = h_arena_malloc(arena, sizeof(HParsedToken)); tok->token_type = TT_UINT; tok->uint = x->chr; break; @@ -377,7 +379,6 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* goto no_parse; if(!charset_isset(x->charset, input)) goto no_parse; - tok = h_arena_malloc(arena, sizeof(HParsedToken)); tok->token_type = TT_UINT; tok->uint = input; break; @@ -390,8 +391,6 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* // 'tok' has been parsed; process it - // XXX set tok->index and tok->bit_offset (don't take directly from stream, cuz peek!) - // perform token reshape if indicated if(x->reshape) tok = (HParsedToken *)x->reshape(make_result(arena, tok)); From 976205f9da21acab80c7bc8ce2e0163d97084f11 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 4 Jun 2013 22:14:06 +0200 Subject: [PATCH 04/39] hook up LALR backend stub --- src/Makefile | 1 + src/backends/lalr.c | 86 +++++++++++++++++++++++++++++++++++++++++++++ src/hammer.c | 1 + src/hammer.h | 8 ++--- src/internal.h | 1 + 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 src/backends/lalr.c diff --git a/src/Makefile b/src/Makefile index 7fac881..1a2bff3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -28,6 +28,7 @@ PARSERS := \ BACKENDS := \ packrat \ llk \ + lalr \ regex HAMMER_PARTS := \ diff --git a/src/backends/lalr.c b/src/backends/lalr.c new file mode 100644 index 0000000..22cd389 --- /dev/null +++ b/src/backends/lalr.c @@ -0,0 +1,86 @@ +#include +#include "../internal.h" +#include "../cfgrammar.h" +#include "../parsers/parser_internal.h" + + + +void h_lalr_free(HParser *parser) +{ + // XXX free data structures + parser->backend_data = NULL; + parser->backend = PB_PACKRAT; +} + + +/* LALR table generation */ + +int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) +{ + return -1; +} + + +/* LR driver */ + +HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) +{ + return NULL; +} + + + + +HParserBackendVTable h__lalr_backend_vtable = { + .compile = h_lalr_compile, + .parse = h_lr_parse, + .free = h_lalr_free +}; + + + + +// dummy! +int test_lalr(void) +{ + /* for k=2: + + S -> A | B + A -> X Y a + B -> Y b + X -> x | '' + Y -> y -- for k=3 use "yy" + */ + + // XXX make LALR example + HParser *X = h_optional(h_ch('x')); + HParser *Y = h_sequence(h_ch('y'), h_ch('y'), NULL); + HParser *A = h_sequence(X, Y, h_ch('a'), NULL); + HParser *B = h_sequence(Y, h_ch('b'), NULL); + HParser *p = h_choice(A, B, NULL); + + HCFGrammar *g = h_cfgrammar(&system_allocator, p); + + if(g == NULL) { + fprintf(stderr, "h_cfgrammar failed\n"); + return 1; + } + + h_pprint_grammar(stdout, g, 0); + // print states of the LR(0) automaton + // print LALR(1) table + + if(h_compile(p, PB_LALR, NULL)) { + fprintf(stderr, "does not compile\n"); + return 2; + } + + + HParseResult *res = h_parse(p, (uint8_t *)"xyya", 4); + if(res) + h_pprint(stdout, res->ast, 0, 2); + else + printf("no parse\n"); + + return 0; +} diff --git a/src/hammer.c b/src/hammer.c index 5f94142..7d5b4e9 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -30,6 +30,7 @@ static HParserBackendVTable *backends[PB_MAX + 1] = { &h__packrat_backend_vtable, &h__regex_backend_vtable, &h__llk_backend_vtable, + &h__lalr_backend_vtable, }; diff --git a/src/hammer.h b/src/hammer.h index 455684c..a5ebcff 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -34,11 +34,11 @@ typedef struct HParseState_ HParseState; typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = PB_MIN, // PB_MIN is always the default. - PB_REGULAR, // - PB_LLk, // - PB_LALR, // Not Implemented + PB_REGULAR, + PB_LLk, + PB_LALR, PB_GLR, // Not Implemented - PB_MAX = PB_LLk + PB_MAX = PB_LALR } HParserBackend; typedef enum HTokenType_ { diff --git a/src/internal.h b/src/internal.h index 926bf02..01861f5 100644 --- a/src/internal.h +++ b/src/internal.h @@ -219,6 +219,7 @@ struct HBitWriter_ { // Backends {{{ extern HParserBackendVTable h__packrat_backend_vtable; extern HParserBackendVTable h__llk_backend_vtable; +extern HParserBackendVTable h__lalr_backend_vtable; // }}} // TODO(thequux): Set symbol visibility for these functions so that they aren't exported. From 636f741d88ac179e74d172e81c1376507a382218 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 5 Jun 2013 15:12:48 +0200 Subject: [PATCH 05/39] add h_hashset_equal (set comparison) --- src/datastructures.c | 52 ++++++++++++++++++++++++++++++++++++++++++++ src/internal.h | 1 + 2 files changed, 53 insertions(+) diff --git a/src/datastructures.c b/src/datastructures.c index a12707e..bd9b4eb 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -232,6 +232,7 @@ int h_hashtable_present(const HHashTable* ht, const void* key) { } return false; } + void h_hashtable_del(HHashTable* ht, const void* key) { HHashValue hashval = ht->hashFunc(key); #ifdef CONSISTENCY_CHECK @@ -257,6 +258,7 @@ void h_hashtable_del(HHashTable* ht, const void* key) { } } } + void h_hashtable_free(HHashTable* ht) { for (size_t i = 0; i < ht->capacity; i++) { HHashTableEntry *hten, *hte = &ht->contents[i]; @@ -272,6 +274,56 @@ void h_hashtable_free(HHashTable* ht) { h_arena_free(ht->arena, ht->contents); } +// helper for hte_equal +static bool hte_same_length(HHashTableEntry *xs, HHashTableEntry *ys) { + for(; xs && ys; xs=xs->next, ys=ys->next) { + // skip NULL keys (= element not present) + if(xs->key == NULL) xs=xs->next; + if(ys->key == NULL) ys=ys->next; + } + return (xs == ys); // both NULL +} + +// helper for hte_equal: are all elements of xs present in ys? +static bool hte_subset(HEqualFunc eq, HHashTableEntry *xs, HHashTableEntry *ys) +{ + for(; xs; xs=xs->next) { + if(xs->key == NULL) continue; // element not present + + HHashTableEntry *hte; + for(hte=ys; hte; hte=hte->next) { + if(hte->key == xs->key) break; // assume an element is equal to itself + if(hte->hashval != xs->hashval) continue; // shortcut + if(eq(hte->key, xs->key)) break; + } + if(hte == NULL) return false; // element not found + } + return true; // all found +} + +// compare two lists of HHashTableEntries +static inline bool hte_equal(HEqualFunc eq, HHashTableEntry *xs, HHashTableEntry *ys) { + return (hte_same_length(xs, ys) && hte_subset(eq, xs, ys)); +} + +/* Set equality of HHashSets. + * Obviously, 'a' and 'b' must use the same equality function. + * Not strictly necessary, but we also assume the same hash function. + */ +bool h_hashset_equal(const HHashSet *a, const HHashSet *b) { + if(a->capacity == b->capacity) { + // iterate over the buckets in parallel + for(size_t i=0; i < a->capacity; i++) { + if(!hte_equal(a->equalFunc, &a->contents[i], &b->contents[i])) + return false; + } + } else { + assert_message(0, "h_hashset_equal called on sets of different capacity"); + // TODO implement general case + } + return true; +} + bool h_eq_ptr(const void *p, const void *q) { return (p==q); } diff --git a/src/internal.h b/src/internal.h index 01861f5..1183682 100644 --- a/src/internal.h +++ b/src/internal.h @@ -272,6 +272,7 @@ typedef HHashTable HHashSet; #define h_hashset_empty(ht) h_hashtable_empty(ht) #define h_hashset_del(ht,el) h_hashtable_del(ht,el) #define h_hashset_free(ht) h_hashtable_free(ht) +bool h_hashset_equal(const HHashSet *a, const HHashSet *b); bool h_eq_ptr(const void *p, const void *q); HHashValue h_hash_ptr(const void *p); From b959bcb5c79a2a82f2b950048bbbd9ab745c63e5 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 6 Jun 2013 13:01:39 +0200 Subject: [PATCH 06/39] fix an array index --- src/test_suite.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_suite.h b/src/test_suite.h index 168ab64..fc008e7 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -153,7 +153,7 @@ } while(0) #define g_check_stringmap_absent(table, key) do { \ - bool end = (key[strlen(key)-2] == '$'); \ + bool end = (key[strlen(key)-1] == '$'); \ if(h_stringmap_present(table, (uint8_t *)key, strlen(key), end)) { \ g_test_message("Check failed: \"%s\" shouldn't have been in map, but was", key); \ g_test_fail(); \ From e7a388d1c7adf034feddc5ba10d6d8b0c70e5f60 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 6 Jun 2013 13:01:54 +0200 Subject: [PATCH 07/39] move djbhash into general availability as h_djbhash --- src/backends/packrat.c | 10 +--------- src/datastructures.c | 11 ++++++++++- src/internal.h | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index c5c9565..8aa1f8e 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -3,14 +3,6 @@ #include "../internal.h" #include "../parsers/parser_internal.h" -static uint32_t djbhash(const uint8_t *buf, size_t len) { - uint32_t hash = 5381; - while (len--) { - hash = hash * 33 + *buf++; - } - return hash; -} - // short-hand for constructing HCachedResult's static HCachedResult *cached_result(const HParseState *state, HParseResult *result) { HCachedResult *ret = a_new(HCachedResult, 1); @@ -214,7 +206,7 @@ void h_packrat_free(HParser *parser) { } static uint32_t cache_key_hash(const void* key) { - return djbhash(key, sizeof(HParserCacheKey)); + return h_djbhash(key, sizeof(HParserCacheKey)); } static bool cache_key_equal(const void* key1, const void* key2) { return memcmp(key1, key2, sizeof(HParserCacheKey)) == 0; diff --git a/src/datastructures.c b/src/datastructures.c index bd9b4eb..730c6b9 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -329,6 +329,15 @@ bool h_eq_ptr(const void *p, const void *q) { } HHashValue h_hash_ptr(const void *p) { - // XXX just djbhash it + // XXX just djbhash it? it does make the benchmark ~7% slower. + //return h_djbhash((const uint8_t *)&p, sizeof(void *)); return (uintptr_t)p >> 4; } + +uint32_t h_djbhash(const uint8_t *buf, size_t len) { + uint32_t hash = 5381; + while (len--) { + hash = hash * 33 + *buf++; + } + return hash; +} diff --git a/src/internal.h b/src/internal.h index 1183682..2f3018d 100644 --- a/src/internal.h +++ b/src/internal.h @@ -276,6 +276,7 @@ bool h_hashset_equal(const HHashSet *a, const HHashSet *b); bool h_eq_ptr(const void *p, const void *q); HHashValue h_hash_ptr(const void *p); +uint32_t h_djbhash(const uint8_t *buf, size_t len); typedef struct HCFSequence_ HCFSequence; From 167e187151cae6c819914e3d6514f2c5288d2579 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 6 Jun 2013 13:05:48 +0200 Subject: [PATCH 08/39] scratch commit of LALR preparations (breaks compile) --- src/backends/lalr.c | 126 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 22cd389..b7b127d 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -4,6 +4,125 @@ #include "../parsers/parser_internal.h" +// PLAN: +// data structures: +// - LR table is an array of hashtables that map grammar symbols (HCFChoice) +// to LRActions. + +// build LR(0) DFA +// extend with lookahead information by either: +// - reworking algorithm to propagate lookahead ("simple LALR generation") +// - follow sets of enhanced grammar ("conversion to SLR") + + +/* Constructing the characteristic automaton (handle recognizer) */ + +// - DFA is a hashset containing states (mapped to numbers) +// - states are hashsets containing LRItems +// - LRItems contain an optional lookahead set (HStringMap) +// - states (hashsets) get hash and comparison functions that ignore the lookahead + +typedef struct HLRDFA_ { + HHashSet *states; + HSlist *transitions; +} HLRDFA; + +typedef struct HLRTransition_ { + HLRState *from; + HCFChoice *symbol; + HLRState *to; +} HLRTransition; + +typedef struct HLRItem_ { + HCFChoice *lhs; + HCFChoice **rhs; + size_t len; // number of elements in rhs + size_t mark; + HStringMap *lookahead; // optional +} HLRItem; + +// compare LALR items - ignores lookahead +static bool eq_lalr_item(const void *p, const void *q) +{ + const HLRItem *a=p, *b=q; + + if(a->lhs != b->lhs) return false; + if(a->mark != b->mark) return false; + if(a->len != b->len) return false; + + for(size_t i=0; ilen; i++) + if(a->rhs[i] != b->rhs[i]) return false; + + return true; +} + +// compare LALR item sets (DFA states) +static inline bool eq_lalr_itemset(const void *p, const void *q) +{ + return h_hashset_equal(p, q); +} + +// hash LALR items +static inline HHashValue hash_lalr_item(const HLRItem *x) +{ + return (h_hash_ptr(x->lhs) + + h_djbhash((uint8_t *)x->rhs, x->len*sizeof(HCFChoice *)) + + x->mark); // XXX is it okay to just add mark? +} + +// hash LALR item sets (DFA states) - hash the elements and sum +static HHashValue hash_lalr_itemset(const void *p) +{ + HHashValue hash = 0; + + const HHashTable *ht = p; + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + hash += hash_lalr_item(hte->key); + } + } + + return hash; +} + +static HHashSet *closure(const HHashSet *items); + +HLRDFA *h_lalr_dfa(HCFGrammar *g) +{ + HHashSet *states = h_hashset_new(g->arena, eq_lalr_itemset, hash_lalr_itemset); + + // make initial state (kernel) + + // while work to do (on some state) + // compute closure + // determine edge symbols + // for each edge symbol: + // advance respective items -> destination state (kernel) + // if destination is a new state: + // add it to state set + // add transition to it + // add it to the work list +} + + + +/* LALR table generation */ + +int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) +{ + // generate grammar + // construct dfa / determine lookahead + // extract table + // create an array of hashtables, one per state + // for each transition a--S-->b: + // add "shift, goto b" to table entry (a,S) + // for each state: + // add reduce entries for its accepting items + return -1; +} void h_lalr_free(HParser *parser) { @@ -13,13 +132,6 @@ void h_lalr_free(HParser *parser) } -/* LALR table generation */ - -int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) -{ - return -1; -} - /* LR driver */ From 3ad4c5107099ec6c49ee861d16865fc2ce21de3e Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 13:46:16 +0200 Subject: [PATCH 09/39] fix hte_same_length() --- src/datastructures.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/datastructures.c b/src/datastructures.c index 730c6b9..55b8345 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -276,10 +276,12 @@ void h_hashtable_free(HHashTable* ht) { // helper for hte_equal static bool hte_same_length(HHashTableEntry *xs, HHashTableEntry *ys) { - for(; xs && ys; xs=xs->next, ys=ys->next) { + while(xs && ys) { + xs=xs->next; + ys=ys->next; // skip NULL keys (= element not present) - if(xs->key == NULL) xs=xs->next; - if(ys->key == NULL) ys=ys->next; + while(xs && xs->key == NULL) xs=xs->next; + while(ys && ys->key == NULL) ys=ys->next; } return (xs == ys); // both NULL } From 43fc07e67bf5fe485b726a3768c856930595f06b Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 13:47:00 +0200 Subject: [PATCH 10/39] add construction of LR(0) automaton (untested) --- src/backends/lalr.c | 203 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 190 insertions(+), 13 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index b7b127d..509f76d 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -17,30 +17,48 @@ /* Constructing the characteristic automaton (handle recognizer) */ -// - DFA is a hashset containing states (mapped to numbers) // - states are hashsets containing LRItems // - LRItems contain an optional lookahead set (HStringMap) // - states (hashsets) get hash and comparison functions that ignore the lookahead +typedef HHashSet HLRState; + typedef struct HLRDFA_ { - HHashSet *states; + size_t nstates; + const HLRState **states; // array of size nstates HSlist *transitions; } HLRDFA; typedef struct HLRTransition_ { - HLRState *from; - HCFChoice *symbol; - HLRState *to; + size_t from, to; // indices into 'states' array + const HCFChoice *symbol; } HLRTransition; typedef struct HLRItem_ { HCFChoice *lhs; - HCFChoice **rhs; + HCFChoice **rhs; // NULL-terminated size_t len; // number of elements in rhs size_t mark; HStringMap *lookahead; // optional } HLRItem; +HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) +{ + HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); + + size_t len = 0; + for(HCFChoice **p=rhs; *p; p++) len++; + assert(mark <= len); + + ret->lhs = lhs; + ret->rhs = rhs; + ret->len = len; + ret->mark = mark; + ret->lookahead = NULL; + + return ret; +} + // compare LALR items - ignores lookahead static bool eq_lalr_item(const void *p, const void *q) { @@ -63,8 +81,9 @@ static inline bool eq_lalr_itemset(const void *p, const void *q) } // hash LALR items -static inline HHashValue hash_lalr_item(const HLRItem *x) +static inline HHashValue hash_lalr_item(const void *p) { + const HLRItem *x = p; return (h_hash_ptr(x->lhs) + h_djbhash((uint8_t *)x->rhs, x->len*sizeof(HCFChoice *)) + x->mark); // XXX is it okay to just add mark? @@ -88,13 +107,90 @@ static HHashValue hash_lalr_itemset(const void *p) return hash; } -static HHashSet *closure(const HHashSet *items); +static inline HLRState *h_lrstate_new(HArena *arena) +{ + return h_hashset_new(arena, eq_lalr_item, hash_lalr_item); +} + +static HLRItem *advance_mark(HArena *arena, const HLRItem *item) +{ + assert(item->rhs[item->mark] != NULL); + HLRItem *ret = h_arena_malloc(arena, sizeof(HLRItem)); + *ret = *item; + ret->mark++; + return ret; +} + +static HHashSet *closure(HCFGrammar *g, const HHashSet *items) +{ + HArena *arena = g->arena; + HHashSet *ret = h_lrstate_new(arena); + HSlist *work = h_slist_new(arena); + + // iterate over items - initialize work list with them + const HHashTable *ht = items; + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HLRItem *item = hte->key; + h_hashset_put(ret, item); + h_slist_push(work, (void *)item); + } + } + + while(!h_slist_empty(work)) { + const HLRItem *item = h_slist_pop(work); + HCFChoice *sym = item->rhs[item->mark]; // symbol after mark + + // if there is a non-terminal after the mark, follow it + // XXX: do we have to count HCF_CHARSET as nonterminal? + if(sym != NULL && sym->type == HCF_CHOICE) { + // add items corresponding to the productions of sym + for(HCFSequence **p=sym->seq; *p; p++) { + HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); + if(!h_hashset_present(ret, it)) { + h_hashset_put(ret, it); + h_slist_push(work, it); + } + } + + // if sym derives epsilon, also advance over it + if(h_derives_epsilon(g, sym)) { + HLRItem *it = advance_mark(arena, item); + h_hashset_put(ret, it); + h_slist_push(work, it); + } + } + } + + return ret; +} HLRDFA *h_lalr_dfa(HCFGrammar *g) { - HHashSet *states = h_hashset_new(g->arena, eq_lalr_itemset, hash_lalr_itemset); + HArena *arena = g->arena; + + HHashSet *states = h_hashset_new(arena, eq_lalr_itemset, hash_lalr_itemset); + // maps itemsets to assigned array indices + HSlist *transitions = h_slist_new(arena); + + // list of states that need to be processed + // to save lookups, we push two elements per state, the itemset and its + // assigned index. + HSlist *work = h_slist_new(arena); + + // XXX augment grammar?! // make initial state (kernel) + HLRState *start = h_lrstate_new(arena); + assert(g->start->type == HCF_CHOICE); + for(HCFSequence **p=g->start->seq; *p; p++) + h_hashset_put(start, h_lritem_new(arena, g->start, (*p)->items, 0)); + h_hashtable_put(states, start, 0); + h_slist_push(work, start); + h_slist_push(work, 0); // while work to do (on some state) // compute closure @@ -105,6 +201,85 @@ HLRDFA *h_lalr_dfa(HCFGrammar *g) // add it to state set // add transition to it // add it to the work list + + while(!h_slist_empty(work)) { + size_t state_idx = (uintptr_t)h_slist_pop(work); + HLRState *state = h_slist_pop(work); + + // maps edge symbols to neighbor states (item sets) of s + HHashTable *neighbors = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); + + // iterate over closure and generate neighboring sets + const HHashTable *ht = closure(g, state); + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HLRItem *item = hte->key; + HCFChoice *sym = item->rhs[item->mark]; // symbol after mark + + if(sym != NULL) { // mark was not at the end + // find or create prospective neighbor set + HLRState *neighbor = h_hashtable_get(neighbors, sym); + if(neighbor == NULL) { + neighbor = h_lrstate_new(arena); + h_hashtable_put(neighbors, sym, neighbor); + } + + // ...and add the advanced item to it + h_hashset_put(neighbor, advance_mark(arena, item)); + } + } + } + + // merge neighbor sets into the set of existing states + ht = neighbors; + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HCFChoice *symbol = hte->key; + HLRState *neighbor = hte->value; + + // look up existing state, allocate new if not found + size_t neighbor_idx; + if(!h_hashset_present(states, neighbor)) { + neighbor_idx = states->used; + h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); + h_slist_push(work, neighbor); + h_slist_push(work, (void *)(uintptr_t)neighbor_idx); + } + + // add transition "state --symbol--> neighbor" + HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); + t->from = state_idx; + t->to = neighbor_idx; + t->symbol = symbol; + h_slist_push(transitions, t); + } + } + } // end while(work) + + // fill DFA struct + HLRDFA *dfa = h_arena_malloc(arena, sizeof(HLRDFA)); + dfa->nstates = states->used; + dfa->states = h_arena_malloc(arena, dfa->nstates*sizeof(HLRState *)); + for(size_t i=0; i < states->capacity; i++) { + for(HHashTableEntry *hte = &states->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HLRState *state = hte->key; + size_t idx = (uintptr_t)hte->value; + + dfa->states[idx] = state; + } + } + dfa->transitions = transitions; + + return dfa; } @@ -172,20 +347,22 @@ int test_lalr(void) HParser *p = h_choice(A, B, NULL); HCFGrammar *g = h_cfgrammar(&system_allocator, p); - if(g == NULL) { fprintf(stderr, "h_cfgrammar failed\n"); return 1; } - h_pprint_grammar(stdout, g, 0); - // print states of the LR(0) automaton - // print LALR(1) table + + HLRDFA *dfa = h_lalr_dfa(g); + if(dfa) { + // print states of the LR(0) automaton + } if(h_compile(p, PB_LALR, NULL)) { fprintf(stderr, "does not compile\n"); return 2; } + // print LALR(1) table HParseResult *res = h_parse(p, (uint8_t *)"xyya", 4); From 373a7aef9447044ba2c7339774e5cfaa21eb3cff Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 16:29:14 +0200 Subject: [PATCH 11/39] make h_pprint_char from cfgrammar public so LR pretty printers can use it --- src/cfgrammar.c | 8 ++++---- src/cfgrammar.h | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/cfgrammar.c b/src/cfgrammar.c index 0c51589..bc7b358 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -595,7 +595,7 @@ static void stringset_extend(HCFGrammar *g, HStringMap *ret, } -static void pprint_char(FILE *f, char c) +void h_pprint_char(FILE *f, char c) { switch(c) { case '"': fputs("\\\"", f); break; @@ -619,7 +619,7 @@ static void pprint_charset_char(FILE *f, char c) case '"': fputc(c, f); break; case '-': fputs("\\-", f); break; case ']': fputs("\\-", f); break; - default: pprint_char(f, c); + default: h_pprint_char(f, c); } } @@ -667,7 +667,7 @@ static HCFChoice **pprint_string(FILE *f, HCFChoice **x) for(; *x; x++) { if((*x)->type != HCF_CHAR) break; - pprint_char(f, (*x)->chr); + h_pprint_char(f, (*x)->chr); } fputc('"', f); return x; @@ -678,7 +678,7 @@ void h_pprint_symbol(FILE *f, const HCFGrammar *g, const HCFChoice *x) switch(x->type) { case HCF_CHAR: fputc('"', f); - pprint_char(f, x->chr); + h_pprint_char(f, x->chr); fputc('"', f); break; case HCF_END: diff --git a/src/cfgrammar.h b/src/cfgrammar.h index 6aba29c..a5de4d6 100644 --- a/src/cfgrammar.h +++ b/src/cfgrammar.h @@ -93,3 +93,4 @@ void h_pprint_sequence(FILE *f, const HCFGrammar *g, const HCFSequence *seq); void h_pprint_symbol(FILE *f, const HCFGrammar *g, const HCFChoice *x); void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, int indent); void h_pprint_stringset(FILE *file, const HStringMap *set, int indent); +void h_pprint_char(FILE *file, char c); From 67e5b2fee0e4ceff7fab9375065e737eda12dfe0 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 16:30:16 +0200 Subject: [PATCH 12/39] add DFA pretty-printer --- src/backends/lalr.c | 99 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 509f76d..15236f5 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -317,6 +317,93 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* +/* Pretty-printers */ + +void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item) +{ + h_pprint_symbol(f, g, item->lhs); + fputs(" ->", f); + + HCFChoice **x = item->rhs; + HCFChoice **mark = item->rhs + item->mark; + if(*x == NULL) { + fputs("\"\"", f); + } else { + while(*x) { + if(x == mark) + fputc('.', f); + else + fputc(' ', f); + + if((*x)->type == HCF_CHAR) { + // condense character strings + fputc('"', f); + h_pprint_char(f, (*x)->chr); + for(x++; *x; x++) { + if(x == mark) + break; + if((*x)->type != HCF_CHAR) + break; + h_pprint_char(f, (*x)->chr); + } + fputc('"', f); + } else { + h_pprint_symbol(f, g, *x); + x++; + } + } + if(x == mark) + fputs(".", f); + } +} + +void h_pprint_lrstate(FILE *f, const HCFGrammar *g, + const HLRState *state, unsigned int indent) +{ + bool first = true; + const HHashTable *ht = state; + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HLRItem *item = hte->key; + + if(!first) + for(unsigned int i=0; isymbol); + fprintf(f, "->%lu", t->to); +} + +void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, + const HLRDFA *dfa, unsigned int indent) +{ + for(size_t i=0; instates; i++) { + unsigned int indent2 = indent + fprintf(f, "%4lu: ", i); + h_pprint_lrstate(f, g, dfa->states[i], indent2); + for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { + const HLRTransition *t = x->elem; + if(t->from == i) { + for(unsigned int i=0; iast, 0, 2); From 732545274afce6cdd1daf212a207a25b71deae21 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 16:55:36 +0200 Subject: [PATCH 13/39] look up neighbor set index properly (was left uninitialized) --- src/backends/lalr.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 15236f5..7a07c3d 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -250,6 +250,8 @@ HLRDFA *h_lalr_dfa(HCFGrammar *g) h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); h_slist_push(work, neighbor); h_slist_push(work, (void *)(uintptr_t)neighbor_idx); + } else { + neighbor_idx = (uintptr_t)h_hashtable_get(states, neighbor); } // add transition "state --symbol--> neighbor" From f041775bb965613a2241facdf3d35f6ed3463c02 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 22:38:26 +0200 Subject: [PATCH 14/39] declare h_cfgrammar_new in cfgrammar.h --- src/cfgrammar.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cfgrammar.h b/src/cfgrammar.h index a5de4d6..62b3320 100644 --- a/src/cfgrammar.h +++ b/src/cfgrammar.h @@ -60,6 +60,8 @@ static inline HStringMap *h_stringmap_get_char(const HStringMap *m, const uint8_ */ HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser); +HCFGrammar *h_cfgrammar_new(HAllocator *mm__); + /* Frees the given grammar and associated data. * Does *not* free parsers' CFG forms as created by h_desugar. */ From 3bb26162c35db804f49a148011f06c828419b028 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 7 Jun 2013 22:39:09 +0200 Subject: [PATCH 15/39] end-of-day scratch commit (LALR table generation) --- src/backends/lalr.c | 134 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 7a07c3d..b3d9b5b 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -4,17 +4,6 @@ #include "../parsers/parser_internal.h" -// PLAN: -// data structures: -// - LR table is an array of hashtables that map grammar symbols (HCFChoice) -// to LRActions. - -// build LR(0) DFA -// extend with lookahead information by either: -// - reworking algorithm to propagate lookahead ("simple LALR generation") -// - follow sets of enhanced grammar ("conversion to SLR") - - /* Constructing the characteristic automaton (handle recognizer) */ // - states are hashsets containing LRItems @@ -288,17 +277,122 @@ HLRDFA *h_lalr_dfa(HCFGrammar *g) /* LALR table generation */ +typedef struct HLRAction_ { + enum {HLR_SHIFT, HLR_REDUCE} type; + union { + size_t nextstate; // used with shift + struct { + HCFChoice *lhs; + HCFChoice **rhs; + } production; // used with reduce + }; +} HLRAction; + +typedef struct HLRTable_ { + size_t nrows; + HHashTable **rows; // map symbols to HLRActions + HCFChoice *start; // start symbol + HArena *arena; + HAllocator *mm__; +} HLRTable; + +HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) +{ + HArena *arena = h_new_arena(mm__, 0); // default blocksize + assert(arena != NULL); + + HLRTable *ret = h_new(HLRTable, 1); + ret->nrows = nrows; + ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->arena = arena; + ret->mm__ = mm__; + + for(size_t i=0; irows[i] = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); + + return ret; +} + +static HCFGrammar *transform_grammar(const HCFGrammar *g, const HLRTable *table, + const HLRDFA *dfa, HHashTable **syms) +{ + HCFGrammar *gt = h_cfgrammar_new(g->mm__); + HArena *arena = gt->arena; + + // old grammar symbol -> + //HHashTable *map = h_hashtable_new( + + for(size_t i=0; instates; i++) { + const HLRState *state = dfa->states[i]; + + syms[i] = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); + + + } + + // iterate over g->nts + const HHashTable *ht = g->nts; + for(size_t i=0; i < ht->capacity; i++) { + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { + if(hte->key == NULL) + continue; + + const HCFChoice *A = hte->key; + + // iterate over the productions of A + for(HCFSequence **p=A->seq; *p; p++) { + // find all transitions marked by A + // yields xAy -> rhs' + // trace rhs starting in state x and following the transitions + } + } + } + + return gt; +} + int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) { - // generate grammar - // construct dfa / determine lookahead - // extract table - // create an array of hashtables, one per state - // for each transition a--S-->b: - // add "shift, goto b" to table entry (a,S) - // for each state: - // add reduce entries for its accepting items - return -1; + // generate CFG from parser + // construct LR(0) DFA + // build parse table, shift-entries only + // for each transition a--S-->b, add "shift, goto b" to table entry (a,S) + // determine lookahead "by conversion to SLR" + // transform grammar to encode transitions in symbols + // -> lookahead for an item is the transformed left-hand side's follow set + // finish table; for each state: + // add reduce entries for its accepting items + // in case of conflict, add lookahead info + + HCFGrammar *g = h_cfgrammar(mm__, parser); + if(g == NULL) // backend not suitable (language not context-free) + return -1; + + HLRDFA *dfa = h_lalr_dfa(g); + if(dfa == NULL) // this should actually not happen + return -1; + + // create table with shift actions + HLRTable *table = h_lrtable_new(mm__, dfa->nstates); + for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { + HLRTransition *t = x->elem; + HLRAction *action = h_arena_malloc(table->arena, sizeof(HLRAction)); + action->type = HLR_SHIFT; + action->nextstate = t->to; + h_hashtable_put(table->rows[t->from], t->symbol, action); + } + + // mapping (state,item)-pairs to the symbols of the new grammar + HHashTable **syms = h_arena_malloc(g->arena, dfa->nstates * sizeof(HHashTable *)); + // XXX use a different arena for this (and other things) + + HCFGrammar *gt = transform_grammar(g, table, dfa, syms); + if(gt == NULL) // this should actually not happen + return -1; + + // XXX fill in reduce actions + + return 0; } void h_lalr_free(HParser *parser) From dabe4b07a94835041aee26f98b5c7e594767f1a2 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 10 Jun 2013 00:46:03 +0200 Subject: [PATCH 16/39] wip end-of-day commit - almost there (untested) --- src/backends/lalr.c | 328 +++++++++++++++++++++++++++++++++----------- 1 file changed, 246 insertions(+), 82 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index b3d9b5b..541a69a 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -4,7 +4,8 @@ #include "../parsers/parser_internal.h" -/* Constructing the characteristic automaton (handle recognizer) */ + +/* Data structures */ // - states are hashsets containing LRItems // - LRItems contain an optional lookahead set (HStringMap) @@ -19,8 +20,9 @@ typedef struct HLRDFA_ { } HLRDFA; typedef struct HLRTransition_ { - size_t from, to; // indices into 'states' array + size_t from; // index into 'states' array const HCFChoice *symbol; + size_t to; // index into 'states' array } HLRTransition; typedef struct HLRItem_ { @@ -31,6 +33,32 @@ typedef struct HLRItem_ { HStringMap *lookahead; // optional } HLRItem; +typedef struct HLRAction_ { + enum {HLR_SHIFT, HLR_REDUCE} type; + union { + size_t nextstate; // used with SHIFT + struct { + HCFChoice *lhs; // symbol carrying semantic actions etc. + size_t length; // # of symbols in rhs + // NB: the rhs symbols are not needed for the parse + } production; // used with REDUCE + }; +} HLRAction; + +typedef struct HLRTable_ { + size_t nrows; + HHashTable **rows; // map symbols to HLRActions + HLRAction **forall; // shortcut to set an action for an entire row + HCFChoice *start; // start symbol + HSlist *inadeq; // indices of any inadequate states + HArena *arena; + HAllocator *mm__; +} HLRTable; + + + +/* Constructing the characteristic automaton (handle recognizer) */ + HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) { HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); @@ -157,7 +185,7 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) return ret; } -HLRDFA *h_lalr_dfa(HCFGrammar *g) +HLRDFA *h_lr0_dfa(HCFGrammar *g) { HArena *arena = g->arena; @@ -275,26 +303,24 @@ HLRDFA *h_lalr_dfa(HCFGrammar *g) -/* LALR table generation */ +/* LR(0) table generation */ -typedef struct HLRAction_ { - enum {HLR_SHIFT, HLR_REDUCE} type; - union { - size_t nextstate; // used with shift - struct { - HCFChoice *lhs; - HCFChoice **rhs; - } production; // used with reduce - }; -} HLRAction; +// XXX replace other hashtable iterations with this +// XXX move to internal.h or something +#define H_FOREACH_(HT) do { \ + const HHashTable *ht = HT; \ + for(size_t i=0; i < ht->capacity; i++) { \ + for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { \ + if(hte->key == NULL) continue; -typedef struct HLRTable_ { - size_t nrows; - HHashTable **rows; // map symbols to HLRActions - HCFChoice *start; // start symbol - HArena *arena; - HAllocator *mm__; -} HLRTable; +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_(HT) \ + const KEYVAR = hte->key; \ + VALVAR = hte->value; + +#define H_END_FOREACH \ + } \ + } \ + } while(0); HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) { @@ -313,91 +339,229 @@ HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) return ret; } -static HCFGrammar *transform_grammar(const HCFGrammar *g, const HLRTable *table, - const HLRDFA *dfa, HHashTable **syms) +void h_lrtable_free(HLRTable *table) { - HCFGrammar *gt = h_cfgrammar_new(g->mm__); - HArena *arena = gt->arena; + HAllocator *mm__ = table->mm__; + h_delete_arena(table->arena); + h_free(table); +} - // old grammar symbol -> - //HHashTable *map = h_hashtable_new( +static HLRAction *shift_action(HArena *arena, size_t nextstate) +{ + HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); + action->type = HLR_SHIFT; + action->nextstate = nextstate; + return action; +} +static HLRAction *reduce_action(HArena *arena, HCFChoice *lhs, size_t rhslen) +{ + HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); + action->type = HLR_REDUCE; + action->production.lhs = lhs; + action->production.length = rhslen; + return action; +} + +HLRTable *h_lr0_table(HCFGrammar *g) +{ + HAllocator *mm__ = g->mm__; + + // construct LR(0) DFA + HLRDFA *dfa = h_lr0_dfa(g); + if(!dfa) return NULL; + + HLRTable *table = h_lrtable_new(mm__, dfa->nstates); + HArena *arena = table->arena; + + // add shift entries + for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { + // for each transition x-A->y, add "shift, goto y" to table entry (x,A) + HLRTransition *t = x->elem; + + HLRAction *action = shift_action(arena, t->to); + h_hashtable_put(table->rows[t->from], t->symbol, action); + } + + // add reduce entries, record inadequate states for(size_t i=0; instates; i++) { - const HLRState *state = dfa->states[i]; - - syms[i] = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); - - - } - - // iterate over g->nts - const HHashTable *ht = g->nts; - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - const HCFChoice *A = hte->key; - - // iterate over the productions of A - for(HCFSequence **p=A->seq; *p; p++) { - // find all transitions marked by A - // yields xAy -> rhs' - // trace rhs starting in state x and following the transitions + // find reducible items in state + H_FOREACH(dfa->states[i], HLRItem *item, void *v_) + if(item->mark == item->len) { // mark at the end + // XXX store more informative stuff in the inadeq records? + if(table->forall[i]) { + // reduce/reduce conflict with a previous item + h_slist_push(table->inadeq, (void *)(uintptr_t)i); + } else if(!h_hashtable_empty(table->rows[i])) { + // shift/reduce conflict with one of the row's entries + h_slist_push(table->inadeq, (void *)(uintptr_t)i); + } else { + // set reduce action for the entire row + table->forall[i] = reduce_action(arena, item->lhs, item->len); + } } - } + H_END_FOREACH } - return gt; + return table; +} + + + +/* LALR-via-SLR grammar transformation */ + +static inline size_t seqsize(void *p_) +{ + size_t n=0; + for(void **p=p_; *p; p++) n++; + return n+1; +} + +static size_t follow_transition(const HLRTable *table, size_t x, HCFChoice *A) +{ + HLRAction *action = h_hashtable_get(table->rows[x], A); + assert(action != NULL); + assert(action->type == HLR_SHIFT); + return action->nextstate; +} + +static HCFChoice *transform_symbol(const HLRTable *table, HHashTable *map, + size_t x, HCFChoice *B, size_t z); + +static HCFChoice *transform_productions(const HLRTable *table, HHashTable *map, + size_t x, HCFChoice *xAy) +{ + HArena *arena = map->arena; + + HCFSequence **seq = h_arena_malloc(arena, seqsize(xAy->seq) + * sizeof(HCFSequence *)); + HCFSequence **p, **q; + for(p=xAy->seq, q=seq; *p; p++, q++) { + // trace rhs starting in state x and following the transitions + // xAy -> xBz ... + + HCFChoice **B = (*p)->items; + HCFChoice **xBz = h_arena_malloc(arena, seqsize(B) * sizeof(HCFChoice *)); + for(; *B; B++, xBz++) { + size_t z = follow_transition(table, x, *B); + *xBz = transform_symbol(table, map, x, *B, z); + x=z; + } + *xBz = NULL; + + *q = h_arena_malloc(arena, sizeof(HCFSequence)); + (*q)->items = xBz; + } + *q = NULL; + xAy->seq = seq; + + return xAy; // pass-through +} + +static inline HLRTransition *transition(HArena *arena, + size_t x, const HCFChoice *A, size_t y) +{ + HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); + t->from = x; + t->symbol = A; + t->to = y; + return t; +} + +static HCFChoice *transform_symbol(const HLRTable *table, HHashTable *map, + size_t x, HCFChoice *B, size_t z) +{ + HArena *arena = map->arena; + + // look up the transition in map, create symbol if not found + HLRTransition *x_B_z = transition(arena, x, B, z); + HCFChoice *xBz = h_hashtable_get(map, x_B_z); + if(!xBz) { + HCFChoice *xBz = h_arena_malloc(arena, sizeof(HCFChoice)); + *xBz = *B; + h_hashtable_put(map, x_B_z, xBz); + } + + return transform_productions(table, map, x, xBz); +} + +static bool eq_transition(const void *p, const void *q) +{ + const HLRTransition *a=p, *b=q; + return (a->from == b->from && a->to == b->to && a->symbol == b->symbol); +} + +static HHashValue hash_transition(const void *p) +{ + const HLRTransition *t = p; + return (h_hash_ptr(t->symbol) + t->from + t->to); // XXX ? +} + +static HHashTable *enhance_grammar(const HCFGrammar *g, const HLRTable *tbl) +{ + HArena *arena = g->arena; // XXX ? + HHashTable *map = h_hashtable_new(arena, eq_transition, hash_transition); + + // copy the start symbol over + HCFChoice *start = h_arena_malloc(arena, sizeof(HCFChoice)); + *start = *(g->start); + h_hashtable_put(map, g->start, start); + + transform_productions(tbl, map, 0, start); + + return map; +} + + + +/* LALR table generation */ + +bool is_inadequate(HLRTable *table, size_t state) +{ + // XXX +} + +bool has_conflicts(HLRTable *table) +{ + return !h_slist_empty(table->inadeq); } int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) { // generate CFG from parser - // construct LR(0) DFA - // build parse table, shift-entries only - // for each transition a--S-->b, add "shift, goto b" to table entry (a,S) - // determine lookahead "by conversion to SLR" - // transform grammar to encode transitions in symbols - // -> lookahead for an item is the transformed left-hand side's follow set - // finish table; for each state: - // add reduce entries for its accepting items - // in case of conflict, add lookahead info + // build LR(0) table + // if necessary, resolve conflicts "by conversion to SLR" HCFGrammar *g = h_cfgrammar(mm__, parser); if(g == NULL) // backend not suitable (language not context-free) return -1; - HLRDFA *dfa = h_lalr_dfa(g); - if(dfa == NULL) // this should actually not happen + HLRTable *table = h_lr0_table(g); + if(table == NULL) // this should normally not happen return -1; - // create table with shift actions - HLRTable *table = h_lrtable_new(mm__, dfa->nstates); - for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { - HLRTransition *t = x->elem; - HLRAction *action = h_arena_malloc(table->arena, sizeof(HLRAction)); - action->type = HLR_SHIFT; - action->nextstate = t->to; - h_hashtable_put(table->rows[t->from], t->symbol, action); + if(has_conflicts(table)) { + HHashTable *map = enhance_grammar(g, table); + if(map == NULL) // this should normally not happen + return -1; + + // XXX resolve conflicts + // iterate over dfa's transitions where 'from' state is inadequate + // look up enhanced symbol corr. to the transition + // for each terminal in follow set of enh. symbol: + // put reduce action into table cell (state, terminal) + // conflict if already occupied } - // mapping (state,item)-pairs to the symbols of the new grammar - HHashTable **syms = h_arena_malloc(g->arena, dfa->nstates * sizeof(HHashTable *)); - // XXX use a different arena for this (and other things) - - HCFGrammar *gt = transform_grammar(g, table, dfa, syms); - if(gt == NULL) // this should actually not happen - return -1; - - // XXX fill in reduce actions - - return 0; + h_cfgrammar_free(g); + parser->backend_data = table; + return has_conflicts(table)? -1 : 0; } void h_lalr_free(HParser *parser) { - // XXX free data structures + HLRTable *table = parser->backend_data; + h_lrtable_free(table); parser->backend_data = NULL; parser->backend = PB_PACKRAT; } @@ -538,7 +702,7 @@ int test_lalr(void) h_pprint_grammar(stdout, g, 0); printf("\n==== D F A ====\n"); - HLRDFA *dfa = h_lalr_dfa(g); + HLRDFA *dfa = h_lr0_dfa(g); if(dfa) h_pprint_lrdfa(stdout, g, dfa, 0); else From 4cd51b8953507fabb0c6cc73544a783e736d0288 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 10 Jun 2013 22:05:12 +0200 Subject: [PATCH 17/39] make it compile and fix some basic segfaults --- src/backends/lalr.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 541a69a..5b340e9 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -308,14 +308,16 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) // XXX replace other hashtable iterations with this // XXX move to internal.h or something #define H_FOREACH_(HT) do { \ - const HHashTable *ht = HT; \ - for(size_t i=0; i < ht->capacity; i++) { \ - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { \ - if(hte->key == NULL) continue; + const HHashTable *ht__ = HT; \ + for(size_t i__=0; i__ < ht__->capacity; i__++) { \ + for(HHashTableEntry *hte__ = &ht__->contents[i]; hte__; hte__ = hte__->next) { \ + if(hte__->key == NULL) continue; -#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_(HT) \ - const KEYVAR = hte->key; \ - VALVAR = hte->value; +#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ + const KEYVAR = hte__->key; \ + +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT) \ + VALVAR = hte__->value; #define H_END_FOREACH \ } \ @@ -330,6 +332,8 @@ HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) HLRTable *ret = h_new(HLRTable, 1); ret->nrows = nrows; ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); + ret->inadeq = h_slist_new(arena); ret->arena = arena; ret->mm__ = mm__; @@ -386,7 +390,7 @@ HLRTable *h_lr0_table(HCFGrammar *g) // add reduce entries, record inadequate states for(size_t i=0; instates; i++) { // find reducible items in state - H_FOREACH(dfa->states[i], HLRItem *item, void *v_) + H_FOREACH_KEY(dfa->states[i], HLRItem *item) if(item->mark == item->len) { // mark at the end // XXX store more informative stuff in the inadeq records? if(table->forall[i]) { @@ -519,6 +523,7 @@ static HHashTable *enhance_grammar(const HCFGrammar *g, const HLRTable *tbl) bool is_inadequate(HLRTable *table, size_t state) { // XXX + return false; } bool has_conflicts(HLRTable *table) From 04487ff80fce6d5e6932a4cf2516e0ea408928b7 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 10 Jun 2013 23:45:25 +0200 Subject: [PATCH 18/39] add table pretty-printer --- src/backends/lalr.c | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 5b340e9..ccffaf3 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -316,7 +316,7 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) #define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ const KEYVAR = hte__->key; \ -#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT) \ +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ VALVAR = hte__->value; #define H_END_FOREACH \ @@ -667,6 +667,40 @@ void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, } } +void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) +{ + if(action->type == HLR_SHIFT) { + fprintf(f, "s%lu", action->nextstate); + } else { + fputc('r', f); + // XXX reference the production somehow + } +} + +void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, + unsigned int indent) +{ + for(size_t i=0; inrows; i++) { + for(unsigned int j=0; jforall[i] && h_hashtable_empty(table->rows[i])) { + fputs(" - ", f); + pprint_lraction(f, g, table->forall[i]); + fputs(" -", f); + } + H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) + fputc(' ', f); // separator + h_pprint_symbol(f, g, symbol); + fputc(':', f); + if(table->forall[i]) { + fputc(action->type == HLR_SHIFT? 's' : 'r', f); + fputc('/', f); + fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); + } + H_END_FOREACH + fputc('\n', f); + } +} @@ -713,12 +747,19 @@ int test_lalr(void) else fprintf(stderr, "h_lalr_dfa failed\n"); + printf("\n==== L R ( 0 ) T A B L E ====\n"); + HLRTable *table0 = h_lr0_table(g); + if(table0) + h_pprint_lrtable(stdout, g, table0, 0); + else + fprintf(stderr, "h_lr0_table failed\n"); + printf("\n==== L A L R T A B L E ====\n"); if(h_compile(p, PB_LALR, NULL)) { fprintf(stderr, "does not compile\n"); return 2; } - // print LALR(1) table + h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); HParseResult *res = h_parse(p, (uint8_t *)"xyya", 4); From bbdced376ef5f577a63a0e2aaa946bb0d1b5efee Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 11 Jun 2013 00:27:34 +0200 Subject: [PATCH 19/39] derps etc --- src/backends/lalr.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index ccffaf3..2927b2a 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -307,14 +307,16 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) // XXX replace other hashtable iterations with this // XXX move to internal.h or something -#define H_FOREACH_(HT) do { \ +#define H_FOREACH_(HT) { \ const HHashTable *ht__ = HT; \ for(size_t i__=0; i__ < ht__->capacity; i__++) { \ - for(HHashTableEntry *hte__ = &ht__->contents[i]; hte__; hte__ = hte__->next) { \ + for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ + hte__; \ + hte__ = hte__->next) { \ if(hte__->key == NULL) continue; #define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ - const KEYVAR = hte__->key; \ + const KEYVAR = hte__->key; #define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ VALVAR = hte__->value; @@ -322,7 +324,7 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) #define H_END_FOREACH \ } \ } \ - } while(0); + } HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) { @@ -337,8 +339,10 @@ HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) ret->arena = arena; ret->mm__ = mm__; - for(size_t i=0; irows[i] = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); + ret->forall[i] = NULL; + } return ret; } @@ -687,17 +691,20 @@ void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, fputs(" - ", f); pprint_lraction(f, g, table->forall[i]); fputs(" -", f); + } else { + H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) + fputc(' ', f); // separator + h_pprint_symbol(f, g, symbol); + fputc(':', f); + if(table->forall[i]) { + fputc(action->type == HLR_SHIFT? 's' : 'r', f); + fputc('/', f); + fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); + } else { + pprint_lraction(f, g, action); + } + H_END_FOREACH } - H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) - fputc(' ', f); // separator - h_pprint_symbol(f, g, symbol); - fputc(':', f); - if(table->forall[i]) { - fputc(action->type == HLR_SHIFT? 's' : 'r', f); - fputc('/', f); - fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); - } - H_END_FOREACH fputc('\n', f); } } @@ -753,6 +760,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, table0, 0); else fprintf(stderr, "h_lr0_table failed\n"); + h_lrtable_free(table0); printf("\n==== L A L R T A B L E ====\n"); if(h_compile(p, PB_LALR, NULL)) { From 0a7548bb2fb39f5cd130cebce41d66facb46bcc3 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 15:07:19 +0200 Subject: [PATCH 20/39] lr driver, first stab (still bugged) --- src/backends/lalr.c | 282 ++++++++++++++++++++++++++++++++------------ 1 file changed, 207 insertions(+), 75 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 2927b2a..5b8ea65 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -7,11 +7,7 @@ /* Data structures */ -// - states are hashsets containing LRItems -// - LRItems contain an optional lookahead set (HStringMap) -// - states (hashsets) get hash and comparison functions that ignore the lookahead - -typedef HHashSet HLRState; +typedef HHashSet HLRState; // states are sets of LRItems typedef struct HLRDFA_ { size_t nstates; @@ -30,7 +26,6 @@ typedef struct HLRItem_ { HCFChoice **rhs; // NULL-terminated size_t len; // number of elements in rhs size_t mark; - HStringMap *lookahead; // optional } HLRItem; typedef struct HLRAction_ { @@ -56,27 +51,28 @@ typedef struct HLRTable_ { } HLRTable; - -/* Constructing the characteristic automaton (handle recognizer) */ - -HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) +// compare symbols - terminals by value, others by pointer +static bool eq_symbol(const void *p, const void *q) { - HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); - - size_t len = 0; - for(HCFChoice **p=rhs; *p; p++) len++; - assert(mark <= len); - - ret->lhs = lhs; - ret->rhs = rhs; - ret->len = len; - ret->mark = mark; - ret->lookahead = NULL; - - return ret; + const HCFChoice *x=p, *y=q; + return (x==y + || (x->type==HCF_END && y->type==HCF_END) + || (x->type==HCF_CHAR && y->type==HCF_CHAR && x->chr==y->chr)); } -// compare LALR items - ignores lookahead +// hash symbols - terminals by value, others by pointer +static HHashValue hash_symbol(const void *p) +{ + const HCFChoice *x=p; + if(x->type == HCF_END) + return 0; + else if(x->type == HCF_CHAR) + return x->chr * 33; + else + return h_hash_ptr(p); +} + +// compare LALR items by value static bool eq_lalr_item(const void *p, const void *q) { const HLRItem *a=p, *b=q; @@ -124,11 +120,80 @@ static HHashValue hash_lalr_itemset(const void *p) return hash; } +HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) +{ + HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); + + size_t len = 0; + for(HCFChoice **p=rhs; *p; p++) len++; + assert(mark <= len); + + ret->lhs = lhs; + ret->rhs = rhs; + ret->len = len; + ret->mark = mark; + + return ret; +} + static inline HLRState *h_lrstate_new(HArena *arena) { return h_hashset_new(arena, eq_lalr_item, hash_lalr_item); } +HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) +{ + HArena *arena = h_new_arena(mm__, 0); // default blocksize + assert(arena != NULL); + + HLRTable *ret = h_new(HLRTable, 1); + ret->nrows = nrows; + ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); + ret->inadeq = h_slist_new(arena); + ret->arena = arena; + ret->mm__ = mm__; + + for(size_t i=0; irows[i] = h_hashtable_new(arena, eq_symbol, hash_symbol); + ret->forall[i] = NULL; + } + + return ret; +} + +void h_lrtable_free(HLRTable *table) +{ + HAllocator *mm__ = table->mm__; + h_delete_arena(table->arena); + h_free(table); +} + +// XXX replace other hashtable iterations with this +// XXX move to internal.h or something +#define H_FOREACH_(HT) { \ + const HHashTable *ht__ = HT; \ + for(size_t i__=0; i__ < ht__->capacity; i__++) { \ + for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ + hte__; \ + hte__ = hte__->next) { \ + if(hte__->key == NULL) continue; + +#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ + const KEYVAR = hte__->key; + +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ + VALVAR = hte__->value; + +#define H_END_FOREACH \ + } \ + } \ + } + + + +/* Constructing the characteristic automaton (handle recognizer) */ + static HLRItem *advance_mark(HArena *arena, const HLRItem *item) { assert(item->rhs[item->mark] != NULL); @@ -224,7 +289,7 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) HLRState *state = h_slist_pop(work); // maps edge symbols to neighbor states (item sets) of s - HHashTable *neighbors = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); + HHashTable *neighbors = h_hashtable_new(arena, eq_symbol, hash_symbol); // iterate over closure and generate neighboring sets const HHashTable *ht = closure(g, state); @@ -305,55 +370,6 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) /* LR(0) table generation */ -// XXX replace other hashtable iterations with this -// XXX move to internal.h or something -#define H_FOREACH_(HT) { \ - const HHashTable *ht__ = HT; \ - for(size_t i__=0; i__ < ht__->capacity; i__++) { \ - for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ - hte__; \ - hte__ = hte__->next) { \ - if(hte__->key == NULL) continue; - -#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ - const KEYVAR = hte__->key; - -#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ - VALVAR = hte__->value; - -#define H_END_FOREACH \ - } \ - } \ - } - -HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) -{ - HArena *arena = h_new_arena(mm__, 0); // default blocksize - assert(arena != NULL); - - HLRTable *ret = h_new(HLRTable, 1); - ret->nrows = nrows; - ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); - ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); - ret->inadeq = h_slist_new(arena); - ret->arena = arena; - ret->mm__ = mm__; - - for(size_t i=0; irows[i] = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); - ret->forall[i] = NULL; - } - - return ret; -} - -void h_lrtable_free(HLRTable *table) -{ - HAllocator *mm__ = table->mm__; - h_delete_arena(table->arena); - h_free(table); -} - static HLRAction *shift_action(HArena *arena, size_t nextstate) { HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); @@ -579,9 +595,125 @@ void h_lalr_free(HParser *parser) /* LR driver */ +const HLRAction * +h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) +{ + assert(state < table->nrows); + if(table->forall[state]) { + assert(h_hashtable_empty(table->rows[state])); // that would be a conflict + return table->forall[state]; + } else { + return h_hashtable_get(table->rows[state], symbol); + } +} + +// XXX also, what about charsets!? + HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) { - return NULL; + HLRTable *table = parser->backend_data; + if(!table) + return NULL; + + HArena *arena = h_new_arena(mm__, 0); // will hold the results + HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse + HSlist *left = h_slist_new(tarena); // left stack; reductions happen here + HSlist *right = h_slist_new(tarena); // right stack; input appears here + + // stack layout: + // on the left stack, we put pairs: (saved state, semantic value) + // on the right stack, we put pairs: (symbol, semantic value) + + // run while the recognizer finds handles in the input + size_t state = 0; + while(1) { + // make sure there is input on the right stack + if(h_slist_empty(right)) { + HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); + HParsedToken *v; + + uint8_t c = h_read_bits(stream, 8, false); + + if(stream->overrun) { // end of input + x->type = HCF_END; + v = NULL; + } else { + x->type = HCF_CHAR; + x->chr = c; + v = h_arena_malloc(arena, sizeof(HParsedToken)); + v->token_type = TT_UINT; + v->uint = c; + } + + h_slist_push(right, v); + h_slist_push(right, x); + } + + // peek at input symbol on the right side + HCFChoice *symbol = right->head->elem; + + // table lookup + const HLRAction *action = h_lr_lookup(table, state, symbol); + if(action == NULL) + break; // no handle recognizable in input, terminate parsing + + if(action->type == HLR_SHIFT) { + h_slist_push(left, (void *)(uintptr_t)state); + h_slist_pop(right); // symbol (discard) + h_slist_push(left, h_slist_pop(right)); // semantic value + state = action->nextstate; + } else { + assert(action->type == HLR_REDUCE); + size_t len = action->production.length; + HCFChoice *symbol = action->production.lhs; + + // semantic value of the reduction result + HParsedToken *value = h_arena_malloc(arena, sizeof(HParsedToken)); + value->token_type = TT_SEQUENCE; + value->seq = h_carray_new_sized(arena, len); + + // pull values off the left stack, rewinding state accordingly + HParsedToken *v; + for(size_t i=0; iseq->elements[len-1-i] = v; + value->seq->used++; + } + // result position equals position of left-most symbol + value->index = v->index; + value->bit_offset = v->bit_offset; + + // perform token reshape if indicated + if(symbol->reshape) + value = (HParsedToken *)symbol->reshape(make_result(arena, value)); + + // call validation and semantic action, if present + if(symbol->pred && !symbol->pred(make_result(tarena, value))) + break; // validation failed -> no parse + if(symbol->action) + value = (HParsedToken *)symbol->action(make_result(arena, value)); + + // push result (value, symbol) onto the right stack + h_slist_push(right, value); + h_slist_push(right, symbol); + } + } + + h_delete_arena(tarena); + + // parsing was successful iff the start symbol is on top of the right stack + if(h_slist_pop(right) == table->start) { + // next on the right stack is the start symbol's semantic value + HParsedToken *result = h_slist_pop(right); + assert(result != NULL); + return make_result(arena, result); + } else { + h_delete_arena(arena); + return NULL; + } } From 805dfeb363ef22ba5e4c64fa19b37bb300cba5aa Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 15:33:53 +0200 Subject: [PATCH 21/39] duh, forgot to record the start symbol in table --- src/backends/lalr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 5b8ea65..a11baf2 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -398,6 +398,9 @@ HLRTable *h_lr0_table(HCFGrammar *g) HLRTable *table = h_lrtable_new(mm__, dfa->nstates); HArena *arena = table->arena; + // remember start symbol + table->start = g->start; + // add shift entries for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { // for each transition x-A->y, add "shift, goto y" to table entry (x,A) From ce387d81b685844ac471cc23e9c3dfb94fd4c34e Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 15:48:52 +0200 Subject: [PATCH 22/39] save production rhs in reduce actions for pretty-printing if not compiled NDEBUG --- src/backends/lalr.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index a11baf2..e974da6 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -35,7 +35,9 @@ typedef struct HLRAction_ { struct { HCFChoice *lhs; // symbol carrying semantic actions etc. size_t length; // # of symbols in rhs - // NB: the rhs symbols are not needed for the parse +#ifndef NDEBUG + HCFChoice **rhs; // NB: the rhs symbols are not needed for the parse +#endif } production; // used with REDUCE }; } HLRAction; @@ -378,12 +380,15 @@ static HLRAction *shift_action(HArena *arena, size_t nextstate) return action; } -static HLRAction *reduce_action(HArena *arena, HCFChoice *lhs, size_t rhslen) +static HLRAction *reduce_action(HArena *arena, const HLRItem *item) { HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); action->type = HLR_REDUCE; - action->production.lhs = lhs; - action->production.length = rhslen; + action->production.lhs = item->lhs; + action->production.length = item->len; +#ifndef NDEBUG + action->production.rhs = item->rhs; +#endif return action; } @@ -424,7 +429,7 @@ HLRTable *h_lr0_table(HCFGrammar *g) h_slist_push(table->inadeq, (void *)(uintptr_t)i); } else { // set reduce action for the entire row - table->forall[i] = reduce_action(arena, item->lhs, item->len); + table->forall[i] = reduce_action(arena, item); } } H_END_FOREACH @@ -811,8 +816,15 @@ void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) if(action->type == HLR_SHIFT) { fprintf(f, "s%lu", action->nextstate); } else { - fputc('r', f); - // XXX reference the production somehow +#ifdef NDEBUG + // if we can't print the production, at least print its length + fprintf(f, "r[%lu]", action->production.length); +#else + fputs("r(", f); + HCFSequence seq = {action->production.rhs}; + h_pprint_sequence(f, g, &seq); + fputc(')', f); +#endif } } From 41d4be4b84b8abd42bc968d2f8bb01d758cf1eac Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 15:54:46 +0200 Subject: [PATCH 23/39] avoid a segfault on getting the token position --- src/backends/lalr.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index e974da6..8178f2d 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -690,9 +690,13 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* value->seq->elements[len-1-i] = v; value->seq->used++; } - // result position equals position of left-most symbol - value->index = v->index; - value->bit_offset = v->bit_offset; + if(v) { + // result position equals position of left-most symbol + value->index = v->index; + value->bit_offset = v->bit_offset; + } else { + // XXX how to get the position in this case? + } // perform token reshape if indicated if(symbol->reshape) From 4b5a93666723380e26e58ea23fad5c244b6ee66a Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 16:38:50 +0200 Subject: [PATCH 24/39] handle charsets --- src/backends/lalr.c | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 8178f2d..66fe42c 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -229,15 +229,33 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) HCFChoice *sym = item->rhs[item->mark]; // symbol after mark // if there is a non-terminal after the mark, follow it - // XXX: do we have to count HCF_CHARSET as nonterminal? - if(sym != NULL && sym->type == HCF_CHOICE) { + // NB: unlike LLk, we do consider HCF_CHARSET a non-terminal here + if(sym != NULL && (sym->type==HCF_CHOICE || sym->type==HCF_CHARSET)) { // add items corresponding to the productions of sym - for(HCFSequence **p=sym->seq; *p; p++) { - HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); - if(!h_hashset_present(ret, it)) { - h_hashset_put(ret, it); - h_slist_push(work, it); + if(sym->type == HCF_CHOICE) { + for(HCFSequence **p=sym->seq; *p; p++) { + HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); + if(!h_hashset_present(ret, it)) { + h_hashset_put(ret, it); + h_slist_push(work, it); + } } + } else { // HCF_CHARSET + for(unsigned int i=0; i<256; i++) { + if(charset_isset(sym->charset, i)) { + HCFChoice **rhs = h_arena_malloc(arena, 2 * sizeof(HCFChoice *)); + rhs[0] = h_arena_malloc(arena, sizeof(HCFChoice)); + rhs[0]->type = HCF_CHAR; + rhs[0]->chr = i; + rhs[1] = NULL; + HLRItem *it = h_lritem_new(arena, sym, rhs, 0); + h_hashset_put(ret, it); + // single-character item needs no further work + } + } + // if sym is a non-terminal, we need a reshape on it + // this seems as good a place as any to set it + sym->reshape = h_act_first; } // if sym derives epsilon, also advance over it @@ -615,8 +633,6 @@ h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) } } -// XXX also, what about charsets!? - HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) { HLRTable *table = parser->backend_data; @@ -884,7 +900,7 @@ int test_lalr(void) */ // XXX make LALR example - HParser *X = h_optional(h_ch('x')); + HParser *X = h_optional(h_in((uint8_t *)"rst", 3)); HParser *Y = h_sequence(h_ch('y'), h_ch('y'), NULL); HParser *A = h_sequence(X, Y, h_ch('a'), NULL); HParser *B = h_sequence(Y, h_ch('b'), NULL); @@ -921,7 +937,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"xyya", 4); + HParseResult *res = h_parse(p, (uint8_t *)"syya", 4); if(res) h_pprint(stdout, res->ast, 0, 2); else From ddfd3796a724b8819744876ea0603e68c7e2d340 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 12 Jun 2013 16:48:59 +0200 Subject: [PATCH 25/39] why not include the lhs in pretty-printed reduce entries? --- src/backends/lalr.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 66fe42c..aad47d4 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -836,15 +836,17 @@ void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) if(action->type == HLR_SHIFT) { fprintf(f, "s%lu", action->nextstate); } else { + fputs("r(", f); + h_pprint_symbol(f, g, action->production.lhs); + fputs(" -> ", f); #ifdef NDEBUG // if we can't print the production, at least print its length - fprintf(f, "r[%lu]", action->production.length); + fprintf(f, "[%lu]", action->production.length); #else - fputs("r(", f); HCFSequence seq = {action->production.rhs}; h_pprint_sequence(f, g, &seq); - fputc(')', f); #endif + fputc(')', f); } } From fd297b636c05c6e46882485131d8f519c6fcb954 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 13 Jun 2013 14:45:26 +0200 Subject: [PATCH 26/39] add LALR conflict resolution (untested) --- src/backends/lalr.c | 170 ++++++++++++++++++++++++++++++++++---------- src/cfgrammar.c | 5 ++ src/cfgrammar.h | 1 + 3 files changed, 138 insertions(+), 38 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index aad47d4..7e83c53 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -52,6 +52,13 @@ typedef struct HLRTable_ { HAllocator *mm__; } HLRTable; +typedef struct HLREnhGrammar_ { + HCFGrammar *grammar; // enhanced grammar + HHashTable *tmap; // maps transitions to enhanced-grammar symbols + HHashTable *smap; // maps enhanced-grammar symbols to transitions + HArena *arena; +} HLREnhGrammar; + // compare symbols - terminals by value, others by pointer static bool eq_symbol(const void *p, const void *q) @@ -410,14 +417,10 @@ static HLRAction *reduce_action(HArena *arena, const HLRItem *item) return action; } -HLRTable *h_lr0_table(HCFGrammar *g) +HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) { HAllocator *mm__ = g->mm__; - // construct LR(0) DFA - HLRDFA *dfa = h_lr0_dfa(g); - if(!dfa) return NULL; - HLRTable *table = h_lrtable_new(mm__, dfa->nstates); HArena *arena = table->arena; @@ -475,13 +478,13 @@ static size_t follow_transition(const HLRTable *table, size_t x, HCFChoice *A) return action->nextstate; } -static HCFChoice *transform_symbol(const HLRTable *table, HHashTable *map, +static HCFChoice *transform_symbol(const HLRTable *table, HLREnhGrammar *eg, size_t x, HCFChoice *B, size_t z); -static HCFChoice *transform_productions(const HLRTable *table, HHashTable *map, +static HCFChoice *transform_productions(const HLRTable *table, HLREnhGrammar *eg, size_t x, HCFChoice *xAy) { - HArena *arena = map->arena; + HArena *arena = eg->arena; HCFSequence **seq = h_arena_malloc(arena, seqsize(xAy->seq) * sizeof(HCFSequence *)); @@ -494,7 +497,7 @@ static HCFChoice *transform_productions(const HLRTable *table, HHashTable *map, HCFChoice **xBz = h_arena_malloc(arena, seqsize(B) * sizeof(HCFChoice *)); for(; *B; B++, xBz++) { size_t z = follow_transition(table, x, *B); - *xBz = transform_symbol(table, map, x, *B, z); + *xBz = transform_symbol(table, eg, x, *B, z); x=z; } *xBz = NULL; @@ -518,21 +521,22 @@ static inline HLRTransition *transition(HArena *arena, return t; } -static HCFChoice *transform_symbol(const HLRTable *table, HHashTable *map, +static HCFChoice *transform_symbol(const HLRTable *table, HLREnhGrammar *eg, size_t x, HCFChoice *B, size_t z) { - HArena *arena = map->arena; + HArena *arena = eg->arena; // look up the transition in map, create symbol if not found HLRTransition *x_B_z = transition(arena, x, B, z); - HCFChoice *xBz = h_hashtable_get(map, x_B_z); + HCFChoice *xBz = h_hashtable_get(eg->tmap, x_B_z); if(!xBz) { HCFChoice *xBz = h_arena_malloc(arena, sizeof(HCFChoice)); *xBz = *B; - h_hashtable_put(map, x_B_z, xBz); + h_hashtable_put(eg->tmap, x_B_z, xBz); + h_hashtable_put(eg->smap, xBz, x_B_z); } - return transform_productions(table, map, x, xBz); + return transform_productions(table, eg, x, xBz); } static bool eq_transition(const void *p, const void *q) @@ -547,39 +551,71 @@ static HHashValue hash_transition(const void *p) return (h_hash_ptr(t->symbol) + t->from + t->to); // XXX ? } -static HHashTable *enhance_grammar(const HCFGrammar *g, const HLRTable *tbl) +static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRTable *tbl) { + HAllocator *mm__ = g->mm__; HArena *arena = g->arena; // XXX ? - HHashTable *map = h_hashtable_new(arena, eq_transition, hash_transition); + + HLREnhGrammar *eg = h_arena_malloc(arena, sizeof(HLREnhGrammar)); + eg->tmap = h_hashtable_new(arena, eq_transition, hash_transition); + eg->smap = h_hashtable_new(arena, eq_transition, hash_transition); + eg->arena = arena; // copy the start symbol over HCFChoice *start = h_arena_malloc(arena, sizeof(HCFChoice)); *start = *(g->start); - h_hashtable_put(map, g->start, start); - transform_productions(tbl, map, 0, start); + transform_productions(tbl, eg, 0, start); - return map; + eg->grammar = h_cfgrammar_(mm__, start); + return eg; } /* LALR table generation */ -bool is_inadequate(HLRTable *table, size_t state) -{ - // XXX - return false; -} - -bool has_conflicts(HLRTable *table) +static inline bool has_conflicts(HLRTable *table) { return !h_slist_empty(table->inadeq); } +// place a new entry in tbl; records conflicts in tbl->inadeq +// returns 0 on success, -1 on conflict +// ignores forall entries +int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) +{ + HLRAction *prev = h_hashtable_get(tbl->rows[state], x); + if(prev && prev != action) { + // conflict + h_slist_push(tbl->inadeq, (void *)(uintptr_t)state); + return -1; + } else { + h_hashtable_put(tbl->rows[state], x, action); + return 0; + } +} + +// check whether a sequence of enhanced-grammar symbols (p) matches the given +// (original-grammar) production rhs and terminates in the given end state. +bool match_production(HLREnhGrammar *eg, HCFChoice **p, + HCFChoice **rhs, size_t endstate) +{ + HLRTransition *t; + for(; *p && *rhs; p++, rhs++) { + t = h_hashtable_get(eg->smap, *p); + assert(t != NULL); + if(!eq_symbol(t->symbol, *rhs)) + return false; + } + return (*p == *rhs // both NULL + && t->to == endstate); +} + int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) { // generate CFG from parser + // construct LR(0) DFA // build LR(0) table // if necessary, resolve conflicts "by conversion to SLR" @@ -587,21 +623,79 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) if(g == NULL) // backend not suitable (language not context-free) return -1; - HLRTable *table = h_lr0_table(g); - if(table == NULL) // this should normally not happen + HLRDFA *dfa = h_lr0_dfa(g); + if(dfa == NULL) { // this should normally not happen + h_cfgrammar_free(g); return -1; + } + + HLRTable *table = h_lr0_table(g, dfa); + if(table == NULL) { // this should normally not happen + h_cfgrammar_free(g); + return -1; + } if(has_conflicts(table)) { - HHashTable *map = enhance_grammar(g, table); - if(map == NULL) // this should normally not happen - return -1; + HArena *arena = table->arena; - // XXX resolve conflicts - // iterate over dfa's transitions where 'from' state is inadequate - // look up enhanced symbol corr. to the transition - // for each terminal in follow set of enh. symbol: - // put reduce action into table cell (state, terminal) - // conflict if already occupied + HLREnhGrammar *eg = enhance_grammar(g, table); + if(eg == NULL) { // this should normally not happen + h_cfgrammar_free(g); + h_lrtable_free(table); + return -1; + } + + // go through the inadequate states; replace inadeq with a new list + HSlist *inadeq = table->inadeq; + table->inadeq = h_slist_new(arena); + + for(HSlistNode *x=inadeq->head; x; x=x->next) { + size_t state = (uintptr_t)x->elem; + + // clear old forall entry, it's being replaced by more fine-grained ones + table->forall[state] = NULL; + + // go through each reducible item of state + H_FOREACH_KEY(dfa->states[state], HLRItem *item) + if(item->mark < item->len) + continue; + + // action to place in the table cells indicated by lookahead + HLRAction *action = reduce_action(arena, item); + + // find all LR(0)-enhanced productions matching item + H_FOREACH(eg->smap, HCFChoice *lhs, HLRTransition *t) + if(t->symbol != item->lhs) + continue; + for(HCFSequence **p=lhs->seq; *p; p++) { + HCFChoice **rhs = (*p)->items; + if(!match_production(eg, rhs, item->rhs, state)) + continue; + + // the left-hand symbol's follow set is this production's + // contribution to the lookahead + const HStringMap *fs = h_follow(1, eg->grammar, lhs); + assert(fs != NULL); + + // for each lookahead symbol, put action into table cell + if(fs->end_branch) { + HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); + terminal->type = HCF_END; + h_lrtable_put(table, state, terminal, action); + } + H_FOREACH(fs->char_branches, void *key, HStringMap *m) + if(!m->epsilon_branch) + continue; + + HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); + terminal->type = HCF_CHAR; + terminal->chr = key_char((HCharKey)key); + + h_lrtable_put(table, state, terminal, action); + H_END_FOREACH // lookahead character + } H_END_FOREACH // enhanced production + H_END_FOREACH // reducible item + } } h_cfgrammar_free(g); @@ -924,7 +1018,7 @@ int test_lalr(void) fprintf(stderr, "h_lalr_dfa failed\n"); printf("\n==== L R ( 0 ) T A B L E ====\n"); - HLRTable *table0 = h_lr0_table(g); + HLRTable *table0 = h_lr0_table(g, dfa); if(table0) h_pprint_lrtable(stdout, g, table0, 0); else diff --git a/src/cfgrammar.c b/src/cfgrammar.c index bc7b358..a874236 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -51,6 +51,11 @@ HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser) if(desugared == NULL) return NULL; // -> backend not suitable for this parser + return h_cfgrammar_(mm__, desugared); +} + +HCFGrammar *h_cfgrammar_(HAllocator* mm__, HCFChoice *desugared) +{ HCFGrammar *g = h_cfgrammar_new(mm__); // recursively traverse the desugared form and collect all HCFChoices that diff --git a/src/cfgrammar.h b/src/cfgrammar.h index 62b3320..c70c68a 100644 --- a/src/cfgrammar.h +++ b/src/cfgrammar.h @@ -59,6 +59,7 @@ static inline HStringMap *h_stringmap_get_char(const HStringMap *m, const uint8_ * A NULL return means we are unable to represent the parser as a CFG. */ HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser); +HCFGrammar *h_cfgrammar_(HAllocator* mm__, HCFChoice *start); HCFGrammar *h_cfgrammar_new(HAllocator *mm__); From 92f16a4d14be273721fff1c46a55864aba2116f4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 12:24:18 +0200 Subject: [PATCH 27/39] proper LALR example with conflict in LR(0) --- src/backends/lalr.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 7e83c53..9c9b864 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -667,6 +667,8 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) H_FOREACH(eg->smap, HCFChoice *lhs, HLRTransition *t) if(t->symbol != item->lhs) continue; + assert(lhs->type == HCF_CHOICE); // XXX could be CHARSET + for(HCFSequence **p=lhs->seq; *p; p++) { HCFChoice **rhs = (*p)->items; if(!match_production(eg, rhs, item->rhs, state)) @@ -986,21 +988,21 @@ HParserBackendVTable h__lalr_backend_vtable = { // dummy! int test_lalr(void) { - /* for k=2: - - S -> A | B - A -> X Y a - B -> Y b - X -> x | '' - Y -> y -- for k=3 use "yy" + /* + S -> E + E -> E '-' T + | T + T -> '(' E ')' + | N + N -> '0' -- also try [0-9] for the charset paths */ - // XXX make LALR example - HParser *X = h_optional(h_in((uint8_t *)"rst", 3)); - HParser *Y = h_sequence(h_ch('y'), h_ch('y'), NULL); - HParser *A = h_sequence(X, Y, h_ch('a'), NULL); - HParser *B = h_sequence(Y, h_ch('b'), NULL); - HParser *p = h_choice(A, B, NULL); + HParser *N = h_sequence(h_ch('n'), NULL); + HParser *E = h_indirect(); + HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), N, NULL); + HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); + h_bind_indirect(E, E_); + HParser *p = h_sequence(E, NULL); printf("\n==== G R A M M A R ====\n"); HCFGrammar *g = h_cfgrammar(&system_allocator, p); From bfc2433320a2af65e2e4bc39cc680f950c35a685 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 17:07:56 +0200 Subject: [PATCH 28/39] don't call compare function on NULL hashtable keys --- src/datastructures.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/datastructures.c b/src/datastructures.c index 55b8345..075b966 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -147,6 +147,8 @@ void* h_hashtable_get(const HHashTable* ht, const void* key) { for (hte = &ht->contents[hashval & (ht->capacity - 1)]; hte != NULL; hte = hte->next) { + if (hte->key == NULL) + continue; if (hte->hashval != hashval) continue; if (ht->equalFunc(key, hte->key)) From 06acbe2fb5415396e5494bbcb6998aa87a3b431c Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 17:17:23 +0200 Subject: [PATCH 29/39] LALR example parses! --- src/backends/lalr.c | 187 +++++++++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 80 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 9c9b864..e3a866b 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -56,6 +56,7 @@ typedef struct HLREnhGrammar_ { HCFGrammar *grammar; // enhanced grammar HHashTable *tmap; // maps transitions to enhanced-grammar symbols HHashTable *smap; // maps enhanced-grammar symbols to transitions + HHashTable *corr; // maps symbols to sets of corresponding e. symbols HArena *arena; } HLREnhGrammar; @@ -441,6 +442,7 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // find reducible items in state H_FOREACH_KEY(dfa->states[i], HLRItem *item) if(item->mark == item->len) { // mark at the end + // check for conflicts // XXX store more informative stuff in the inadeq records? if(table->forall[i]) { // reduce/reduce conflict with a previous item @@ -448,10 +450,10 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) } else if(!h_hashtable_empty(table->rows[i])) { // shift/reduce conflict with one of the row's entries h_slist_push(table->inadeq, (void *)(uintptr_t)i); - } else { - // set reduce action for the entire row - table->forall[i] = reduce_action(arena, item); } + + // set reduce action for the entire row + table->forall[i] = reduce_action(arena, item); } H_END_FOREACH } @@ -478,39 +480,6 @@ static size_t follow_transition(const HLRTable *table, size_t x, HCFChoice *A) return action->nextstate; } -static HCFChoice *transform_symbol(const HLRTable *table, HLREnhGrammar *eg, - size_t x, HCFChoice *B, size_t z); - -static HCFChoice *transform_productions(const HLRTable *table, HLREnhGrammar *eg, - size_t x, HCFChoice *xAy) -{ - HArena *arena = eg->arena; - - HCFSequence **seq = h_arena_malloc(arena, seqsize(xAy->seq) - * sizeof(HCFSequence *)); - HCFSequence **p, **q; - for(p=xAy->seq, q=seq; *p; p++, q++) { - // trace rhs starting in state x and following the transitions - // xAy -> xBz ... - - HCFChoice **B = (*p)->items; - HCFChoice **xBz = h_arena_malloc(arena, seqsize(B) * sizeof(HCFChoice *)); - for(; *B; B++, xBz++) { - size_t z = follow_transition(table, x, *B); - *xBz = transform_symbol(table, eg, x, *B, z); - x=z; - } - *xBz = NULL; - - *q = h_arena_malloc(arena, sizeof(HCFSequence)); - (*q)->items = xBz; - } - *q = NULL; - xAy->seq = seq; - - return xAy; // pass-through -} - static inline HLRTransition *transition(HArena *arena, size_t x, const HCFChoice *A, size_t y) { @@ -521,51 +490,101 @@ static inline HLRTransition *transition(HArena *arena, return t; } -static HCFChoice *transform_symbol(const HLRTable *table, HLREnhGrammar *eg, - size_t x, HCFChoice *B, size_t z) +// no-op on terminal symbols +static void transform_productions(const HLRTable *table, HLREnhGrammar *eg, + size_t x, HCFChoice *xAy) { + if(xAy->type != HCF_CHOICE) + return; + // XXX CHARSET? + HArena *arena = eg->arena; - // look up the transition in map, create symbol if not found - HLRTransition *x_B_z = transition(arena, x, B, z); - HCFChoice *xBz = h_hashtable_get(eg->tmap, x_B_z); - if(!xBz) { - HCFChoice *xBz = h_arena_malloc(arena, sizeof(HCFChoice)); - *xBz = *B; - h_hashtable_put(eg->tmap, x_B_z, xBz); - h_hashtable_put(eg->smap, xBz, x_B_z); - } + HCFSequence **seq = h_arena_malloc(arena, seqsize(xAy->seq) + * sizeof(HCFSequence *)); + HCFSequence **p, **q; + for(p=xAy->seq, q=seq; *p; p++, q++) { + // trace rhs starting in state x and following the transitions + // xAy -> ... iBj ... - return transform_productions(table, eg, x, xBz); + size_t i = x; + HCFChoice **B = (*p)->items; + HCFChoice **items = h_arena_malloc(arena, seqsize(B) * sizeof(HCFChoice *)); + HCFChoice **iBj = items; + for(; *B; B++, iBj++) { + size_t j = follow_transition(table, i, *B); + HLRTransition *i_B_j = transition(arena, i, *B, j); + *iBj = h_hashtable_get(eg->tmap, i_B_j); + assert(*iBj != NULL); + i = j; + } + *iBj = NULL; + + *q = h_arena_malloc(arena, sizeof(HCFSequence)); + (*q)->items = items; + } + *q = NULL; + xAy->seq = seq; } static bool eq_transition(const void *p, const void *q) { const HLRTransition *a=p, *b=q; - return (a->from == b->from && a->to == b->to && a->symbol == b->symbol); + return (a->from == b->from && a->to == b->to && eq_symbol(a->symbol, b->symbol)); } static HHashValue hash_transition(const void *p) { const HLRTransition *t = p; - return (h_hash_ptr(t->symbol) + t->from + t->to); // XXX ? + return (hash_symbol(t->symbol) + t->from + t->to); // XXX ? } -static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRTable *tbl) +HCFChoice *new_enhanced_symbol(HLREnhGrammar *eg, const HCFChoice *sym) +{ + HArena *arena = eg->arena; + HCFChoice *esym = h_arena_malloc(arena, sizeof(HCFChoice)); + *esym = *sym; + + HHashSet *cs = h_hashtable_get(eg->corr, sym); + if(!cs) { + cs = h_hashset_new(arena, eq_symbol, hash_symbol); + h_hashtable_put(eg->corr, sym, cs); + } + h_hashset_put(cs, esym); + + return esym; +} +static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRDFA *dfa, + const HLRTable *table) { HAllocator *mm__ = g->mm__; HArena *arena = g->arena; // XXX ? HLREnhGrammar *eg = h_arena_malloc(arena, sizeof(HLREnhGrammar)); eg->tmap = h_hashtable_new(arena, eq_transition, hash_transition); - eg->smap = h_hashtable_new(arena, eq_transition, hash_transition); + eg->smap = h_hashtable_new(arena, eq_symbol, hash_symbol); + eg->corr = h_hashtable_new(arena, eq_symbol, hash_symbol); eg->arena = arena; - // copy the start symbol over - HCFChoice *start = h_arena_malloc(arena, sizeof(HCFChoice)); - *start = *(g->start); + // establish mapping between transitions and symbols + for(HSlistNode *x=dfa->transitions->head; x; x=x->next) { + HLRTransition *t = x->elem; - transform_productions(tbl, eg, 0, start); + assert(!h_hashtable_present(eg->tmap, t)); + + HCFChoice *sym = new_enhanced_symbol(eg, t->symbol); + h_hashtable_put(eg->tmap, t, sym); + h_hashtable_put(eg->smap, sym, t); + } + + // transform the productions + H_FOREACH(eg->tmap, HLRTransition *t, HCFChoice *sym) + transform_productions(table, eg, t->from, sym); + H_END_FOREACH + + // add the start symbol + HCFChoice *start = new_enhanced_symbol(eg, g->start); + transform_productions(table, eg, 0, start); eg->grammar = h_cfgrammar_(mm__, start); return eg; @@ -638,7 +657,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) if(has_conflicts(table)) { HArena *arena = table->arena; - HLREnhGrammar *eg = enhance_grammar(g, table); + HLREnhGrammar *eg = enhance_grammar(g, dfa, table); if(eg == NULL) { // this should normally not happen h_cfgrammar_free(g); h_lrtable_free(table); @@ -664,10 +683,10 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) HLRAction *action = reduce_action(arena, item); // find all LR(0)-enhanced productions matching item - H_FOREACH(eg->smap, HCFChoice *lhs, HLRTransition *t) - if(t->symbol != item->lhs) - continue; - assert(lhs->type == HCF_CHOICE); // XXX could be CHARSET + HHashSet *lhss = h_hashtable_get(eg->corr, item->lhs); + assert(lhss != NULL); + H_FOREACH_KEY(lhss, HCFChoice *lhs) + assert(lhs->type == HCF_CHOICE); // XXX could be CHARSET? for(HCFSequence **p=lhs->seq; *p; p++) { HCFChoice **rhs = (*p)->items; @@ -952,26 +971,35 @@ void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, for(size_t i=0; inrows; i++) { for(unsigned int j=0; jforall[i] && h_hashtable_empty(table->rows[i])) { + if(table->forall[i]) { fputs(" - ", f); pprint_lraction(f, g, table->forall[i]); fputs(" -", f); - } else { - H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) - fputc(' ', f); // separator - h_pprint_symbol(f, g, symbol); - fputc(':', f); - if(table->forall[i]) { - fputc(action->type == HLR_SHIFT? 's' : 'r', f); - fputc('/', f); - fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); - } else { - pprint_lraction(f, g, action); - } - H_END_FOREACH + if(!h_hashtable_empty(table->rows[i])) + fputs(" !!", f); } + H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) + fputc(' ', f); // separator + h_pprint_symbol(f, g, symbol); + fputc(':', f); + if(table->forall[i]) { + fputc(action->type == HLR_SHIFT? 's' : 'r', f); + fputc('/', f); + fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); + } else { + pprint_lraction(f, g, action); + } + H_END_FOREACH fputc('\n', f); } + +#if 0 + fputs("inadeq=", f); + for(HSlistNode *x=table->inadeq->head; x; x=x->next) { + fprintf(f, "%lu ", (uintptr_t)x->elem); + } + fputc('\n', f); +#endif } @@ -993,13 +1021,12 @@ int test_lalr(void) E -> E '-' T | T T -> '(' E ')' - | N - N -> '0' -- also try [0-9] for the charset paths + | 'n' -- also try [0-9] for the charset paths */ - HParser *N = h_sequence(h_ch('n'), NULL); + HParser *n = h_ch('n'); HParser *E = h_indirect(); - HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), N, NULL); + HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), n, NULL); HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); h_bind_indirect(E, E_); HParser *p = h_sequence(E, NULL); @@ -1035,7 +1062,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"syya", 4); + HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 11); if(res) h_pprint(stdout, res->ast, 0, 2); else From 1918c97e1c464045eb3dc491d5632bceb2482ecf Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 17:36:30 +0200 Subject: [PATCH 30/39] fix a premature arena delete --- src/backends/lalr.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index e3a866b..c0be736 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -845,18 +845,22 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* } } - h_delete_arena(tarena); + // parsing was successful iff the start symbol is on top of the right stack + HParseResult *result = NULL; if(h_slist_pop(right) == table->start) { // next on the right stack is the start symbol's semantic value - HParsedToken *result = h_slist_pop(right); - assert(result != NULL); - return make_result(arena, result); + HParsedToken *tok = h_slist_pop(right); + assert(tok != NULL); + result = make_result(arena, tok); } else { h_delete_arena(arena); - return NULL; + result = NULL; } + + h_delete_arena(tarena); + return result; } @@ -922,7 +926,7 @@ void h_pprint_lrstate(FILE *f, const HCFGrammar *g, } } -void pprint_transition(FILE *f, const HCFGrammar *g, const HLRTransition *t) +static void pprint_transition(FILE *f, const HCFGrammar *g, const HLRTransition *t) { fputs("-", f); h_pprint_symbol(f, g, t->symbol); @@ -1062,7 +1066,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 11); + HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 13); if(res) h_pprint(stdout, res->ast, 0, 2); else From d51e13173acdf276ca0597b0f2a8b43be77ffed5 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 17:36:46 +0200 Subject: [PATCH 31/39] add LALR test suite --- src/t_parser.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/t_parser.c b/src/t_parser.c index fa19151..a0e4040 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -405,7 +405,7 @@ static void test_not(gconstpointer backend) { g_check_parse_ok(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", 3, "(u0x61 (u0x2b) u0x62)"); g_check_parse_ok(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", 4, "(u0x61 <2b.2b> u0x62)"); } -/* + static void test_leftrec(gconstpointer backend) { HParser *a_ = h_ch('a'); @@ -416,7 +416,7 @@ static void test_leftrec(gconstpointer backend) { g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)"); g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "((u0x61 u0x61) u0x61)"); } -*/ + static void test_rightrec(gconstpointer backend) { HParser *a_ = h_ch('a'); @@ -547,4 +547,42 @@ void register_parser_tests(void) { g_test_add_data_func("/core/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); g_test_add_data_func("/core/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); g_test_add_data_func("/core/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); + + g_test_add_data_func("/core/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); + g_test_add_data_func("/core/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); + g_test_add_data_func("/core/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); + g_test_add_data_func("/core/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); + g_test_add_data_func("/core/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); + g_test_add_data_func("/core/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); + g_test_add_data_func("/core/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); + g_test_add_data_func("/core/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); + g_test_add_data_func("/core/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); + g_test_add_data_func("/core/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); + g_test_add_data_func("/core/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); + g_test_add_data_func("/core/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); +#if 0 + g_test_add_data_func("/core/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); + g_test_add_data_func("/core/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); +#endif + g_test_add_data_func("/core/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); + g_test_add_data_func("/core/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); + g_test_add_data_func("/core/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); + g_test_add_data_func("/core/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); + g_test_add_data_func("/core/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); + g_test_add_data_func("/core/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); + g_test_add_data_func("/core/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); + g_test_add_data_func("/core/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); + g_test_add_data_func("/core/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); + g_test_add_data_func("/core/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); + g_test_add_data_func("/core/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); + g_test_add_data_func("/core/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); + g_test_add_data_func("/core/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); + g_test_add_data_func("/core/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); + g_test_add_data_func("/core/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); + g_test_add_data_func("/core/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); + g_test_add_data_func("/core/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); + g_test_add_data_func("/core/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); + g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); + g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); + g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); } From a7fe3824cda87f6f74b565abb53c9e1ad27889ef Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 14 Jun 2013 19:07:26 +0200 Subject: [PATCH 32/39] minor bugfixies --- src/backends/lalr.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index c0be736..fbdb554 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -215,6 +215,7 @@ static HLRItem *advance_mark(HArena *arena, const HLRItem *item) static HHashSet *closure(HCFGrammar *g, const HHashSet *items) { + HAllocator *mm__ = g->mm__; HArena *arena = g->arena; HHashSet *ret = h_lrstate_new(arena); HSlist *work = h_slist_new(arena); @@ -251,8 +252,9 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) } else { // HCF_CHARSET for(unsigned int i=0; i<256; i++) { if(charset_isset(sym->charset, i)) { - HCFChoice **rhs = h_arena_malloc(arena, 2 * sizeof(HCFChoice *)); - rhs[0] = h_arena_malloc(arena, sizeof(HCFChoice)); + // XXX allocatethese single-character symbols statically somewhere + HCFChoice **rhs = h_new(HCFChoice *, 2); + rhs[0] = h_new(HCFChoice, 1); rhs[0]->type = HCF_CHAR; rhs[0]->chr = i; rhs[1] = NULL; @@ -558,7 +560,7 @@ static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRDFA *dfa, const HLRTable *table) { HAllocator *mm__ = g->mm__; - HArena *arena = g->arena; // XXX ? + HArena *arena = g->arena; HLREnhGrammar *eg = h_arena_malloc(arena, sizeof(HLREnhGrammar)); eg->tmap = h_hashtable_new(arena, eq_transition, hash_transition); @@ -851,8 +853,8 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HParseResult *result = NULL; if(h_slist_pop(right) == table->start) { // next on the right stack is the start symbol's semantic value + assert(!h_slist_empty(right)); HParsedToken *tok = h_slist_pop(right); - assert(tok != NULL); result = make_result(arena, tok); } else { h_delete_arena(arena); From e56f05225522e517c2319054b0f11f4619e96fcf Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 15 Jun 2013 16:04:29 +0200 Subject: [PATCH 33/39] remove the extra +1 from h_arena_malloc --- src/allocator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allocator.c b/src/allocator.c index e345c87..80fa921 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -65,10 +65,10 @@ void* h_arena_malloc(HArena *arena, size_t size) { if (size <= arena->head->free) { // fast path.. void* ret = arena->head->rest + arena->head->used; - arena->used += size + 1; + arena->used += size; arena->wasted -= size; - arena->head->used += size + 1; - arena->head->free -= size + 1; + arena->head->used += size; + arena->head->free -= size; return ret; } else if (size > arena->block_size) { // We need a new, dedicated block for it, because it won't fit in a standard sized one. From 9a9631493129f08e97726608d8034c07312d72f4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 15 Jun 2013 19:06:10 +0200 Subject: [PATCH 34/39] refactor hashtable iterations to use H_FOREACH --- src/backends/lalr.c | 180 +++++++++++++++++--------------------------- 1 file changed, 68 insertions(+), 112 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index fbdb554..54f6dc8 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -61,6 +61,27 @@ typedef struct HLREnhGrammar_ { } HLREnhGrammar; +// XXX move to internal.h or something +// XXX replace other hashtable iterations with this +#define H_FOREACH_(HT) { \ + const HHashTable *ht__ = HT; \ + for(size_t i__=0; i__ < ht__->capacity; i__++) { \ + for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ + hte__; \ + hte__ = hte__->next) { \ + if(hte__->key == NULL) continue; + +#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ + const KEYVAR = hte__->key; + +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ + VALVAR = hte__->value; + +#define H_END_FOREACH \ + } \ + } \ + } + // compare symbols - terminals by value, others by pointer static bool eq_symbol(const void *p, const void *q) { @@ -117,15 +138,9 @@ static HHashValue hash_lalr_itemset(const void *p) { HHashValue hash = 0; - const HHashTable *ht = p; - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - hash += hash_lalr_item(hte->key); - } - } + H_FOREACH_KEY((const HHashSet *)p, HLRItem *item) + hash += hash_lalr_item(item); + H_END_FOREACH return hash; } @@ -179,27 +194,6 @@ void h_lrtable_free(HLRTable *table) h_free(table); } -// XXX replace other hashtable iterations with this -// XXX move to internal.h or something -#define H_FOREACH_(HT) { \ - const HHashTable *ht__ = HT; \ - for(size_t i__=0; i__ < ht__->capacity; i__++) { \ - for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ - hte__; \ - hte__ = hte__->next) { \ - if(hte__->key == NULL) continue; - -#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ - const KEYVAR = hte__->key; - -#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ - VALVAR = hte__->value; - -#define H_END_FOREACH \ - } \ - } \ - } - /* Constructing the characteristic automaton (handle recognizer) */ @@ -220,18 +214,11 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) HHashSet *ret = h_lrstate_new(arena); HSlist *work = h_slist_new(arena); - // iterate over items - initialize work list with them - const HHashTable *ht = items; - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - const HLRItem *item = hte->key; + // initialize work list with items + H_FOREACH_KEY(items, HLRItem *item) h_hashset_put(ret, item); h_slist_push(work, (void *)item); - } - } + H_END_FOREACH while(!h_slist_empty(work)) { const HLRItem *item = h_slist_pop(work); @@ -322,75 +309,52 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) HHashTable *neighbors = h_hashtable_new(arena, eq_symbol, hash_symbol); // iterate over closure and generate neighboring sets - const HHashTable *ht = closure(g, state); - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; + H_FOREACH_KEY(closure(g, state), HLRItem *item) + HCFChoice *sym = item->rhs[item->mark]; // symbol after mark - const HLRItem *item = hte->key; - HCFChoice *sym = item->rhs[item->mark]; // symbol after mark - - if(sym != NULL) { // mark was not at the end - // find or create prospective neighbor set - HLRState *neighbor = h_hashtable_get(neighbors, sym); - if(neighbor == NULL) { - neighbor = h_lrstate_new(arena); - h_hashtable_put(neighbors, sym, neighbor); - } - - // ...and add the advanced item to it - h_hashset_put(neighbor, advance_mark(arena, item)); + if(sym != NULL) { // mark was not at the end + // find or create prospective neighbor set + HLRState *neighbor = h_hashtable_get(neighbors, sym); + if(neighbor == NULL) { + neighbor = h_lrstate_new(arena); + h_hashtable_put(neighbors, sym, neighbor); } + + // ...and add the advanced item to it + h_hashset_put(neighbor, advance_mark(arena, item)); } - } + H_END_FOREACH // merge neighbor sets into the set of existing states - ht = neighbors; - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - const HCFChoice *symbol = hte->key; - HLRState *neighbor = hte->value; - - // look up existing state, allocate new if not found - size_t neighbor_idx; - if(!h_hashset_present(states, neighbor)) { - neighbor_idx = states->used; - h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); - h_slist_push(work, neighbor); - h_slist_push(work, (void *)(uintptr_t)neighbor_idx); - } else { - neighbor_idx = (uintptr_t)h_hashtable_get(states, neighbor); - } - - // add transition "state --symbol--> neighbor" - HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); - t->from = state_idx; - t->to = neighbor_idx; - t->symbol = symbol; - h_slist_push(transitions, t); + H_FOREACH(neighbors, HCFChoice *symbol, HLRState *neighbor) + // look up existing state, allocate new if not found + size_t neighbor_idx; + if(!h_hashset_present(states, neighbor)) { + neighbor_idx = states->used; + h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); + h_slist_push(work, neighbor); + h_slist_push(work, (void *)(uintptr_t)neighbor_idx); + } else { + neighbor_idx = (uintptr_t)h_hashtable_get(states, neighbor); } - } + + // add transition "state --symbol--> neighbor" + HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); + t->from = state_idx; + t->to = neighbor_idx; + t->symbol = symbol; + h_slist_push(transitions, t); + H_END_FOREACH } // end while(work) // fill DFA struct HLRDFA *dfa = h_arena_malloc(arena, sizeof(HLRDFA)); dfa->nstates = states->used; dfa->states = h_arena_malloc(arena, dfa->nstates*sizeof(HLRState *)); - for(size_t i=0; i < states->capacity; i++) { - for(HHashTableEntry *hte = &states->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - const HLRState *state = hte->key; - size_t idx = (uintptr_t)hte->value; - - dfa->states[idx] = state; - } - } + H_FOREACH(states, HLRState *state, void *v) + size_t idx = (uintptr_t)v; + dfa->states[idx] = state; + H_END_FOREACH dfa->transitions = transitions; return dfa; @@ -911,21 +875,13 @@ void h_pprint_lrstate(FILE *f, const HCFGrammar *g, const HLRState *state, unsigned int indent) { bool first = true; - const HHashTable *ht = state; - for(size_t i=0; i < ht->capacity; i++) { - for(HHashTableEntry *hte = &ht->contents[i]; hte; hte = hte->next) { - if(hte->key == NULL) - continue; - - const HLRItem *item = hte->key; - - if(!first) - for(unsigned int i=0; i Date: Sat, 15 Jun 2013 20:30:57 +0200 Subject: [PATCH 35/39] some refactoring trying to get h_many to work --- src/backends/lalr.c | 46 ++++++++++++++++++++++----------------------- src/cfgrammar.c | 6 ++++++ src/cfgrammar.h | 1 + 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 54f6dc8..0dd1d07 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -207,16 +207,14 @@ static HLRItem *advance_mark(HArena *arena, const HLRItem *item) return ret; } -static HHashSet *closure(HCFGrammar *g, const HHashSet *items) +static void expand_to_closure(HCFGrammar *g, HHashSet *items) { HAllocator *mm__ = g->mm__; HArena *arena = g->arena; - HHashSet *ret = h_lrstate_new(arena); HSlist *work = h_slist_new(arena); // initialize work list with items H_FOREACH_KEY(items, HLRItem *item) - h_hashset_put(ret, item); h_slist_push(work, (void *)item); H_END_FOREACH @@ -231,22 +229,22 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) if(sym->type == HCF_CHOICE) { for(HCFSequence **p=sym->seq; *p; p++) { HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); - if(!h_hashset_present(ret, it)) { - h_hashset_put(ret, it); + if(!h_hashset_present(items, it)) { + h_hashset_put(items, it); h_slist_push(work, it); } } } else { // HCF_CHARSET for(unsigned int i=0; i<256; i++) { if(charset_isset(sym->charset, i)) { - // XXX allocatethese single-character symbols statically somewhere + // XXX allocate these single-character symbols statically somewhere HCFChoice **rhs = h_new(HCFChoice *, 2); rhs[0] = h_new(HCFChoice, 1); rhs[0]->type = HCF_CHAR; rhs[0]->chr = i; rhs[1] = NULL; HLRItem *it = h_lritem_new(arena, sym, rhs, 0); - h_hashset_put(ret, it); + h_hashset_put(items, it); // single-character item needs no further work } } @@ -254,17 +252,8 @@ static HHashSet *closure(HCFGrammar *g, const HHashSet *items) // this seems as good a place as any to set it sym->reshape = h_act_first; } - - // if sym derives epsilon, also advance over it - if(h_derives_epsilon(g, sym)) { - HLRItem *it = advance_mark(arena, item); - h_hashset_put(ret, it); - h_slist_push(work, it); - } } } - - return ret; } HLRDFA *h_lr0_dfa(HCFGrammar *g) @@ -287,15 +276,16 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) assert(g->start->type == HCF_CHOICE); for(HCFSequence **p=g->start->seq; *p; p++) h_hashset_put(start, h_lritem_new(arena, g->start, (*p)->items, 0)); + expand_to_closure(g, start); h_hashtable_put(states, start, 0); h_slist_push(work, start); h_slist_push(work, 0); // while work to do (on some state) - // compute closure // determine edge symbols // for each edge symbol: // advance respective items -> destination state (kernel) + // compute closure // if destination is a new state: // add it to state set // add transition to it @@ -308,8 +298,8 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) // maps edge symbols to neighbor states (item sets) of s HHashTable *neighbors = h_hashtable_new(arena, eq_symbol, hash_symbol); - // iterate over closure and generate neighboring sets - H_FOREACH_KEY(closure(g, state), HLRItem *item) + // iterate over state (closure) and generate neighboring sets + H_FOREACH_KEY(state, HLRItem *item) HCFChoice *sym = item->rhs[item->mark]; // symbol after mark if(sym != NULL) { // mark was not at the end @@ -325,8 +315,10 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) } H_END_FOREACH - // merge neighbor sets into the set of existing states + // merge expanded neighbor sets into the set of existing states H_FOREACH(neighbors, HCFChoice *symbol, HLRState *neighbor) + expand_to_closure(g, neighbor); + // look up existing state, allocate new if not found size_t neighbor_idx; if(!h_hashset_present(states, neighbor)) { @@ -528,8 +520,9 @@ static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRDFA *dfa, HLREnhGrammar *eg = h_arena_malloc(arena, sizeof(HLREnhGrammar)); eg->tmap = h_hashtable_new(arena, eq_transition, hash_transition); - eg->smap = h_hashtable_new(arena, eq_symbol, hash_symbol); + eg->smap = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); eg->corr = h_hashtable_new(arena, eq_symbol, hash_symbol); + // XXX must use h_eq/hash_ptr for symbols! so enhanced CHARs are different eg->arena = arena; // establish mapping between transitions and symbols @@ -663,6 +656,8 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) // contribution to the lookahead const HStringMap *fs = h_follow(1, eg->grammar, lhs); assert(fs != NULL); + assert(fs->epsilon_branch == NULL); + assert(!h_stringmap_empty(fs)); // for each lookahead symbol, put action into table cell if(fs->end_branch) { @@ -734,6 +729,7 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* while(1) { // make sure there is input on the right stack if(h_slist_empty(right)) { + // XXX use statically-allocated terminal symbols HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); HParsedToken *v; @@ -841,7 +837,7 @@ void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item) HCFChoice **x = item->rhs; HCFChoice **mark = item->rhs + item->mark; if(*x == NULL) { - fputs("\"\"", f); + fputc('.', f); } else { while(*x) { if(x == mark) @@ -986,12 +982,14 @@ int test_lalr(void) | 'n' -- also try [0-9] for the charset paths */ +#if 0 HParser *n = h_ch('n'); HParser *E = h_indirect(); HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), n, NULL); HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); h_bind_indirect(E, E_); - HParser *p = h_sequence(E, NULL); +#endif + HParser *p = h_choice(h_many(h_ch('x')), h_ch('n'), NULL); //h_sequence(E, NULL); printf("\n==== G R A M M A R ====\n"); HCFGrammar *g = h_cfgrammar(&system_allocator, p); @@ -1024,7 +1022,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 13); + HParseResult *res = h_parse(p, (uint8_t *)"xxn-(n-((n)))-n", 13); if(res) h_pprint(stdout, res->ast, 0, 2); else diff --git a/src/cfgrammar.c b/src/cfgrammar.c index a874236..199ef5f 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -331,6 +331,12 @@ bool h_stringmap_present_epsilon(const HStringMap *m) return (m->epsilon_branch != NULL); } +bool h_stringmap_empty(const HStringMap *m) +{ + return (m->epsilon_branch == NULL + && m->end_branch == NULL + && h_hashtable_empty(m->char_branches)); +} const HStringMap *h_first(size_t k, HCFGrammar *g, const HCFChoice *x) { diff --git a/src/cfgrammar.h b/src/cfgrammar.h index c70c68a..57f6f68 100644 --- a/src/cfgrammar.h +++ b/src/cfgrammar.h @@ -49,6 +49,7 @@ void h_stringmap_replace(HStringMap *m, void *old, void *new); void *h_stringmap_get(const HStringMap *m, const uint8_t *str, size_t n, bool end); bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end); bool h_stringmap_present_epsilon(const HStringMap *m); +bool h_stringmap_empty(const HStringMap *m); static inline HStringMap *h_stringmap_get_char(const HStringMap *m, const uint8_t c) { return h_hashtable_get(m->char_branches, (void *)char_key(c)); } From f0cd2de55c4ddfaf1db47aa838c8b1562a3b6b77 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 15 Jun 2013 20:54:53 +0200 Subject: [PATCH 36/39] fix match_production for the empty-sequence case --- src/backends/lalr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 0dd1d07..773b457 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -579,15 +579,16 @@ int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) bool match_production(HLREnhGrammar *eg, HCFChoice **p, HCFChoice **rhs, size_t endstate) { - HLRTransition *t; + size_t state = endstate; // initialized to end in case of empty rhs for(; *p && *rhs; p++, rhs++) { - t = h_hashtable_get(eg->smap, *p); + HLRTransition *t = h_hashtable_get(eg->smap, *p); assert(t != NULL); if(!eq_symbol(t->symbol, *rhs)) return false; + state = t->to; } return (*p == *rhs // both NULL - && t->to == endstate); + && state == endstate); } int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) @@ -1022,7 +1023,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"xxn-(n-((n)))-n", 13); + HParseResult *res = h_parse(p, (uint8_t *)"xxn-(n-((n)))-n", 2); if(res) h_pprint(stdout, res->ast, 0, 2); else From 8d21c782e742971db9395af516edb48ce8988675 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 15 Jun 2013 21:24:27 +0200 Subject: [PATCH 37/39] fix eq/hash_lalr_item to compare symbols by value --- src/backends/lalr.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 773b457..e041489 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -108,12 +108,12 @@ static bool eq_lalr_item(const void *p, const void *q) { const HLRItem *a=p, *b=q; - if(a->lhs != b->lhs) return false; + if(!eq_symbol(a->lhs, b->lhs)) return false; if(a->mark != b->mark) return false; if(a->len != b->len) return false; for(size_t i=0; ilen; i++) - if(a->rhs[i] != b->rhs[i]) return false; + if(!eq_symbol(a->rhs[i], b->rhs[i])) return false; return true; } @@ -128,9 +128,14 @@ static inline bool eq_lalr_itemset(const void *p, const void *q) static inline HHashValue hash_lalr_item(const void *p) { const HLRItem *x = p; - return (h_hash_ptr(x->lhs) - + h_djbhash((uint8_t *)x->rhs, x->len*sizeof(HCFChoice *)) - + x->mark); // XXX is it okay to just add mark? + HHashValue hash = 0; + + hash += hash_symbol(x->lhs); + for(HCFChoice **p=x->rhs; *p; p++) + hash += hash_symbol(*p); + hash += x->mark; + + return hash; } // hash LALR item sets (DFA states) - hash the elements and sum @@ -215,7 +220,7 @@ static void expand_to_closure(HCFGrammar *g, HHashSet *items) // initialize work list with items H_FOREACH_KEY(items, HLRItem *item) - h_slist_push(work, (void *)item); + h_slist_push(work, (void *)item); H_END_FOREACH while(!h_slist_empty(work)) { @@ -990,7 +995,7 @@ int test_lalr(void) HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); h_bind_indirect(E, E_); #endif - HParser *p = h_choice(h_many(h_ch('x')), h_ch('n'), NULL); //h_sequence(E, NULL); + HParser *p = h_whitespace(h_ch('n')); //h_sequence(E, NULL); printf("\n==== G R A M M A R ====\n"); HCFGrammar *g = h_cfgrammar(&system_allocator, p); @@ -1023,7 +1028,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"xxn-(n-((n)))-n", 2); + HParseResult *res = h_parse(p, (uint8_t *)" n-(n-((n)))-n", 13); if(res) h_pprint(stdout, res->ast, 0, 2); else From 24c15f34cc263765b77820d9bb3662ed39f70a93 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sun, 16 Jun 2013 22:23:35 +0200 Subject: [PATCH 38/39] augment grammar to ensure start symbol never occurs on rhs --- src/backends/lalr.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index e041489..bcb1a02 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -2,6 +2,7 @@ #include "../internal.h" #include "../cfgrammar.h" #include "../parsers/parser_internal.h" +#include "contextfree.h" @@ -274,8 +275,6 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) // assigned index. HSlist *work = h_slist_new(arena); - // XXX augment grammar?! - // make initial state (kernel) HLRState *start = h_lrstate_new(arena); assert(g->start->type == HCF_CHOICE); @@ -596,14 +595,33 @@ bool match_production(HLREnhGrammar *eg, HCFChoice **p, && state == endstate); } +// desugar parser with a fresh start symbol +// this guarantees that the start symbol will not occur in any productions +static HCFChoice *augment(HAllocator *mm__, HParser *parser) +{ + HCFChoice *augmented = h_new(HCFChoice, 1); + + HCFStack *stk__ = h_cfstack_new(mm__); + stk__->prealloc = augmented; + HCFS_BEGIN_CHOICE() { + HCFS_BEGIN_SEQ() { + HCFS_DESUGAR(parser); + } HCFS_END_SEQ(); + HCFS_THIS_CHOICE->reshape = h_act_first; + } HCFS_END_CHOICE(); + h_cfstack_free(mm__, stk__); + + return augmented; +} + int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) { - // generate CFG from parser + // generate (augmented) CFG from parser // construct LR(0) DFA // build LR(0) table // if necessary, resolve conflicts "by conversion to SLR" - HCFGrammar *g = h_cfgrammar(mm__, parser); + HCFGrammar *g = h_cfgrammar_(mm__, augment(mm__, parser)); if(g == NULL) // backend not suitable (language not context-free) return -1; @@ -981,21 +999,18 @@ HParserBackendVTable h__lalr_backend_vtable = { int test_lalr(void) { /* - S -> E E -> E '-' T | T T -> '(' E ')' | 'n' -- also try [0-9] for the charset paths */ -#if 0 HParser *n = h_ch('n'); HParser *E = h_indirect(); HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), n, NULL); HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); h_bind_indirect(E, E_); -#endif - HParser *p = h_whitespace(h_ch('n')); //h_sequence(E, NULL); + HParser *p = E; printf("\n==== G R A M M A R ====\n"); HCFGrammar *g = h_cfgrammar(&system_allocator, p); @@ -1028,7 +1043,7 @@ int test_lalr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)" n-(n-((n)))-n", 13); + HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 13); if(res) h_pprint(stdout, res->ast, 0, 2); else From 67b6e9666c376ac1bcda6efac1ba21941533c018 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sun, 16 Jun 2013 22:29:53 +0200 Subject: [PATCH 39/39] fix a potentially uninitialized variable --- src/backends/lalr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index bcb1a02..7a1c04a 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -798,7 +798,7 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* value->seq = h_carray_new_sized(arena, len); // pull values off the left stack, rewinding state accordingly - HParsedToken *v; + HParsedToken *v = NULL; for(size_t i=0; i