From 7eff4b8d94a35e4d28360e1c15f9a2506276d52e Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 17 Jun 2013 19:11:18 +0200 Subject: [PATCH 01/27] factor out a struct HLREngine --- src/backends/lalr.c | 241 ++++++++++++++++++++++++++------------------ 1 file changed, 141 insertions(+), 100 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 7a1c04a..79e03c7 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -61,6 +61,16 @@ typedef struct HLREnhGrammar_ { HArena *arena; } HLREnhGrammar; +typedef struct HLREngine_ { + const HLRTable *table; + HSlist *left; // left stack; reductions happen here + HSlist *right; // right stack; input appears here + size_t state; + bool running; + HArena *arena; // will hold the results + HArena *tarena; // tmp, deleted after parse +} HLREngine; + // XXX move to internal.h or something // XXX replace other hashtable iterations with this @@ -733,6 +743,132 @@ h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) } } +HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) +{ + HLREngine *engine = h_arena_malloc(tarena, sizeof(HLREngine)); + + engine->table = table; + engine->left = h_slist_new(tarena); + engine->right = h_slist_new(tarena); + engine->state = 0; + engine->running = 1; + engine->arena = arena; + engine->tarena = tarena; + + return engine; +} + +void h_lrengine_step(HLREngine *engine, HInputStream *stream) +{ + // short-hand names + HSlist *left = engine->left; + HSlist *right = engine->right; + HArena *arena = engine->arena; + HArena *tarena = engine->tarena; + + // stack layout: + // on the left stack, we put pairs: (saved state, semantic value) + // on the right stack, we put pairs: (symbol, semantic value) + + // make sure there is input on the right stack + if(h_slist_empty(right)) { + // XXX use statically-allocated terminal symbols + HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); + HParsedToken *v; + + uint8_t c = h_read_bits(stream, 8, false); + + if(stream->overrun) { // end of input + x->type = HCF_END; + v = NULL; + } else { + x->type = HCF_CHAR; + x->chr = c; + v = h_arena_malloc(arena, sizeof(HParsedToken)); + v->token_type = TT_UINT; + v->uint = c; + } + + h_slist_push(right, v); + h_slist_push(right, x); + } + + // peek at input symbol on the right side + HCFChoice *symbol = right->head->elem; + + // table lookup + const HLRAction *action = h_lr_lookup(engine->table, engine->state, symbol); + if(action == NULL) { + // no handle recognizable in input, terminate + engine->running = false; + return; + } + + if(action->type == HLR_SHIFT) { + h_slist_push(left, (void *)(uintptr_t)engine->state); + h_slist_pop(right); // symbol (discard) + h_slist_push(left, h_slist_pop(right)); // semantic value + engine->state = action->nextstate; + } else { + assert(action->type == HLR_REDUCE); + size_t len = action->production.length; + HCFChoice *symbol = action->production.lhs; + + // semantic value of the reduction result + HParsedToken *value = h_arena_malloc(arena, sizeof(HParsedToken)); + value->token_type = TT_SEQUENCE; + value->seq = h_carray_new_sized(arena, len); + + // pull values off the left stack, rewinding state accordingly + HParsedToken *v = NULL; + for(size_t i=0; istate = (uintptr_t)h_slist_pop(left); + + // collect values in result sequence + value->seq->elements[len-1-i] = v; + value->seq->used++; + } + if(v) { + // result position equals position of left-most symbol + value->index = v->index; + value->bit_offset = v->bit_offset; + } else { + // XXX how to get the position in this case? + } + + // perform token reshape if indicated + if(symbol->reshape) + value = (HParsedToken *)symbol->reshape(make_result(arena, value)); + + // call validation and semantic action, if present + if(symbol->pred && !symbol->pred(make_result(tarena, value))) { + // validation failed -> no parse; terminate + engine->running = false; + return; + } + if(symbol->action) + value = (HParsedToken *)symbol->action(make_result(arena, value)); + + // push result (value, symbol) onto the right stack + h_slist_push(right, value); + h_slist_push(right, symbol); + } +} + +HParseResult *h_lrengine_result(HLREngine *engine) +{ + // parsing was successful iff the start symbol is on top of the right stack + if(h_slist_pop(engine->right) == engine->table->start) { + // next on the right stack is the start symbol's semantic value + assert(!h_slist_empty(engine->right)); + HParsedToken *tok = h_slist_pop(engine->right); + return make_result(engine->arena, tok); + } else { + return NULL; + } +} + HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) { HLRTable *table = parser->backend_data; @@ -741,110 +877,15 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - HSlist *left = h_slist_new(tarena); // left stack; reductions happen here - HSlist *right = h_slist_new(tarena); // right stack; input appears here - - // stack layout: - // on the left stack, we put pairs: (saved state, semantic value) - // on the right stack, we put pairs: (symbol, semantic value) + HLREngine *engine = h_lrengine_new(arena, tarena, table); // run while the recognizer finds handles in the input - size_t state = 0; - while(1) { - // make sure there is input on the right stack - if(h_slist_empty(right)) { - // XXX use statically-allocated terminal symbols - HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); - HParsedToken *v; + while(engine->running) + h_lrengine_step(engine, stream); - uint8_t c = h_read_bits(stream, 8, false); - - if(stream->overrun) { // end of input - x->type = HCF_END; - v = NULL; - } else { - x->type = HCF_CHAR; - x->chr = c; - v = h_arena_malloc(arena, sizeof(HParsedToken)); - v->token_type = TT_UINT; - v->uint = c; - } - - h_slist_push(right, v); - h_slist_push(right, x); - } - - // peek at input symbol on the right side - HCFChoice *symbol = right->head->elem; - - // table lookup - const HLRAction *action = h_lr_lookup(table, state, symbol); - if(action == NULL) - break; // no handle recognizable in input, terminate parsing - - if(action->type == HLR_SHIFT) { - h_slist_push(left, (void *)(uintptr_t)state); - h_slist_pop(right); // symbol (discard) - h_slist_push(left, h_slist_pop(right)); // semantic value - state = action->nextstate; - } else { - assert(action->type == HLR_REDUCE); - size_t len = action->production.length; - HCFChoice *symbol = action->production.lhs; - - // semantic value of the reduction result - HParsedToken *value = h_arena_malloc(arena, sizeof(HParsedToken)); - value->token_type = TT_SEQUENCE; - value->seq = h_carray_new_sized(arena, len); - - // pull values off the left stack, rewinding state accordingly - HParsedToken *v = NULL; - for(size_t i=0; iseq->elements[len-1-i] = v; - value->seq->used++; - } - if(v) { - // result position equals position of left-most symbol - value->index = v->index; - value->bit_offset = v->bit_offset; - } else { - // XXX how to get the position in this case? - } - - // perform token reshape if indicated - if(symbol->reshape) - value = (HParsedToken *)symbol->reshape(make_result(arena, value)); - - // call validation and semantic action, if present - if(symbol->pred && !symbol->pred(make_result(tarena, value))) - break; // validation failed -> no parse - if(symbol->action) - value = (HParsedToken *)symbol->action(make_result(arena, value)); - - // push result (value, symbol) onto the right stack - h_slist_push(right, value); - h_slist_push(right, symbol); - } - } - - - - // parsing was successful iff the start symbol is on top of the right stack - HParseResult *result = NULL; - if(h_slist_pop(right) == table->start) { - // next on the right stack is the start symbol's semantic value - assert(!h_slist_empty(right)); - HParsedToken *tok = h_slist_pop(right); - result = make_result(arena, tok); - } else { + HParseResult *result = h_lrengine_result(engine); + if(!result) h_delete_arena(arena); - result = NULL; - } - h_delete_arena(tarena); return result; } From 129d50c0ef1d7867ea8450c2df589bdf823bee6f Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 17 Jun 2013 20:08:25 +0200 Subject: [PATCH 02/27] split LR code into lr.c, lr0.c, and lalr.c --- src/Makefile | 2 + src/backends/lalr.c | 760 +------------------------------------------- src/backends/lr.c | 458 ++++++++++++++++++++++++++ src/backends/lr.h | 131 ++++++++ src/backends/lr0.c | 205 ++++++++++++ 5 files changed, 806 insertions(+), 750 deletions(-) create mode 100644 src/backends/lr.c create mode 100644 src/backends/lr.h create mode 100644 src/backends/lr0.c diff --git a/src/Makefile b/src/Makefile index 1a2bff3..380436a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -43,6 +43,8 @@ HAMMER_PARTS := \ benchmark.o \ cfgrammar.o \ glue.o \ + backends/lr.o \ + backends/lr0.o \ $(PARSERS:%=parsers/%.o) \ $(BACKENDS:%=backends/%.o) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 79e03c7..fa67e5a 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -1,437 +1,6 @@ #include -#include "../internal.h" -#include "../cfgrammar.h" -#include "../parsers/parser_internal.h" #include "contextfree.h" - - - -/* Data structures */ - -typedef HHashSet HLRState; // states are sets of LRItems - -typedef struct HLRDFA_ { - size_t nstates; - const HLRState **states; // array of size nstates - HSlist *transitions; -} HLRDFA; - -typedef struct HLRTransition_ { - size_t from; // index into 'states' array - const HCFChoice *symbol; - size_t to; // index into 'states' array -} HLRTransition; - -typedef struct HLRItem_ { - HCFChoice *lhs; - HCFChoice **rhs; // NULL-terminated - size_t len; // number of elements in rhs - size_t mark; -} HLRItem; - -typedef struct HLRAction_ { - enum {HLR_SHIFT, HLR_REDUCE} type; - union { - size_t nextstate; // used with SHIFT - struct { - HCFChoice *lhs; // symbol carrying semantic actions etc. - size_t length; // # of symbols in rhs -#ifndef NDEBUG - HCFChoice **rhs; // NB: the rhs symbols are not needed for the parse -#endif - } production; // used with REDUCE - }; -} HLRAction; - -typedef struct HLRTable_ { - size_t nrows; - HHashTable **rows; // map symbols to HLRActions - HLRAction **forall; // shortcut to set an action for an entire row - HCFChoice *start; // start symbol - HSlist *inadeq; // indices of any inadequate states - HArena *arena; - HAllocator *mm__; -} HLRTable; - -typedef struct HLREnhGrammar_ { - HCFGrammar *grammar; // enhanced grammar - HHashTable *tmap; // maps transitions to enhanced-grammar symbols - HHashTable *smap; // maps enhanced-grammar symbols to transitions - HHashTable *corr; // maps symbols to sets of corresponding e. symbols - HArena *arena; -} HLREnhGrammar; - -typedef struct HLREngine_ { - const HLRTable *table; - HSlist *left; // left stack; reductions happen here - HSlist *right; // right stack; input appears here - size_t state; - bool running; - HArena *arena; // will hold the results - HArena *tarena; // tmp, deleted after parse -} HLREngine; - - -// XXX move to internal.h or something -// XXX replace other hashtable iterations with this -#define H_FOREACH_(HT) { \ - const HHashTable *ht__ = HT; \ - for(size_t i__=0; i__ < ht__->capacity; i__++) { \ - for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ - hte__; \ - hte__ = hte__->next) { \ - if(hte__->key == NULL) continue; - -#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ - const KEYVAR = hte__->key; - -#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ - VALVAR = hte__->value; - -#define H_END_FOREACH \ - } \ - } \ - } - -// compare symbols - terminals by value, others by pointer -static bool eq_symbol(const void *p, const void *q) -{ - const HCFChoice *x=p, *y=q; - return (x==y - || (x->type==HCF_END && y->type==HCF_END) - || (x->type==HCF_CHAR && y->type==HCF_CHAR && x->chr==y->chr)); -} - -// hash symbols - terminals by value, others by pointer -static HHashValue hash_symbol(const void *p) -{ - const HCFChoice *x=p; - if(x->type == HCF_END) - return 0; - else if(x->type == HCF_CHAR) - return x->chr * 33; - else - return h_hash_ptr(p); -} - -// compare LALR items by value -static bool eq_lalr_item(const void *p, const void *q) -{ - const HLRItem *a=p, *b=q; - - if(!eq_symbol(a->lhs, b->lhs)) return false; - if(a->mark != b->mark) return false; - if(a->len != b->len) return false; - - for(size_t i=0; ilen; i++) - if(!eq_symbol(a->rhs[i], b->rhs[i])) return false; - - return true; -} - -// compare LALR item sets (DFA states) -static inline bool eq_lalr_itemset(const void *p, const void *q) -{ - return h_hashset_equal(p, q); -} - -// hash LALR items -static inline HHashValue hash_lalr_item(const void *p) -{ - const HLRItem *x = p; - HHashValue hash = 0; - - hash += hash_symbol(x->lhs); - for(HCFChoice **p=x->rhs; *p; p++) - hash += hash_symbol(*p); - hash += x->mark; - - return hash; -} - -// hash LALR item sets (DFA states) - hash the elements and sum -static HHashValue hash_lalr_itemset(const void *p) -{ - HHashValue hash = 0; - - H_FOREACH_KEY((const HHashSet *)p, HLRItem *item) - hash += hash_lalr_item(item); - H_END_FOREACH - - return hash; -} - -HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) -{ - HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); - - size_t len = 0; - for(HCFChoice **p=rhs; *p; p++) len++; - assert(mark <= len); - - ret->lhs = lhs; - ret->rhs = rhs; - ret->len = len; - ret->mark = mark; - - return ret; -} - -static inline HLRState *h_lrstate_new(HArena *arena) -{ - return h_hashset_new(arena, eq_lalr_item, hash_lalr_item); -} - -HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) -{ - HArena *arena = h_new_arena(mm__, 0); // default blocksize - assert(arena != NULL); - - HLRTable *ret = h_new(HLRTable, 1); - ret->nrows = nrows; - ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); - ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); - ret->inadeq = h_slist_new(arena); - ret->arena = arena; - ret->mm__ = mm__; - - for(size_t i=0; irows[i] = h_hashtable_new(arena, eq_symbol, hash_symbol); - ret->forall[i] = NULL; - } - - return ret; -} - -void h_lrtable_free(HLRTable *table) -{ - HAllocator *mm__ = table->mm__; - h_delete_arena(table->arena); - h_free(table); -} - - - -/* Constructing the characteristic automaton (handle recognizer) */ - -static HLRItem *advance_mark(HArena *arena, const HLRItem *item) -{ - assert(item->rhs[item->mark] != NULL); - HLRItem *ret = h_arena_malloc(arena, sizeof(HLRItem)); - *ret = *item; - ret->mark++; - return ret; -} - -static void expand_to_closure(HCFGrammar *g, HHashSet *items) -{ - HAllocator *mm__ = g->mm__; - HArena *arena = g->arena; - HSlist *work = h_slist_new(arena); - - // initialize work list with items - H_FOREACH_KEY(items, HLRItem *item) - h_slist_push(work, (void *)item); - H_END_FOREACH - - while(!h_slist_empty(work)) { - const HLRItem *item = h_slist_pop(work); - HCFChoice *sym = item->rhs[item->mark]; // symbol after mark - - // if there is a non-terminal after the mark, follow it - // NB: unlike LLk, we do consider HCF_CHARSET a non-terminal here - if(sym != NULL && (sym->type==HCF_CHOICE || sym->type==HCF_CHARSET)) { - // add items corresponding to the productions of sym - if(sym->type == HCF_CHOICE) { - for(HCFSequence **p=sym->seq; *p; p++) { - HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); - if(!h_hashset_present(items, it)) { - h_hashset_put(items, it); - h_slist_push(work, it); - } - } - } else { // HCF_CHARSET - for(unsigned int i=0; i<256; i++) { - if(charset_isset(sym->charset, i)) { - // XXX allocate these single-character symbols statically somewhere - HCFChoice **rhs = h_new(HCFChoice *, 2); - rhs[0] = h_new(HCFChoice, 1); - rhs[0]->type = HCF_CHAR; - rhs[0]->chr = i; - rhs[1] = NULL; - HLRItem *it = h_lritem_new(arena, sym, rhs, 0); - h_hashset_put(items, it); - // single-character item needs no further work - } - } - // if sym is a non-terminal, we need a reshape on it - // this seems as good a place as any to set it - sym->reshape = h_act_first; - } - } - } -} - -HLRDFA *h_lr0_dfa(HCFGrammar *g) -{ - HArena *arena = g->arena; - - HHashSet *states = h_hashset_new(arena, eq_lalr_itemset, hash_lalr_itemset); - // maps itemsets to assigned array indices - HSlist *transitions = h_slist_new(arena); - - // list of states that need to be processed - // to save lookups, we push two elements per state, the itemset and its - // assigned index. - HSlist *work = h_slist_new(arena); - - // make initial state (kernel) - HLRState *start = h_lrstate_new(arena); - assert(g->start->type == HCF_CHOICE); - for(HCFSequence **p=g->start->seq; *p; p++) - h_hashset_put(start, h_lritem_new(arena, g->start, (*p)->items, 0)); - expand_to_closure(g, start); - h_hashtable_put(states, start, 0); - h_slist_push(work, start); - h_slist_push(work, 0); - - // while work to do (on some state) - // determine edge symbols - // for each edge symbol: - // advance respective items -> destination state (kernel) - // compute closure - // if destination is a new state: - // add it to state set - // add transition to it - // add it to the work list - - while(!h_slist_empty(work)) { - size_t state_idx = (uintptr_t)h_slist_pop(work); - HLRState *state = h_slist_pop(work); - - // maps edge symbols to neighbor states (item sets) of s - HHashTable *neighbors = h_hashtable_new(arena, eq_symbol, hash_symbol); - - // iterate over state (closure) and generate neighboring sets - H_FOREACH_KEY(state, HLRItem *item) - HCFChoice *sym = item->rhs[item->mark]; // symbol after mark - - if(sym != NULL) { // mark was not at the end - // find or create prospective neighbor set - HLRState *neighbor = h_hashtable_get(neighbors, sym); - if(neighbor == NULL) { - neighbor = h_lrstate_new(arena); - h_hashtable_put(neighbors, sym, neighbor); - } - - // ...and add the advanced item to it - h_hashset_put(neighbor, advance_mark(arena, item)); - } - H_END_FOREACH - - // merge expanded neighbor sets into the set of existing states - H_FOREACH(neighbors, HCFChoice *symbol, HLRState *neighbor) - expand_to_closure(g, neighbor); - - // look up existing state, allocate new if not found - size_t neighbor_idx; - if(!h_hashset_present(states, neighbor)) { - neighbor_idx = states->used; - h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); - h_slist_push(work, neighbor); - h_slist_push(work, (void *)(uintptr_t)neighbor_idx); - } else { - neighbor_idx = (uintptr_t)h_hashtable_get(states, neighbor); - } - - // add transition "state --symbol--> neighbor" - HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); - t->from = state_idx; - t->to = neighbor_idx; - t->symbol = symbol; - h_slist_push(transitions, t); - H_END_FOREACH - } // end while(work) - - // fill DFA struct - HLRDFA *dfa = h_arena_malloc(arena, sizeof(HLRDFA)); - dfa->nstates = states->used; - dfa->states = h_arena_malloc(arena, dfa->nstates*sizeof(HLRState *)); - H_FOREACH(states, HLRState *state, void *v) - size_t idx = (uintptr_t)v; - dfa->states[idx] = state; - H_END_FOREACH - dfa->transitions = transitions; - - return dfa; -} - - - -/* LR(0) table generation */ - -static HLRAction *shift_action(HArena *arena, size_t nextstate) -{ - HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); - action->type = HLR_SHIFT; - action->nextstate = nextstate; - return action; -} - -static HLRAction *reduce_action(HArena *arena, const HLRItem *item) -{ - HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); - action->type = HLR_REDUCE; - action->production.lhs = item->lhs; - action->production.length = item->len; -#ifndef NDEBUG - action->production.rhs = item->rhs; -#endif - return action; -} - -HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) -{ - HAllocator *mm__ = g->mm__; - - HLRTable *table = h_lrtable_new(mm__, dfa->nstates); - HArena *arena = table->arena; - - // remember start symbol - table->start = g->start; - - // add shift entries - for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { - // for each transition x-A->y, add "shift, goto y" to table entry (x,A) - HLRTransition *t = x->elem; - - HLRAction *action = shift_action(arena, t->to); - h_hashtable_put(table->rows[t->from], t->symbol, action); - } - - // add reduce entries, record inadequate states - for(size_t i=0; instates; i++) { - // find reducible items in state - H_FOREACH_KEY(dfa->states[i], HLRItem *item) - if(item->mark == item->len) { // mark at the end - // check for conflicts - // XXX store more informative stuff in the inadeq records? - if(table->forall[i]) { - // reduce/reduce conflict with a previous item - h_slist_push(table->inadeq, (void *)(uintptr_t)i); - } else if(!h_hashtable_empty(table->rows[i])) { - // shift/reduce conflict with one of the row's entries - h_slist_push(table->inadeq, (void *)(uintptr_t)i); - } - - // set reduce action for the entire row - table->forall[i] = reduce_action(arena, item); - } - H_END_FOREACH - } - - return table; -} +#include "lr.h" @@ -499,19 +68,7 @@ static void transform_productions(const HLRTable *table, HLREnhGrammar *eg, xAy->seq = seq; } -static bool eq_transition(const void *p, const void *q) -{ - const HLRTransition *a=p, *b=q; - return (a->from == b->from && a->to == b->to && eq_symbol(a->symbol, b->symbol)); -} - -static HHashValue hash_transition(const void *p) -{ - const HLRTransition *t = p; - return (hash_symbol(t->symbol) + t->from + t->to); // XXX ? -} - -HCFChoice *new_enhanced_symbol(HLREnhGrammar *eg, const HCFChoice *sym) +static HCFChoice *new_enhanced_symbol(HLREnhGrammar *eg, const HCFChoice *sym) { HArena *arena = eg->arena; HCFChoice *esym = h_arena_malloc(arena, sizeof(HCFChoice)); @@ -519,13 +76,14 @@ HCFChoice *new_enhanced_symbol(HLREnhGrammar *eg, const HCFChoice *sym) HHashSet *cs = h_hashtable_get(eg->corr, sym); if(!cs) { - cs = h_hashset_new(arena, eq_symbol, hash_symbol); + cs = h_hashset_new(arena, h_eq_symbol, h_hash_symbol); h_hashtable_put(eg->corr, sym, cs); } h_hashset_put(cs, esym); return esym; } + static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRDFA *dfa, const HLRTable *table) { @@ -533,9 +91,9 @@ static HLREnhGrammar *enhance_grammar(const HCFGrammar *g, const HLRDFA *dfa, HArena *arena = g->arena; HLREnhGrammar *eg = h_arena_malloc(arena, sizeof(HLREnhGrammar)); - eg->tmap = h_hashtable_new(arena, eq_transition, hash_transition); + eg->tmap = h_hashtable_new(arena, h_eq_transition, h_hash_transition); eg->smap = h_hashtable_new(arena, h_eq_ptr, h_hash_ptr); - eg->corr = h_hashtable_new(arena, eq_symbol, hash_symbol); + eg->corr = h_hashtable_new(arena, h_eq_symbol, h_hash_symbol); // XXX must use h_eq/hash_ptr for symbols! so enhanced CHARs are different eg->arena = arena; @@ -590,14 +148,14 @@ int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) // check whether a sequence of enhanced-grammar symbols (p) matches the given // (original-grammar) production rhs and terminates in the given end state. -bool match_production(HLREnhGrammar *eg, HCFChoice **p, - HCFChoice **rhs, size_t endstate) +static bool match_production(HLREnhGrammar *eg, HCFChoice **p, + HCFChoice **rhs, size_t endstate) { size_t state = endstate; // initialized to end in case of empty rhs for(; *p && *rhs; p++, rhs++) { HLRTransition *t = h_hashtable_get(eg->smap, *p); assert(t != NULL); - if(!eq_symbol(t->symbol, *rhs)) + if(!h_eq_symbol(t->symbol, *rhs)) return false; state = t->to; } @@ -673,7 +231,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) continue; // action to place in the table cells indicated by lookahead - HLRAction *action = reduce_action(arena, item); + HLRAction *action = h_reduce_action(arena, item); // find all LR(0)-enhanced productions matching item HHashSet *lhss = h_hashtable_get(eg->corr, item->lhs); @@ -729,304 +287,6 @@ void h_lalr_free(HParser *parser) -/* LR driver */ - -const HLRAction * -h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) -{ - assert(state < table->nrows); - if(table->forall[state]) { - assert(h_hashtable_empty(table->rows[state])); // that would be a conflict - return table->forall[state]; - } else { - return h_hashtable_get(table->rows[state], symbol); - } -} - -HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) -{ - HLREngine *engine = h_arena_malloc(tarena, sizeof(HLREngine)); - - engine->table = table; - engine->left = h_slist_new(tarena); - engine->right = h_slist_new(tarena); - engine->state = 0; - engine->running = 1; - engine->arena = arena; - engine->tarena = tarena; - - return engine; -} - -void h_lrengine_step(HLREngine *engine, HInputStream *stream) -{ - // short-hand names - HSlist *left = engine->left; - HSlist *right = engine->right; - HArena *arena = engine->arena; - HArena *tarena = engine->tarena; - - // stack layout: - // on the left stack, we put pairs: (saved state, semantic value) - // on the right stack, we put pairs: (symbol, semantic value) - - // make sure there is input on the right stack - if(h_slist_empty(right)) { - // XXX use statically-allocated terminal symbols - HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); - HParsedToken *v; - - uint8_t c = h_read_bits(stream, 8, false); - - if(stream->overrun) { // end of input - x->type = HCF_END; - v = NULL; - } else { - x->type = HCF_CHAR; - x->chr = c; - v = h_arena_malloc(arena, sizeof(HParsedToken)); - v->token_type = TT_UINT; - v->uint = c; - } - - h_slist_push(right, v); - h_slist_push(right, x); - } - - // peek at input symbol on the right side - HCFChoice *symbol = right->head->elem; - - // table lookup - const HLRAction *action = h_lr_lookup(engine->table, engine->state, symbol); - if(action == NULL) { - // no handle recognizable in input, terminate - engine->running = false; - return; - } - - if(action->type == HLR_SHIFT) { - h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_pop(right); // symbol (discard) - h_slist_push(left, h_slist_pop(right)); // semantic value - engine->state = action->nextstate; - } else { - assert(action->type == HLR_REDUCE); - size_t len = action->production.length; - HCFChoice *symbol = action->production.lhs; - - // semantic value of the reduction result - HParsedToken *value = h_arena_malloc(arena, sizeof(HParsedToken)); - value->token_type = TT_SEQUENCE; - value->seq = h_carray_new_sized(arena, len); - - // pull values off the left stack, rewinding state accordingly - HParsedToken *v = NULL; - for(size_t i=0; istate = (uintptr_t)h_slist_pop(left); - - // collect values in result sequence - value->seq->elements[len-1-i] = v; - value->seq->used++; - } - if(v) { - // result position equals position of left-most symbol - value->index = v->index; - value->bit_offset = v->bit_offset; - } else { - // XXX how to get the position in this case? - } - - // perform token reshape if indicated - if(symbol->reshape) - value = (HParsedToken *)symbol->reshape(make_result(arena, value)); - - // call validation and semantic action, if present - if(symbol->pred && !symbol->pred(make_result(tarena, value))) { - // validation failed -> no parse; terminate - engine->running = false; - return; - } - if(symbol->action) - value = (HParsedToken *)symbol->action(make_result(arena, value)); - - // push result (value, symbol) onto the right stack - h_slist_push(right, value); - h_slist_push(right, symbol); - } -} - -HParseResult *h_lrengine_result(HLREngine *engine) -{ - // parsing was successful iff the start symbol is on top of the right stack - if(h_slist_pop(engine->right) == engine->table->start) { - // next on the right stack is the start symbol's semantic value - assert(!h_slist_empty(engine->right)); - HParsedToken *tok = h_slist_pop(engine->right); - return make_result(engine->arena, tok); - } else { - return NULL; - } -} - -HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) -{ - HLRTable *table = parser->backend_data; - if(!table) - return NULL; - - HArena *arena = h_new_arena(mm__, 0); // will hold the results - HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - HLREngine *engine = h_lrengine_new(arena, tarena, table); - - // run while the recognizer finds handles in the input - while(engine->running) - h_lrengine_step(engine, stream); - - HParseResult *result = h_lrengine_result(engine); - if(!result) - h_delete_arena(arena); - h_delete_arena(tarena); - return result; -} - - - -/* Pretty-printers */ - -void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item) -{ - h_pprint_symbol(f, g, item->lhs); - fputs(" ->", f); - - HCFChoice **x = item->rhs; - HCFChoice **mark = item->rhs + item->mark; - if(*x == NULL) { - fputc('.', f); - } else { - while(*x) { - if(x == mark) - fputc('.', f); - else - fputc(' ', f); - - if((*x)->type == HCF_CHAR) { - // condense character strings - fputc('"', f); - h_pprint_char(f, (*x)->chr); - for(x++; *x; x++) { - if(x == mark) - break; - if((*x)->type != HCF_CHAR) - break; - h_pprint_char(f, (*x)->chr); - } - fputc('"', f); - } else { - h_pprint_symbol(f, g, *x); - x++; - } - } - if(x == mark) - fputs(".", f); - } -} - -void h_pprint_lrstate(FILE *f, const HCFGrammar *g, - const HLRState *state, unsigned int indent) -{ - bool first = true; - H_FOREACH_KEY(state, HLRItem *item) - if(!first) - for(unsigned int i=0; isymbol); - fprintf(f, "->%lu", t->to); -} - -void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, - const HLRDFA *dfa, unsigned int indent) -{ - for(size_t i=0; instates; i++) { - unsigned int indent2 = indent + fprintf(f, "%4lu: ", i); - h_pprint_lrstate(f, g, dfa->states[i], indent2); - for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { - const HLRTransition *t = x->elem; - if(t->from == i) { - for(unsigned int i=0; itype == HLR_SHIFT) { - fprintf(f, "s%lu", action->nextstate); - } else { - fputs("r(", f); - h_pprint_symbol(f, g, action->production.lhs); - fputs(" -> ", f); -#ifdef NDEBUG - // if we can't print the production, at least print its length - fprintf(f, "[%lu]", action->production.length); -#else - HCFSequence seq = {action->production.rhs}; - h_pprint_sequence(f, g, &seq); -#endif - fputc(')', f); - } -} - -void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, - unsigned int indent) -{ - for(size_t i=0; inrows; i++) { - for(unsigned int j=0; jforall[i]) { - fputs(" - ", f); - pprint_lraction(f, g, table->forall[i]); - fputs(" -", f); - if(!h_hashtable_empty(table->rows[i])) - fputs(" !!", f); - } - H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) - fputc(' ', f); // separator - h_pprint_symbol(f, g, symbol); - fputc(':', f); - if(table->forall[i]) { - fputc(action->type == HLR_SHIFT? 's' : 'r', f); - fputc('/', f); - fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); - } else { - pprint_lraction(f, g, action); - } - H_END_FOREACH - fputc('\n', f); - } - -#if 0 - fputs("inadeq=", f); - for(HSlistNode *x=table->inadeq->head; x; x=x->next) { - fprintf(f, "%lu ", (uintptr_t)x->elem); - } - fputc('\n', f); -#endif -} - - - HParserBackendVTable h__lalr_backend_vtable = { .compile = h_lalr_compile, .parse = h_lr_parse, diff --git a/src/backends/lr.c b/src/backends/lr.c new file mode 100644 index 0000000..c3062d5 --- /dev/null +++ b/src/backends/lr.c @@ -0,0 +1,458 @@ +#include +#include "../parsers/parser_internal.h" +#include "lr.h" + + + +/* Comparison and hashing functions */ + +// compare symbols - terminals by value, others by pointer +bool h_eq_symbol(const void *p, const void *q) +{ + const HCFChoice *x=p, *y=q; + return (x==y + || (x->type==HCF_END && y->type==HCF_END) + || (x->type==HCF_CHAR && y->type==HCF_CHAR && x->chr==y->chr)); +} + +// hash symbols - terminals by value, others by pointer +HHashValue h_hash_symbol(const void *p) +{ + const HCFChoice *x=p; + if(x->type == HCF_END) + return 0; + else if(x->type == HCF_CHAR) + return x->chr * 33; + else + return h_hash_ptr(p); +} + +// compare LR items by value +static bool eq_lr_item(const void *p, const void *q) +{ + const HLRItem *a=p, *b=q; + + if(!h_eq_symbol(a->lhs, b->lhs)) return false; + if(a->mark != b->mark) return false; + if(a->len != b->len) return false; + + for(size_t i=0; ilen; i++) + if(!h_eq_symbol(a->rhs[i], b->rhs[i])) return false; + + return true; +} + +// hash LALR items +static inline HHashValue hash_lr_item(const void *p) +{ + const HLRItem *x = p; + HHashValue hash = 0; + + hash += h_hash_symbol(x->lhs); + for(HCFChoice **p=x->rhs; *p; p++) + hash += h_hash_symbol(*p); + hash += x->mark; + + return hash; +} + +// compare item sets (DFA states) +bool h_eq_lr_itemset(const void *p, const void *q) +{ + return h_hashset_equal(p, q); +} + +// hash LR item sets (DFA states) - hash the elements and sum +HHashValue h_hash_lr_itemset(const void *p) +{ + HHashValue hash = 0; + + H_FOREACH_KEY((const HHashSet *)p, HLRItem *item) + hash += hash_lr_item(item); + H_END_FOREACH + + return hash; +} + +bool h_eq_transition(const void *p, const void *q) +{ + const HLRTransition *a=p, *b=q; + return (a->from == b->from && a->to == b->to && h_eq_symbol(a->symbol, b->symbol)); +} + +HHashValue h_hash_transition(const void *p) +{ + const HLRTransition *t = p; + return (h_hash_symbol(t->symbol) + t->from + t->to); // XXX ? +} + + + +/* Constructors */ + +HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark) +{ + HLRItem *ret = h_arena_malloc(a, sizeof(HLRItem)); + + size_t len = 0; + for(HCFChoice **p=rhs; *p; p++) len++; + assert(mark <= len); + + ret->lhs = lhs; + ret->rhs = rhs; + ret->len = len; + ret->mark = mark; + + return ret; +} + +HLRState *h_lrstate_new(HArena *arena) +{ + return h_hashset_new(arena, eq_lr_item, hash_lr_item); +} + +HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) +{ + HArena *arena = h_new_arena(mm__, 0); // default blocksize + assert(arena != NULL); + + HLRTable *ret = h_new(HLRTable, 1); + ret->nrows = nrows; + ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); + ret->inadeq = h_slist_new(arena); + ret->arena = arena; + ret->mm__ = mm__; + + for(size_t i=0; irows[i] = h_hashtable_new(arena, h_eq_symbol, h_hash_symbol); + ret->forall[i] = NULL; + } + + return ret; +} + +void h_lrtable_free(HLRTable *table) +{ + HAllocator *mm__ = table->mm__; + h_delete_arena(table->arena); + h_free(table); +} + +HLRAction *h_shift_action(HArena *arena, size_t nextstate) +{ + HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); + action->type = HLR_SHIFT; + action->nextstate = nextstate; + return action; +} + +HLRAction *h_reduce_action(HArena *arena, const HLRItem *item) +{ + HLRAction *action = h_arena_malloc(arena, sizeof(HLRAction)); + action->type = HLR_REDUCE; + action->production.lhs = item->lhs; + action->production.length = item->len; +#ifndef NDEBUG + action->production.rhs = item->rhs; +#endif + return action; +} + + + +/* LR driver */ + +const HLRAction * +h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) +{ + assert(state < table->nrows); + if(table->forall[state]) { + assert(h_hashtable_empty(table->rows[state])); // that would be a conflict + return table->forall[state]; + } else { + return h_hashtable_get(table->rows[state], symbol); + } +} + +HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) +{ + HLREngine *engine = h_arena_malloc(tarena, sizeof(HLREngine)); + + engine->table = table; + engine->left = h_slist_new(tarena); + engine->right = h_slist_new(tarena); + engine->state = 0; + engine->running = 1; + engine->arena = arena; + engine->tarena = tarena; + + return engine; +} + +void h_lrengine_step(HLREngine *engine, HInputStream *stream) +{ + // short-hand names + HSlist *left = engine->left; + HSlist *right = engine->right; + HArena *arena = engine->arena; + HArena *tarena = engine->tarena; + + // stack layout: + // on the left stack, we put pairs: (saved state, semantic value) + // on the right stack, we put pairs: (symbol, semantic value) + + // make sure there is input on the right stack + if(h_slist_empty(right)) { + // XXX use statically-allocated terminal symbols + HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); + HParsedToken *v; + + uint8_t c = h_read_bits(stream, 8, false); + + if(stream->overrun) { // end of input + x->type = HCF_END; + v = NULL; + } else { + x->type = HCF_CHAR; + x->chr = c; + v = h_arena_malloc(arena, sizeof(HParsedToken)); + v->token_type = TT_UINT; + v->uint = c; + } + + h_slist_push(right, v); + h_slist_push(right, x); + } + + // peek at input symbol on the right side + HCFChoice *symbol = right->head->elem; + + // table lookup + const HLRAction *action = h_lr_lookup(engine->table, engine->state, symbol); + if(action == NULL) { + // no handle recognizable in input, terminate + engine->running = false; + return; + } + + if(action->type == HLR_SHIFT) { + h_slist_push(left, (void *)(uintptr_t)engine->state); + h_slist_pop(right); // symbol (discard) + h_slist_push(left, h_slist_pop(right)); // semantic value + engine->state = action->nextstate; + } else { + assert(action->type == HLR_REDUCE); + size_t len = action->production.length; + HCFChoice *symbol = action->production.lhs; + + // semantic value of the reduction result + HParsedToken *value = h_arena_malloc(arena, sizeof(HParsedToken)); + value->token_type = TT_SEQUENCE; + value->seq = h_carray_new_sized(arena, len); + + // pull values off the left stack, rewinding state accordingly + HParsedToken *v = NULL; + for(size_t i=0; istate = (uintptr_t)h_slist_pop(left); + + // collect values in result sequence + value->seq->elements[len-1-i] = v; + value->seq->used++; + } + if(v) { + // result position equals position of left-most symbol + value->index = v->index; + value->bit_offset = v->bit_offset; + } else { + // XXX how to get the position in this case? + } + + // perform token reshape if indicated + if(symbol->reshape) + value = (HParsedToken *)symbol->reshape(make_result(arena, value)); + + // call validation and semantic action, if present + if(symbol->pred && !symbol->pred(make_result(tarena, value))) { + // validation failed -> no parse; terminate + engine->running = false; + return; + } + if(symbol->action) + value = (HParsedToken *)symbol->action(make_result(arena, value)); + + // push result (value, symbol) onto the right stack + h_slist_push(right, value); + h_slist_push(right, symbol); + } +} + +HParseResult *h_lrengine_result(HLREngine *engine) +{ + // parsing was successful iff the start symbol is on top of the right stack + if(h_slist_pop(engine->right) == engine->table->start) { + // next on the right stack is the start symbol's semantic value + assert(!h_slist_empty(engine->right)); + HParsedToken *tok = h_slist_pop(engine->right); + return make_result(engine->arena, tok); + } else { + return NULL; + } +} + +HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) +{ + HLRTable *table = parser->backend_data; + if(!table) + return NULL; + + HArena *arena = h_new_arena(mm__, 0); // will hold the results + HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse + HLREngine *engine = h_lrengine_new(arena, tarena, table); + + // run while the recognizer finds handles in the input + while(engine->running) + h_lrengine_step(engine, stream); + + HParseResult *result = h_lrengine_result(engine); + if(!result) + h_delete_arena(arena); + h_delete_arena(tarena); + return result; +} + + + +/* Pretty-printers */ + +void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item) +{ + h_pprint_symbol(f, g, item->lhs); + fputs(" ->", f); + + HCFChoice **x = item->rhs; + HCFChoice **mark = item->rhs + item->mark; + if(*x == NULL) { + fputc('.', f); + } else { + while(*x) { + if(x == mark) + fputc('.', f); + else + fputc(' ', f); + + if((*x)->type == HCF_CHAR) { + // condense character strings + fputc('"', f); + h_pprint_char(f, (*x)->chr); + for(x++; *x; x++) { + if(x == mark) + break; + if((*x)->type != HCF_CHAR) + break; + h_pprint_char(f, (*x)->chr); + } + fputc('"', f); + } else { + h_pprint_symbol(f, g, *x); + x++; + } + } + if(x == mark) + fputs(".", f); + } +} + +void h_pprint_lrstate(FILE *f, const HCFGrammar *g, + const HLRState *state, unsigned int indent) +{ + bool first = true; + H_FOREACH_KEY(state, HLRItem *item) + if(!first) + for(unsigned int i=0; isymbol); + fprintf(f, "->%lu", t->to); +} + +void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, + const HLRDFA *dfa, unsigned int indent) +{ + for(size_t i=0; instates; i++) { + unsigned int indent2 = indent + fprintf(f, "%4lu: ", i); + h_pprint_lrstate(f, g, dfa->states[i], indent2); + for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { + const HLRTransition *t = x->elem; + if(t->from == i) { + for(unsigned int i=0; itype == HLR_SHIFT) { + fprintf(f, "s%lu", action->nextstate); + } else { + fputs("r(", f); + h_pprint_symbol(f, g, action->production.lhs); + fputs(" -> ", f); +#ifdef NDEBUG + // if we can't print the production, at least print its length + fprintf(f, "[%lu]", action->production.length); +#else + HCFSequence seq = {action->production.rhs}; + h_pprint_sequence(f, g, &seq); +#endif + fputc(')', f); + } +} + +void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, + unsigned int indent) +{ + for(size_t i=0; inrows; i++) { + for(unsigned int j=0; jforall[i]) { + fputs(" - ", f); + pprint_lraction(f, g, table->forall[i]); + fputs(" -", f); + if(!h_hashtable_empty(table->rows[i])) + fputs(" !!", f); + } + H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) + fputc(' ', f); // separator + h_pprint_symbol(f, g, symbol); + fputc(':', f); + if(table->forall[i]) { + fputc(action->type == HLR_SHIFT? 's' : 'r', f); + fputc('/', f); + fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); + } else { + pprint_lraction(f, g, action); + } + H_END_FOREACH + fputc('\n', f); + } + +#if 0 + fputs("inadeq=", f); + for(HSlistNode *x=table->inadeq->head; x; x=x->next) { + fprintf(f, "%lu ", (uintptr_t)x->elem); + } + fputc('\n', f); +#endif +} diff --git a/src/backends/lr.h b/src/backends/lr.h new file mode 100644 index 0000000..afd4042 --- /dev/null +++ b/src/backends/lr.h @@ -0,0 +1,131 @@ +#ifndef HAMMER_BACKENDS_LR__H +#define HAMMER_BACKENDS_LR__H + +#include "../hammer.h" +#include "../cfgrammar.h" +#include "../internal.h" + + +typedef HHashSet HLRState; // states are sets of LRItems + +typedef struct HLRDFA_ { + size_t nstates; + const HLRState **states; // array of size nstates + HSlist *transitions; +} HLRDFA; + +typedef struct HLRTransition_ { + size_t from; // index into 'states' array + const HCFChoice *symbol; + size_t to; // index into 'states' array +} HLRTransition; + +typedef struct HLRItem_ { + HCFChoice *lhs; + HCFChoice **rhs; // NULL-terminated + size_t len; // number of elements in rhs + size_t mark; +} HLRItem; + +typedef struct HLRAction_ { + enum {HLR_SHIFT, HLR_REDUCE} type; + union { + size_t nextstate; // used with SHIFT + struct { + HCFChoice *lhs; // symbol carrying semantic actions etc. + size_t length; // # of symbols in rhs +#ifndef NDEBUG + HCFChoice **rhs; // NB: the rhs symbols are not needed for the parse +#endif + } production; // used with REDUCE + }; +} HLRAction; + +typedef struct HLRTable_ { + size_t nrows; + HHashTable **rows; // map symbols to HLRActions + HLRAction **forall; // shortcut to set an action for an entire row + HCFChoice *start; // start symbol + HSlist *inadeq; // indices of any inadequate states + HArena *arena; + HAllocator *mm__; +} HLRTable; + +typedef struct HLREnhGrammar_ { + HCFGrammar *grammar; // enhanced grammar + HHashTable *tmap; // maps transitions to enhanced-grammar symbols + HHashTable *smap; // maps enhanced-grammar symbols to transitions + HHashTable *corr; // maps symbols to sets of corresponding e. symbols + HArena *arena; +} HLREnhGrammar; + +typedef struct HLREngine_ { + const HLRTable *table; + HSlist *left; // left stack; reductions happen here + HSlist *right; // right stack; input appears here + size_t state; + bool running; + HArena *arena; // will hold the results + HArena *tarena; // tmp, deleted after parse +} HLREngine; + + +// XXX move to internal.h or something +// XXX replace other hashtable iterations with this +#define H_FOREACH_(HT) { \ + const HHashTable *ht__ = HT; \ + for(size_t i__=0; i__ < ht__->capacity; i__++) { \ + for(HHashTableEntry *hte__ = &ht__->contents[i__]; \ + hte__; \ + hte__ = hte__->next) { \ + if(hte__->key == NULL) continue; + +#define H_FOREACH_KEY(HT, KEYVAR) H_FOREACH_(HT) \ + const KEYVAR = hte__->key; + +#define H_FOREACH(HT, KEYVAR, VALVAR) H_FOREACH_KEY(HT, KEYVAR) \ + VALVAR = hte__->value; + +#define H_END_FOREACH \ + } \ + } \ + } + + + +HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark); +HLRState *h_lrstate_new(HArena *arena); +HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows); +void h_lrtable_free(HLRTable *table); +HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table); +HLRAction *h_reduce_action(HArena *arena, const HLRItem *item); +HLRAction *h_shift_action(HArena *arena, size_t nextstate); + +bool h_eq_symbol(const void *p, const void *q); +bool h_eq_lr_itemset(const void *p, const void *q); +bool h_eq_transition(const void *p, const void *q); +HHashValue h_hash_symbol(const void *p); +HHashValue h_hash_lr_itemset(const void *p); +HHashValue h_hash_transition(const void *p); + +HLRDFA *h_lr0_dfa(HCFGrammar *g); +HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa); +int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action); + +int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); +void h_lalr_free(HParser *parser); + +const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); +void h_lrengine_step(HLREngine *engine, HInputStream *stream); +HParseResult *h_lrengine_result(HLREngine *engine); +HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); + +void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item); +void h_pprint_lrstate(FILE *f, const HCFGrammar *g, + const HLRState *state, unsigned int indent); +void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, + const HLRDFA *dfa, unsigned int indent); +void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, + unsigned int indent); + +#endif diff --git a/src/backends/lr0.c b/src/backends/lr0.c new file mode 100644 index 0000000..aab2ad1 --- /dev/null +++ b/src/backends/lr0.c @@ -0,0 +1,205 @@ +#include +#include "lr.h" + + + +/* Constructing the characteristic automaton (handle recognizer) */ + +static HLRItem *advance_mark(HArena *arena, const HLRItem *item) +{ + assert(item->rhs[item->mark] != NULL); + HLRItem *ret = h_arena_malloc(arena, sizeof(HLRItem)); + *ret = *item; + ret->mark++; + return ret; +} + +static void expand_to_closure(HCFGrammar *g, HHashSet *items) +{ + HAllocator *mm__ = g->mm__; + HArena *arena = g->arena; + HSlist *work = h_slist_new(arena); + + // initialize work list with items + H_FOREACH_KEY(items, HLRItem *item) + h_slist_push(work, (void *)item); + H_END_FOREACH + + while(!h_slist_empty(work)) { + const HLRItem *item = h_slist_pop(work); + HCFChoice *sym = item->rhs[item->mark]; // symbol after mark + + // if there is a non-terminal after the mark, follow it + // NB: unlike LLk, we do consider HCF_CHARSET a non-terminal here + if(sym != NULL && (sym->type==HCF_CHOICE || sym->type==HCF_CHARSET)) { + // add items corresponding to the productions of sym + if(sym->type == HCF_CHOICE) { + for(HCFSequence **p=sym->seq; *p; p++) { + HLRItem *it = h_lritem_new(arena, sym, (*p)->items, 0); + if(!h_hashset_present(items, it)) { + h_hashset_put(items, it); + h_slist_push(work, it); + } + } + } else { // HCF_CHARSET + for(unsigned int i=0; i<256; i++) { + if(charset_isset(sym->charset, i)) { + // XXX allocate these single-character symbols statically somewhere + HCFChoice **rhs = h_new(HCFChoice *, 2); + rhs[0] = h_new(HCFChoice, 1); + rhs[0]->type = HCF_CHAR; + rhs[0]->chr = i; + rhs[1] = NULL; + HLRItem *it = h_lritem_new(arena, sym, rhs, 0); + h_hashset_put(items, it); + // single-character item needs no further work + } + } + // if sym is a non-terminal, we need a reshape on it + // this seems as good a place as any to set it + sym->reshape = h_act_first; + } + } + } +} + +HLRDFA *h_lr0_dfa(HCFGrammar *g) +{ + HArena *arena = g->arena; + + HHashSet *states = h_hashset_new(arena, h_eq_lr_itemset, h_hash_lr_itemset); + // maps itemsets to assigned array indices + HSlist *transitions = h_slist_new(arena); + + // list of states that need to be processed + // to save lookups, we push two elements per state, the itemset and its + // assigned index. + HSlist *work = h_slist_new(arena); + + // make initial state (kernel) + HLRState *start = h_lrstate_new(arena); + assert(g->start->type == HCF_CHOICE); + for(HCFSequence **p=g->start->seq; *p; p++) + h_hashset_put(start, h_lritem_new(arena, g->start, (*p)->items, 0)); + expand_to_closure(g, start); + h_hashtable_put(states, start, 0); + h_slist_push(work, start); + h_slist_push(work, 0); + + // while work to do (on some state) + // determine edge symbols + // for each edge symbol: + // advance respective items -> destination state (kernel) + // compute closure + // if destination is a new state: + // add it to state set + // add transition to it + // add it to the work list + + while(!h_slist_empty(work)) { + size_t state_idx = (uintptr_t)h_slist_pop(work); + HLRState *state = h_slist_pop(work); + + // maps edge symbols to neighbor states (item sets) of s + HHashTable *neighbors = h_hashtable_new(arena, h_eq_symbol, h_hash_symbol); + + // iterate over state (closure) and generate neighboring sets + H_FOREACH_KEY(state, HLRItem *item) + HCFChoice *sym = item->rhs[item->mark]; // symbol after mark + + if(sym != NULL) { // mark was not at the end + // find or create prospective neighbor set + HLRState *neighbor = h_hashtable_get(neighbors, sym); + if(neighbor == NULL) { + neighbor = h_lrstate_new(arena); + h_hashtable_put(neighbors, sym, neighbor); + } + + // ...and add the advanced item to it + h_hashset_put(neighbor, advance_mark(arena, item)); + } + H_END_FOREACH + + // merge expanded neighbor sets into the set of existing states + H_FOREACH(neighbors, HCFChoice *symbol, HLRState *neighbor) + expand_to_closure(g, neighbor); + + // look up existing state, allocate new if not found + size_t neighbor_idx; + if(!h_hashset_present(states, neighbor)) { + neighbor_idx = states->used; + h_hashtable_put(states, neighbor, (void *)(uintptr_t)neighbor_idx); + h_slist_push(work, neighbor); + h_slist_push(work, (void *)(uintptr_t)neighbor_idx); + } else { + neighbor_idx = (uintptr_t)h_hashtable_get(states, neighbor); + } + + // add transition "state --symbol--> neighbor" + HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition)); + t->from = state_idx; + t->to = neighbor_idx; + t->symbol = symbol; + h_slist_push(transitions, t); + H_END_FOREACH + } // end while(work) + + // fill DFA struct + HLRDFA *dfa = h_arena_malloc(arena, sizeof(HLRDFA)); + dfa->nstates = states->used; + dfa->states = h_arena_malloc(arena, dfa->nstates*sizeof(HLRState *)); + H_FOREACH(states, HLRState *state, void *v) + size_t idx = (uintptr_t)v; + dfa->states[idx] = state; + H_END_FOREACH + dfa->transitions = transitions; + + return dfa; +} + + + +/* LR(0) table generation */ + +HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) +{ + HAllocator *mm__ = g->mm__; + + HLRTable *table = h_lrtable_new(mm__, dfa->nstates); + HArena *arena = table->arena; + + // remember start symbol + table->start = g->start; + + // add shift entries + for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { + // for each transition x-A->y, add "shift, goto y" to table entry (x,A) + HLRTransition *t = x->elem; + + HLRAction *action = h_shift_action(arena, t->to); + h_hashtable_put(table->rows[t->from], t->symbol, action); + } + + // add reduce entries, record inadequate states + for(size_t i=0; instates; i++) { + // find reducible items in state + H_FOREACH_KEY(dfa->states[i], HLRItem *item) + if(item->mark == item->len) { // mark at the end + // check for conflicts + // XXX store more informative stuff in the inadeq records? + if(table->forall[i]) { + // reduce/reduce conflict with a previous item + h_slist_push(table->inadeq, (void *)(uintptr_t)i); + } else if(!h_hashtable_empty(table->rows[i])) { + // shift/reduce conflict with one of the row's entries + h_slist_push(table->inadeq, (void *)(uintptr_t)i); + } + + // set reduce action for the entire row + table->forall[i] = h_reduce_action(arena, item); + } + H_END_FOREACH + } + + return table; +} From 409d33c91642abe327823a0792cda2fa65d978a9 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 14:09:39 +0200 Subject: [PATCH 03/27] split out h_lrengine_action() --- src/backends/lr.c | 22 ++++++++++++++-------- src/backends/lr.h | 8 +++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index c3062d5..0ad4569 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -190,18 +190,12 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) return engine; } -void h_lrengine_step(HLREngine *engine, HInputStream *stream) +const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream) { - // short-hand names - HSlist *left = engine->left; HSlist *right = engine->right; HArena *arena = engine->arena; HArena *tarena = engine->tarena; - // stack layout: - // on the left stack, we put pairs: (saved state, semantic value) - // on the right stack, we put pairs: (symbol, semantic value) - // make sure there is input on the right stack if(h_slist_empty(right)) { // XXX use statically-allocated terminal symbols @@ -230,6 +224,18 @@ void h_lrengine_step(HLREngine *engine, HInputStream *stream) // table lookup const HLRAction *action = h_lr_lookup(engine->table, engine->state, symbol); + + return action; +} + +void h_lrengine_step(HLREngine *engine, const HLRAction *action) +{ + // short-hand names + HSlist *left = engine->left; + HSlist *right = engine->right; + HArena *arena = engine->arena; + HArena *tarena = engine->tarena; + if(action == NULL) { // no handle recognizable in input, terminate engine->running = false; @@ -313,7 +319,7 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* // run while the recognizer finds handles in the input while(engine->running) - h_lrengine_step(engine, stream); + h_lrengine_step(engine, h_lrengine_action(engine, stream)); HParseResult *result = h_lrengine_result(engine); if(!result) diff --git a/src/backends/lr.h b/src/backends/lr.h index afd4042..9312237 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -61,8 +61,13 @@ typedef struct HLREnhGrammar_ { typedef struct HLREngine_ { const HLRTable *table; + + // stack layout: + // on the left stack, we put pairs: (saved state, semantic value) + // on the right stack, we put pairs: (symbol, semantic value) HSlist *left; // left stack; reductions happen here HSlist *right; // right stack; input appears here + size_t state; bool running; HArena *arena; // will hold the results @@ -116,7 +121,8 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); void h_lalr_free(HParser *parser); const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); -void h_lrengine_step(HLREngine *engine, HInputStream *stream); +const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream); +void h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); From 168760b10a6234259cd80b1c455fbf95d76d10ba Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 14:16:34 +0200 Subject: [PATCH 04/27] return running state from h_lrengine_step --- src/backends/lr.c | 25 ++++++++++--------------- src/backends/lr.h | 3 +-- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index 0ad4569..2d329b1 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -183,7 +183,6 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) engine->left = h_slist_new(tarena); engine->right = h_slist_new(tarena); engine->state = 0; - engine->running = 1; engine->arena = arena; engine->tarena = tarena; @@ -228,7 +227,8 @@ const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream) return action; } -void h_lrengine_step(HLREngine *engine, const HLRAction *action) +// run LR parser for one round; returns false when finished +bool h_lrengine_step(HLREngine *engine, const HLRAction *action) { // short-hand names HSlist *left = engine->left; @@ -236,11 +236,8 @@ void h_lrengine_step(HLREngine *engine, const HLRAction *action) HArena *arena = engine->arena; HArena *tarena = engine->tarena; - if(action == NULL) { - // no handle recognizable in input, terminate - engine->running = false; - return; - } + if(action == NULL) + return false; // no handle recognizable in input, terminate if(action->type == HLR_SHIFT) { h_slist_push(left, (void *)(uintptr_t)engine->state); @@ -280,11 +277,8 @@ void h_lrengine_step(HLREngine *engine, const HLRAction *action) value = (HParsedToken *)symbol->reshape(make_result(arena, value)); // call validation and semantic action, if present - if(symbol->pred && !symbol->pred(make_result(tarena, value))) { - // validation failed -> no parse; terminate - engine->running = false; - return; - } + if(symbol->pred && !symbol->pred(make_result(tarena, value))) + return false; // validation failed -> no parse; terminate if(symbol->action) value = (HParsedToken *)symbol->action(make_result(arena, value)); @@ -292,6 +286,8 @@ void h_lrengine_step(HLREngine *engine, const HLRAction *action) h_slist_push(right, value); h_slist_push(right, symbol); } + + return true; } HParseResult *h_lrengine_result(HLREngine *engine) @@ -317,9 +313,8 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse HLREngine *engine = h_lrengine_new(arena, tarena, table); - // run while the recognizer finds handles in the input - while(engine->running) - h_lrengine_step(engine, h_lrengine_action(engine, stream)); + // iterate engine to completion + while(h_lrengine_step(engine, h_lrengine_action(engine, stream))); HParseResult *result = h_lrengine_result(engine); if(!result) diff --git a/src/backends/lr.h b/src/backends/lr.h index 9312237..b95c133 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -69,7 +69,6 @@ typedef struct HLREngine_ { HSlist *right; // right stack; input appears here size_t state; - bool running; HArena *arena; // will hold the results HArena *tarena; // tmp, deleted after parse } HLREngine; @@ -122,7 +121,7 @@ void h_lalr_free(HParser *parser); const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream); -void h_lrengine_step(HLREngine *engine, const HLRAction *action); +bool h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); From 54ba62bfb7e77db43df7f50dbdfc9640bc10ebb2 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 17:01:13 +0200 Subject: [PATCH 05/27] record conflicts in a special HLRAction type --- src/backends/lalr.c | 13 ++++++++++--- src/backends/lr.c | 21 +++++++++++++++++++++ src/backends/lr.h | 13 ++++++++++--- src/backends/lr0.c | 24 +++++++++++++++--------- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index fa67e5a..698b106 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -138,7 +138,8 @@ int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) HLRAction *prev = h_hashtable_get(tbl->rows[state], x); if(prev && prev != action) { // conflict - h_slist_push(tbl->inadeq, (void *)(uintptr_t)state); + action = h_lr_conflict(tbl->arena, prev, action); + h_hashtable_put(tbl->rows[state], x, action); return -1; } else { h_hashtable_put(tbl->rows[state], x, action); @@ -221,6 +222,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) for(HSlistNode *x=inadeq->head; x; x=x->next) { size_t state = (uintptr_t)x->elem; + bool inadeq = false; // clear old forall entry, it's being replaced by more fine-grained ones table->forall[state] = NULL; @@ -255,7 +257,8 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) if(fs->end_branch) { HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); terminal->type = HCF_END; - h_lrtable_put(table, state, terminal, action); + if(h_lrtable_put(table, state, terminal, action) < 0) + inadeq = true; } H_FOREACH(fs->char_branches, void *key, HStringMap *m) if(!m->epsilon_branch) @@ -265,10 +268,14 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) terminal->type = HCF_CHAR; terminal->chr = key_char((HCharKey)key); - h_lrtable_put(table, state, terminal, action); + if(h_lrtable_put(table, state, terminal, action) < 0) + inadeq = true; H_END_FOREACH // lookahead character } H_END_FOREACH // enhanced production H_END_FOREACH // reducible item + + if(inadeq) + h_slist_push(table->inadeq, (void *)(uintptr_t)state); } } diff --git a/src/backends/lr.c b/src/backends/lr.c index 2d329b1..bf06645 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -159,6 +159,25 @@ HLRAction *h_reduce_action(HArena *arena, const HLRItem *item) return action; } +// adds 'new' to the branches of 'action' +// returns a 'action' if it is already of type HLR_CONFLICT +// allocates a new HLRAction otherwise +HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new) +{ + if(action->type != HLR_CONFLICT) { + HLRAction *old = action; + action = h_arena_malloc(arena, sizeof(HLRAction)); + action->type = HLR_CONFLICT; + action->branches = h_slist_new(arena); + h_slist_push(action->branches, old); + } + + assert(action->type == HLR_CONFLICT); + h_slist_push(action->branches, new); + + return action; +} + /* LR driver */ @@ -239,6 +258,8 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) if(action == NULL) return false; // no handle recognizable in input, terminate + assert(action->type == HLR_SHIFT || action->type == HLR_REDUCE); + if(action->type == HLR_SHIFT) { h_slist_push(left, (void *)(uintptr_t)engine->state); h_slist_pop(right); // symbol (discard) diff --git a/src/backends/lr.h b/src/backends/lr.h index b95c133..13e10d4 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -28,16 +28,22 @@ typedef struct HLRItem_ { } HLRItem; typedef struct HLRAction_ { - enum {HLR_SHIFT, HLR_REDUCE} type; + enum {HLR_SHIFT, HLR_REDUCE, HLR_CONFLICT} type; union { - size_t nextstate; // used with SHIFT + // used with HLR_SHIFT + size_t nextstate; + + // used with HLR_REDUCE struct { HCFChoice *lhs; // symbol carrying semantic actions etc. size_t length; // # of symbols in rhs #ifndef NDEBUG HCFChoice **rhs; // NB: the rhs symbols are not needed for the parse #endif - } production; // used with REDUCE + } production; + + // used with HLR_CONFLICT + HSlist *branches; // list of possible HLRActions }; } HLRAction; @@ -104,6 +110,7 @@ void h_lrtable_free(HLRTable *table); HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table); HLRAction *h_reduce_action(HArena *arena, const HLRItem *item); HLRAction *h_shift_action(HArena *arena, size_t nextstate); +HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new); bool h_eq_symbol(const void *p, const void *q); bool h_eq_lr_itemset(const void *p, const void *q); diff --git a/src/backends/lr0.c b/src/backends/lr0.c index aab2ad1..67cf2aa 100644 --- a/src/backends/lr0.c +++ b/src/backends/lr0.c @@ -182,23 +182,29 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // add reduce entries, record inadequate states for(size_t i=0; instates; i++) { + bool inadeq = false; + // find reducible items in state H_FOREACH_KEY(dfa->states[i], HLRItem *item) if(item->mark == item->len) { // mark at the end - // check for conflicts - // XXX store more informative stuff in the inadeq records? + HLRAction *reduce = h_reduce_action(arena, item); + + // check for reduce/reduce conflict on forall if(table->forall[i]) { - // reduce/reduce conflict with a previous item - h_slist_push(table->inadeq, (void *)(uintptr_t)i); - } else if(!h_hashtable_empty(table->rows[i])) { - // shift/reduce conflict with one of the row's entries - h_slist_push(table->inadeq, (void *)(uintptr_t)i); + reduce = h_lr_conflict(arena, table->forall[i], reduce); + inadeq = true; } + table->forall[i] = reduce; - // set reduce action for the entire row - table->forall[i] = h_reduce_action(arena, item); + // check for shift/reduce conflict with other entries + // NOTE: these are not recorded as HLR_CONFLICTs at this point + if(!h_hashtable_empty(table->rows[i])) + inadeq = true; } H_END_FOREACH + + if(inadeq) + h_slist_push(table->inadeq, (void *)(uintptr_t)i); } return table; From 55c9a3d9c51fb752e89c85f49987a124c35acc88 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 17:20:53 +0200 Subject: [PATCH 06/27] add stub GLR backend with h_glr_parse() a copy of h_lr_parse() --- src/Makefile | 3 +- src/backends/glr.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++ src/backends/lr.h | 1 + src/hammer.c | 1 + src/hammer.h | 4 +- src/internal.h | 1 + src/t_parser.c | 38 +++++++++++++++++++ 7 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 src/backends/glr.c diff --git a/src/Makefile b/src/Makefile index 380436a..9ce6d9f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -27,9 +27,10 @@ PARSERS := \ BACKENDS := \ packrat \ + regex \ llk \ lalr \ - regex + glr HAMMER_PARTS := \ bitreader.o \ diff --git a/src/backends/glr.c b/src/backends/glr.c new file mode 100644 index 0000000..c57ffd6 --- /dev/null +++ b/src/backends/glr.c @@ -0,0 +1,93 @@ +#include "lr.h" + + + +/* GLR driver */ + +HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) +{ + HLRTable *table = parser->backend_data; + if(!table) + return NULL; + + HArena *arena = h_new_arena(mm__, 0); // will hold the results + HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse + HLREngine *engine = h_lrengine_new(arena, tarena, table); + + // iterate engine to completion + while(h_lrengine_step(engine, h_lrengine_action(engine, stream))); + + HParseResult *result = h_lrengine_result(engine); + if(!result) + h_delete_arena(arena); + h_delete_arena(tarena); + return result; +} + + + +HParserBackendVTable h__glr_backend_vtable = { + .compile = h_lalr_compile, + .parse = h_glr_parse, + .free = h_lalr_free +}; + + + + +// dummy! +int test_glr(void) +{ + /* + E -> E '-' T + | T + T -> '(' E ')' + | 'n' -- also try [0-9] for the charset paths + */ + + HParser *n = h_ch('n'); + HParser *E = h_indirect(); + HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), n, NULL); + HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); + h_bind_indirect(E, E_); + HParser *p = E; + + printf("\n==== G R A M M A R ====\n"); + HCFGrammar *g = h_cfgrammar(&system_allocator, p); + if(g == NULL) { + fprintf(stderr, "h_cfgrammar failed\n"); + return 1; + } + h_pprint_grammar(stdout, g, 0); + + printf("\n==== D F A ====\n"); + HLRDFA *dfa = h_lr0_dfa(g); + if(dfa) + h_pprint_lrdfa(stdout, g, dfa, 0); + else + fprintf(stderr, "h_lalr_dfa failed\n"); + + printf("\n==== L R ( 0 ) T A B L E ====\n"); + HLRTable *table0 = h_lr0_table(g, dfa); + if(table0) + h_pprint_lrtable(stdout, g, table0, 0); + else + fprintf(stderr, "h_lr0_table failed\n"); + h_lrtable_free(table0); + + printf("\n==== L A L R T A B L E ====\n"); + if(h_compile(p, PB_GLR, NULL)) { + fprintf(stderr, "does not compile\n"); + return 2; + } + h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); + + printf("\n==== P A R S E R E S U L T ====\n"); + HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 13); + if(res) + h_pprint(stdout, res->ast, 0, 2); + else + printf("no parse\n"); + + return 0; +} diff --git a/src/backends/lr.h b/src/backends/lr.h index 13e10d4..f766d5b 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -131,6 +131,7 @@ const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream); bool h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); +HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); void h_pprint_lritem(FILE *f, const HCFGrammar *g, const HLRItem *item); void h_pprint_lrstate(FILE *f, const HCFGrammar *g, diff --git a/src/hammer.c b/src/hammer.c index 7d5b4e9..7fc80db 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -31,6 +31,7 @@ static HParserBackendVTable *backends[PB_MAX + 1] = { &h__regex_backend_vtable, &h__llk_backend_vtable, &h__lalr_backend_vtable, + &h__glr_backend_vtable, }; diff --git a/src/hammer.h b/src/hammer.h index a5ebcff..67fb8e4 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -37,8 +37,8 @@ typedef enum HParserBackend_ { PB_REGULAR, PB_LLk, PB_LALR, - PB_GLR, // Not Implemented - PB_MAX = PB_LALR + PB_GLR, + PB_MAX = PB_GLR } HParserBackend; typedef enum HTokenType_ { diff --git a/src/internal.h b/src/internal.h index 2f3018d..d8b221a 100644 --- a/src/internal.h +++ b/src/internal.h @@ -220,6 +220,7 @@ struct HBitWriter_ { extern HParserBackendVTable h__packrat_backend_vtable; extern HParserBackendVTable h__llk_backend_vtable; extern HParserBackendVTable h__lalr_backend_vtable; +extern HParserBackendVTable h__glr_backend_vtable; // }}} // TODO(thequux): Set symbol visibility for these functions so that they aren't exported. diff --git a/src/t_parser.c b/src/t_parser.c index a0e4040..57486cd 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -585,4 +585,42 @@ void register_parser_tests(void) { g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); + + g_test_add_data_func("/core/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); + g_test_add_data_func("/core/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); + g_test_add_data_func("/core/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); + g_test_add_data_func("/core/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); + g_test_add_data_func("/core/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); + g_test_add_data_func("/core/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); + g_test_add_data_func("/core/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); + g_test_add_data_func("/core/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); + g_test_add_data_func("/core/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); + g_test_add_data_func("/core/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); + g_test_add_data_func("/core/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); + g_test_add_data_func("/core/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); +#if 0 + g_test_add_data_func("/core/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); + g_test_add_data_func("/core/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); +#endif + g_test_add_data_func("/core/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); + g_test_add_data_func("/core/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); + g_test_add_data_func("/core/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); + g_test_add_data_func("/core/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); + g_test_add_data_func("/core/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); + g_test_add_data_func("/core/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); + g_test_add_data_func("/core/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); + g_test_add_data_func("/core/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); + g_test_add_data_func("/core/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); + g_test_add_data_func("/core/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); + g_test_add_data_func("/core/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); + g_test_add_data_func("/core/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); + g_test_add_data_func("/core/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); + g_test_add_data_func("/core/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); + g_test_add_data_func("/core/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); + g_test_add_data_func("/core/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); + g_test_add_data_func("/core/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); + g_test_add_data_func("/core/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); + g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); + g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); + g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); } From ecfc0a8e62a11df893282fc8ff01ece3436551a8 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 18:21:53 +0200 Subject: [PATCH 07/27] fix conflict pretty-printing --- src/backends/lr.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index bf06645..392d3f6 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -425,9 +425,11 @@ void h_pprint_lrdfa(FILE *f, const HCFGrammar *g, void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) { - if(action->type == HLR_SHIFT) { + switch(action->type) { + case HLR_SHIFT: fprintf(f, "s%lu", action->nextstate); - } else { + break; + case HLR_REDUCE: fputs("r(", f); h_pprint_symbol(f, g, action->production.lhs); fputs(" -> ", f); @@ -439,6 +441,18 @@ void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) h_pprint_sequence(f, g, &seq); #endif fputc(')', f); + break; + case HLR_CONFLICT: + fputc('!', f); + for(HSlistNode *x=action->branches->head; x; x=x->next) { + HLRAction *branch = x->elem; + assert(branch->type != HLR_CONFLICT); // no nesting + pprint_lraction(f, g, branch); + if(x->next) fputc('/', f); // separator + } + break; + default: + assert_message(0, "not reached"); } } @@ -459,13 +473,7 @@ void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, fputc(' ', f); // separator h_pprint_symbol(f, g, symbol); fputc(':', f); - if(table->forall[i]) { - fputc(action->type == HLR_SHIFT? 's' : 'r', f); - fputc('/', f); - fputc(table->forall[i]->type == HLR_SHIFT? 's' : 'r', f); - } else { - pprint_lraction(f, g, action); - } + pprint_lraction(f, g, action); H_END_FOREACH fputc('\n', f); } From 572f1c8f9e14e521317c54045ee22639d65b5ffe Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 18:22:19 +0200 Subject: [PATCH 08/27] expand stub GLR backend --- src/backends/glr.c | 60 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index c57ffd6..429d06b 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -1,6 +1,25 @@ #include "lr.h" +/* GLR compilation (LALR w/o failing on conflict) */ + +int h_glr_compile(HAllocator* mm__, HParser* parser, const void* params) +{ + int result = h_lalr_compile(mm__, parser, params); + + if(result == -1 && parser->backend_data) { + // table is there, just has conflicts? nevermind, that's okay. + result = 0; + } + + return result; +} + +void h_glr_free(HParser *parser) +{ + h_lalr_free(parser); +} + /* GLR driver */ @@ -12,12 +31,28 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - HLREngine *engine = h_lrengine_new(arena, tarena, table); - // iterate engine to completion - while(h_lrengine_step(engine, h_lrengine_action(engine, stream))); + HSlist *engines = h_slist_new(tarena); + h_slist_push(engines, h_lrengine_new(arena, tarena, table)); + + HParseResult *result = NULL; + while(result == NULL && !h_slist_empty(engines)) { + for(HSlistNode **x = &engines->head; *x; ) { + HLREngine *engine = (*x)->elem; + + const HLRAction *action = h_lrengine_action(engine, stream); + // XXX handle conflicts -> fork engine + bool running = h_lrengine_step(engine, action); + + if(running) { + x = &(*x)->next; // go to next + } else { + *x = (*x)->next; // remove from list + result = h_lrengine_result(engine); + } + } + } - HParseResult *result = h_lrengine_result(engine); if(!result) h_delete_arena(arena); h_delete_arena(tarena); @@ -27,9 +62,9 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HParserBackendVTable h__glr_backend_vtable = { - .compile = h_lalr_compile, + .compile = h_glr_compile, .parse = h_glr_parse, - .free = h_lalr_free + .free = h_glr_free }; @@ -39,16 +74,13 @@ HParserBackendVTable h__glr_backend_vtable = { int test_glr(void) { /* - E -> E '-' T - | T - T -> '(' E ')' - | 'n' -- also try [0-9] for the charset paths + E -> E '+' E + | 'd' */ - HParser *n = h_ch('n'); + HParser *d = h_ch('d'); HParser *E = h_indirect(); - HParser *T = h_choice(h_sequence(h_ch('('), E, h_ch(')'), NULL), n, NULL); - HParser *E_ = h_choice(h_sequence(E, h_ch('-'), T, NULL), T, NULL); + HParser *E_ = h_choice(h_sequence(E, h_ch('+'), E, NULL), d, NULL); h_bind_indirect(E, E_); HParser *p = E; @@ -83,7 +115,7 @@ int test_glr(void) h_pprint_lrtable(stdout, g, (HLRTable *)p->backend_data, 0); printf("\n==== P A R S E R E S U L T ====\n"); - HParseResult *res = h_parse(p, (uint8_t *)"n-(n-((n)))-n", 13); + HParseResult *res = h_parse(p, (uint8_t *)"d+d+d", 5); if(res) h_pprint(stdout, res->ast, 0, 2); else From 4f36fcd2c105008014364131c48f7001597bdcf7 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 19 Jun 2013 18:51:16 +0200 Subject: [PATCH 09/27] avoid duplicate conflict branches --- src/backends/lr.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index 392d3f6..4ab53bc 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -170,11 +170,19 @@ HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new) action->type = HLR_CONFLICT; action->branches = h_slist_new(arena); h_slist_push(action->branches, old); + h_slist_push(action->branches, new); + } else { + // check if 'new' is already among branches + HSlistNode *x; + for(x=action->branches->head; x; x=x->next) { + if(x->elem == new) + break; + } + // add 'new' if it is not already in list + if(x == NULL) + h_slist_push(action->branches, new); } - assert(action->type == HLR_CONFLICT); - h_slist_push(action->branches, new); - return action; } From b1e8e297740256e44233837c6aff309369608c23 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 20 Jun 2013 11:05:57 +0200 Subject: [PATCH 10/27] add engine forking --- src/backends/glr.c | 55 ++++++++++++++++++++++++++++++++++++++++++-- src/backends/lr.c | 12 +++++----- src/backends/lr.h | 2 +- src/datastructures.c | 10 ++++++++ src/internal.h | 1 + 5 files changed, 71 insertions(+), 9 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 429d06b..2b4b974 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -1,3 +1,4 @@ +#include #include "lr.h" @@ -23,6 +24,26 @@ void h_glr_free(HParser *parser) /* GLR driver */ +HLREngine *fork_engine(const HLREngine *engine) +{ + HLREngine *eng2 = h_arena_malloc(engine->tarena, sizeof(HLREngine)); + eng2->table = engine->table; + eng2->state = engine->state; + + // shallow-copy the stacks + // this works because h_slist_push and h_slist_pop never modify + // the underlying structure of HSlistNodes, only the head pointer. + // in fact, this gives us prefix sharing for free. + eng2->left = h_arena_malloc(engine->tarena, sizeof(HSlist)); + eng2->right = h_arena_malloc(engine->tarena, sizeof(HSlist)); + *eng2->left = *engine->left; + *eng2->right = *engine->right; + + eng2->arena = engine->arena; + eng2->tarena = engine->tarena; + return eng2; +} + HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) { HLRTable *table = parser->backend_data; @@ -41,14 +62,44 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HLREngine *engine = (*x)->elem; const HLRAction *action = h_lrengine_action(engine, stream); - // XXX handle conflicts -> fork engine + + // fork engine on conflicts + if(action && action->type == HLR_CONFLICT) { + const HSlist *branches = action->branches; + + // there should be at least two conflicting actions + assert(branches->head); + assert(branches->head->next); + + // save first action for use with old engine below + action = branches->head->elem; + + // fork a new engine for all the other actions + for(HSlistNode *x=branches->head->next; x; x=x->next) { + HLRAction *act = x->elem; + HLREngine *eng = fork_engine(engine); + + // perform one step; add engine to list if it wants to keep running + bool run = h_lrengine_step(eng, act); + if(run) { + h_slist_push(engines, eng); + } else { + HParseResult *res = h_lrengine_result(eng); + if(res) + result = res; + } + } + } + bool running = h_lrengine_step(engine, action); if(running) { x = &(*x)->next; // go to next } else { *x = (*x)->next; // remove from list - result = h_lrengine_result(engine); + HParseResult *res = h_lrengine_result(engine); + if(res) + result = res; } } } diff --git a/src/backends/lr.c b/src/backends/lr.c index 4ab53bc..1ea6a39 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -270,8 +270,8 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) if(action->type == HLR_SHIFT) { h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_pop(right); // symbol (discard) - h_slist_push(left, h_slist_pop(right)); // semantic value + h_slist_drop(right); // symbol (discard) + h_slist_push(left, h_slist_drop(right)); // semantic value engine->state = action->nextstate; } else { assert(action->type == HLR_REDUCE); @@ -286,8 +286,8 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // pull values off the left stack, rewinding state accordingly HParsedToken *v = NULL; for(size_t i=0; istate = (uintptr_t)h_slist_pop(left); + v = h_slist_drop(left); + engine->state = (uintptr_t)h_slist_drop(left); // collect values in result sequence value->seq->elements[len-1-i] = v; @@ -322,10 +322,10 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) HParseResult *h_lrengine_result(HLREngine *engine) { // parsing was successful iff the start symbol is on top of the right stack - if(h_slist_pop(engine->right) == engine->table->start) { + if(h_slist_drop(engine->right) == engine->table->start) { // next on the right stack is the start symbol's semantic value assert(!h_slist_empty(engine->right)); - HParsedToken *tok = h_slist_pop(engine->right); + HParsedToken *tok = h_slist_drop(engine->right); return make_result(engine->arena, tok); } else { return NULL; diff --git a/src/backends/lr.h b/src/backends/lr.h index f766d5b..edf0871 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -67,6 +67,7 @@ typedef struct HLREnhGrammar_ { typedef struct HLREngine_ { const HLRTable *table; + size_t state; // stack layout: // on the left stack, we put pairs: (saved state, semantic value) @@ -74,7 +75,6 @@ typedef struct HLREngine_ { HSlist *left; // left stack; reductions happen here HSlist *right; // right stack; input appears here - size_t state; HArena *arena; // will hold the results HArena *tarena; // tmp, deleted after parse } HLREngine; diff --git a/src/datastructures.c b/src/datastructures.c index 075b966..94bc901 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -62,6 +62,16 @@ HSlist* h_slist_copy(HSlist *slist) { return ret; } +// like h_slist_pop, but does not deallocate the head node +void* h_slist_drop(HSlist *slist) { + HSlistNode *head = slist->head; + if (!head) + return NULL; + void* ret = head->elem; + slist->head = head->next; + return ret; +} + void* h_slist_pop(HSlist *slist) { HSlistNode *head = slist->head; if (!head) diff --git a/src/internal.h b/src/internal.h index d8b221a..a897e9f 100644 --- a/src/internal.h +++ b/src/internal.h @@ -248,6 +248,7 @@ void h_carray_append(HCountedArray *array, void* item); HSlist* h_slist_new(HArena *arena); HSlist* h_slist_copy(HSlist *slist); void* h_slist_pop(HSlist *slist); +void* h_slist_drop(HSlist *slist); void h_slist_push(HSlist *slist, void* item); bool h_slist_find(HSlist *slist, const void* item); HSlist* h_slist_remove_all(HSlist *slist, const void* item); From f5d4ea90da9f4829714b5460bd0028e5851929e0 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 20 Jun 2013 11:30:30 +0200 Subject: [PATCH 11/27] add ambiguous test case for GLR --- src/t_parser.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/t_parser.c b/src/t_parser.c index 57486cd..7522e92 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -428,6 +428,17 @@ static void test_rightrec(gconstpointer backend) { g_check_parse_ok(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "(u0x61 (u0x61 (u0x61)))"); } +static void test_ambiguous(gconstpointer backend) { + HParser *d_ = h_ch('d'); + HParser *E_ = h_indirect(); + h_bind_indirect(E_, h_choice(h_sequence(E_, h_ch('+'), E_, NULL), d_, NULL)); + + g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "u0x64"); + g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)"); + g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", 5, "(u0x64 u0x2b (u0x64 u0x2b u0x64))"); + g_check_parse_failed(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+", 2); +} + void register_parser_tests(void) { g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); @@ -623,4 +634,5 @@ void register_parser_tests(void) { g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); + g_test_add_data_func("/core/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); } From bbbaf1634cead71bd3997e062213bb63bd3ce544 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 15:00:04 +0200 Subject: [PATCH 12/27] add an assertion checking that HLR_REDUCE is always followed by HLR_SHIFT --- src/backends/lr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backends/lr.c b/src/backends/lr.c index 1ea6a39..2f7d5e4 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -314,6 +314,12 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // push result (value, symbol) onto the right stack h_slist_push(right, value); h_slist_push(right, symbol); + + // this is LR, building a right-most derivation bottom-up, so no reduce can + // follow a reduce. we can also assume no conflict follows for GLR if we + // use LALR tables, because only terminal symbols (lookahead) get reduces. + const HLRAction *next = h_lr_lookup(engine->table, engine->state, symbol); + assert(next == NULL || next->type == HLR_SHIFT); } return true; From 7cd143c0c42779778eec755d4386ea752fb5f1f4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 15:06:31 +0200 Subject: [PATCH 13/27] piggy-back the next shift directly onto the reduce --- src/backends/lr.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index 2f7d5e4..03264b5 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -268,12 +268,7 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) assert(action->type == HLR_SHIFT || action->type == HLR_REDUCE); - if(action->type == HLR_SHIFT) { - h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_drop(right); // symbol (discard) - h_slist_push(left, h_slist_drop(right)); // semantic value - engine->state = action->nextstate; - } else { + if(action->type == HLR_REDUCE) { assert(action->type == HLR_REDUCE); size_t len = action->production.length; HCFChoice *symbol = action->production.lhs; @@ -318,8 +313,18 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we // use LALR tables, because only terminal symbols (lookahead) get reduces. - const HLRAction *next = h_lr_lookup(engine->table, engine->state, symbol); - assert(next == NULL || next->type == HLR_SHIFT); + action = h_lr_lookup(engine->table, engine->state, symbol); + if(action == NULL) + return false; // no handle after reduce; terminate + assert(action->type == HLR_SHIFT); + } + + // this could be the original action, or a shift piggy-backed onto reduce + if(action->type == HLR_SHIFT) { + h_slist_push(left, (void *)(uintptr_t)engine->state); + h_slist_drop(right); // symbol (discard) + h_slist_push(left, h_slist_drop(right)); // semantic value + engine->state = action->nextstate; } return true; From bf3e3c162e2bd199effd17412813d8271dba40ee Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 15:31:37 +0200 Subject: [PATCH 14/27] don't even touch the right stack on piggy-back shifts --- src/backends/lr.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index 03264b5..33d597b 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -269,7 +269,6 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) assert(action->type == HLR_SHIFT || action->type == HLR_REDUCE); if(action->type == HLR_REDUCE) { - assert(action->type == HLR_REDUCE); size_t len = action->production.length; HCFChoice *symbol = action->production.lhs; @@ -306,21 +305,24 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) if(symbol->action) value = (HParsedToken *)symbol->action(make_result(arena, value)); - // push result (value, symbol) onto the right stack - h_slist_push(right, value); - h_slist_push(right, symbol); - // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we // use LALR tables, because only terminal symbols (lookahead) get reduces. - action = h_lr_lookup(engine->table, engine->state, symbol); - if(action == NULL) - return false; // no handle after reduce; terminate - assert(action->type == HLR_SHIFT); - } + const HLRAction *next = h_lr_lookup(engine->table, engine->state, symbol); + if(next) { + assert(next->type == HLR_SHIFT); - // this could be the original action, or a shift piggy-backed onto reduce - if(action->type == HLR_SHIFT) { + // piggy-back the shift onto here, never touching the right stack + h_slist_push(left, (void *)(uintptr_t)engine->state); + h_slist_push(left, value); + engine->state = next->nextstate; + } else { + // fallback + h_slist_push(right, value); + h_slist_push(right, symbol); + } + } else { + assert(action->type == HLR_SHIFT); h_slist_push(left, (void *)(uintptr_t)engine->state); h_slist_drop(right); // symbol (discard) h_slist_push(left, h_slist_drop(right)); // semantic value From ec88580b22866a65cf874c8b313ec2336eb0a061 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 18:46:16 +0200 Subject: [PATCH 15/27] make h_lrengine_step a void function again --- src/backends/glr.c | 51 +++++++++++++++++++++++++++++----------------- src/backends/lr.c | 46 +++++++++++++++++++++++------------------ src/backends/lr.h | 3 ++- src/backends/lr0.c | 5 +++++ 4 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 2b4b974..d460e8a 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -61,6 +61,20 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* for(HSlistNode **x = &engines->head; *x; ) { HLREngine *engine = (*x)->elem; + // check for terminated engines + if(engine->run) { + x = &(*x)->next; // advance x with no change + } else { + *x = (*x)->next; // advance x, removing the current element + + // check for parse success + HParseResult *res = h_lrengine_result(engine); + if(res) + result = res; + + continue; + } + const HLRAction *action = h_lrengine_action(engine, stream); // fork engine on conflicts @@ -79,28 +93,13 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HLRAction *act = x->elem; HLREngine *eng = fork_engine(engine); - // perform one step; add engine to list if it wants to keep running - bool run = h_lrengine_step(eng, act); - if(run) { - h_slist_push(engines, eng); - } else { - HParseResult *res = h_lrengine_result(eng); - if(res) - result = res; - } + // perform one step and add to list + h_lrengine_step(eng, act); + h_slist_push(engines, eng); } } - bool running = h_lrengine_step(engine, action); - - if(running) { - x = &(*x)->next; // go to next - } else { - *x = (*x)->next; // remove from list - HParseResult *res = h_lrengine_result(engine); - if(res) - result = res; - } + h_lrengine_step(engine, action); } } @@ -120,6 +119,20 @@ HParserBackendVTable h__glr_backend_vtable = { +// XXX TODO +// - eliminate right stack by always doing a shift after reduce +// (shift should always follow reduce because rightmost) +// - split tables into +// - one mapping input bytes to actions (shift or reduce or conflict) +// - one mapping reduced-to lhs nonterminals to shift states +// - can there still be conflicts here? +// - use HStringMap to represent lookahead sets and the "piggyback" table +// - implement engine merging +// - triggered when two enter the same state +// - old stacks (/engines?) saved +// - new common suffix stack created +// - when rewinding (during reduce), watch for empty stack -> demerge + // dummy! int test_glr(void) diff --git a/src/backends/lr.c b/src/backends/lr.c index 33d597b..2603ff2 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -207,9 +207,10 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) HLREngine *engine = h_arena_malloc(tarena, sizeof(HLREngine)); engine->table = table; + engine->state = 0; + engine->run = true; engine->left = h_slist_new(tarena); engine->right = h_slist_new(tarena); - engine->state = 0; engine->arena = arena; engine->tarena = tarena; @@ -255,7 +256,7 @@ const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream) } // run LR parser for one round; returns false when finished -bool h_lrengine_step(HLREngine *engine, const HLRAction *action) +static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) { // short-hand names HSlist *left = engine->left; @@ -308,19 +309,18 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we // use LALR tables, because only terminal symbols (lookahead) get reduces. - const HLRAction *next = h_lr_lookup(engine->table, engine->state, symbol); - if(next) { - assert(next->type == HLR_SHIFT); + const HLRAction *shift = h_lr_lookup(engine->table, engine->state, symbol); + if(shift == NULL) + return false; // parse error + assert(shift->type == HLR_SHIFT); - // piggy-back the shift onto here, never touching the right stack - h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_push(left, value); - engine->state = next->nextstate; - } else { - // fallback - h_slist_push(right, value); - h_slist_push(right, symbol); - } + // piggy-back the shift right here, never touching the input + h_slist_push(left, (void *)(uintptr_t)engine->state); + h_slist_push(left, value); + engine->state = shift->nextstate; + + if(symbol == engine->table->start) + return false; // reduced to start symbol; accept! } else { assert(action->type == HLR_SHIFT); h_slist_push(left, (void *)(uintptr_t)engine->state); @@ -332,13 +332,18 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) return true; } +// run LR parser for one round; sets engine->run +void h_lrengine_step(HLREngine *engine, const HLRAction *action) +{ + engine->run = h_lrengine_step_(engine, action); +} + HParseResult *h_lrengine_result(HLREngine *engine) { - // parsing was successful iff the start symbol is on top of the right stack - if(h_slist_drop(engine->right) == engine->table->start) { - // next on the right stack is the start symbol's semantic value - assert(!h_slist_empty(engine->right)); - HParsedToken *tok = h_slist_drop(engine->right); + // parsing was successful iff after a shift the engine is back in state 0 + if(engine->state == 0 && !h_slist_empty(engine->left)) { + // on top of the stack is the start symbol's semantic value + HParsedToken *tok = engine->left->head->elem; return make_result(engine->arena, tok); } else { return NULL; @@ -356,7 +361,8 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HLREngine *engine = h_lrengine_new(arena, tarena, table); // iterate engine to completion - while(h_lrengine_step(engine, h_lrengine_action(engine, stream))); + while(engine->run) + h_lrengine_step(engine, h_lrengine_action(engine, stream)); HParseResult *result = h_lrengine_result(engine); if(!result) diff --git a/src/backends/lr.h b/src/backends/lr.h index edf0871..5e2f032 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -68,6 +68,7 @@ typedef struct HLREnhGrammar_ { typedef struct HLREngine_ { const HLRTable *table; size_t state; + bool run; // stack layout: // on the left stack, we put pairs: (saved state, semantic value) @@ -128,7 +129,7 @@ void h_lalr_free(HParser *parser); const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream); -bool h_lrengine_step(HLREngine *engine, const HLRAction *action); +void h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); diff --git a/src/backends/lr0.c b/src/backends/lr0.c index 67cf2aa..9f350b6 100644 --- a/src/backends/lr0.c +++ b/src/backends/lr0.c @@ -171,6 +171,11 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // remember start symbol table->start = g->start; + // add dummy shift entry for the start symbol so h_lrengine_step can always + // find a shift. + // NB: nextstate=0 is used for the "victory condition" by h_lrengine_result. + h_hashtable_put(table->rows[0], g->start, h_shift_action(arena, 0)); + // add shift entries for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { // for each transition x-A->y, add "shift, goto y" to table entry (x,A) From 1e59e461fa5a544d12dd5350ddc7098454bb5a91 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 19:58:58 +0200 Subject: [PATCH 16/27] make ambiguous test ignore order of precedence --- src/t_parser.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/t_parser.c b/src/t_parser.c index 7522e92..59adf36 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -430,13 +430,15 @@ static void test_rightrec(gconstpointer backend) { static void test_ambiguous(gconstpointer backend) { HParser *d_ = h_ch('d'); + HParser *p_ = h_ch('+'); HParser *E_ = h_indirect(); - h_bind_indirect(E_, h_choice(h_sequence(E_, h_ch('+'), E_, NULL), d_, NULL)); + h_bind_indirect(E_, h_choice(h_sequence(E_, p_, E_, NULL), d_, NULL)); + HParser *expr_ = h_action(E_, h_act_flatten); - g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "u0x64"); - g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)"); - g_check_parse_ok(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", 5, "(u0x64 u0x2b (u0x64 u0x2b u0x64))"); - g_check_parse_failed(E_, (HParserBackend)GPOINTER_TO_INT(backend), "d+", 2); + g_check_parse_ok(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "(u0x64)"); + g_check_parse_ok(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)"); + g_check_parse_ok(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", 5, "(u0x64 u0x2b u0x64 u0x2b u0x64)"); + g_check_parse_failed(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+", 2); } void register_parser_tests(void) { From c32cf709b2e51924f8f1f91693febc1856fedc43 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 20:11:19 +0200 Subject: [PATCH 17/27] eliminate the right stack; work with the HInputStream directly --- src/backends/glr.c | 15 ++++---- src/backends/lr.c | 87 +++++++++++++++++++++++----------------------- src/backends/lr.h | 12 +++---- 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index d460e8a..7a5f8f5 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -29,15 +29,14 @@ HLREngine *fork_engine(const HLREngine *engine) HLREngine *eng2 = h_arena_malloc(engine->tarena, sizeof(HLREngine)); eng2->table = engine->table; eng2->state = engine->state; + eng2->input = engine->input; - // shallow-copy the stacks + // shallow-copy the stack // this works because h_slist_push and h_slist_pop never modify // the underlying structure of HSlistNodes, only the head pointer. // in fact, this gives us prefix sharing for free. - eng2->left = h_arena_malloc(engine->tarena, sizeof(HSlist)); - eng2->right = h_arena_malloc(engine->tarena, sizeof(HSlist)); - *eng2->left = *engine->left; - *eng2->right = *engine->right; + eng2->stack = h_arena_malloc(engine->tarena, sizeof(HSlist)); + *eng2->stack = *engine->stack; eng2->arena = engine->arena; eng2->tarena = engine->tarena; @@ -54,7 +53,7 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse HSlist *engines = h_slist_new(tarena); - h_slist_push(engines, h_lrengine_new(arena, tarena, table)); + h_slist_push(engines, h_lrengine_new(arena, tarena, table, stream)); HParseResult *result = NULL; while(result == NULL && !h_slist_empty(engines)) { @@ -75,7 +74,7 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* continue; } - const HLRAction *action = h_lrengine_action(engine, stream); + const HLRAction *action = h_lrengine_action(engine); // fork engine on conflicts if(action && action->type == HLR_CONFLICT) { @@ -120,8 +119,6 @@ HParserBackendVTable h__glr_backend_vtable = { // XXX TODO -// - eliminate right stack by always doing a shift after reduce -// (shift should always follow reduce because rightmost) // - split tables into // - one mapping input bytes to actions (shift or reduce or conflict) // - one mapping reduced-to lhs nonterminals to shift states diff --git a/src/backends/lr.c b/src/backends/lr.c index 2603ff2..f33aab8 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -202,65 +202,64 @@ h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) } } -HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table) +HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, + const HInputStream *stream) { HLREngine *engine = h_arena_malloc(tarena, sizeof(HLREngine)); engine->table = table; engine->state = 0; engine->run = true; - engine->left = h_slist_new(tarena); - engine->right = h_slist_new(tarena); + engine->stack = h_slist_new(tarena); + engine->input = *stream; engine->arena = arena; engine->tarena = tarena; return engine; } -const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream) +const HLRAction *h_lrengine_action(const HLREngine *engine) { - HSlist *right = engine->right; - HArena *arena = engine->arena; HArena *tarena = engine->tarena; - // make sure there is input on the right stack - if(h_slist_empty(right)) { - // XXX use statically-allocated terminal symbols - HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); - HParsedToken *v; + // XXX use statically-allocated terminal symbols + HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); - uint8_t c = h_read_bits(stream, 8, false); + HInputStream lookahead = engine->input; + uint8_t c = h_read_bits(&lookahead, 8, false); - if(stream->overrun) { // end of input - x->type = HCF_END; - v = NULL; - } else { - x->type = HCF_CHAR; - x->chr = c; - v = h_arena_malloc(arena, sizeof(HParsedToken)); - v->token_type = TT_UINT; - v->uint = c; - } - - h_slist_push(right, v); - h_slist_push(right, x); + if(lookahead.overrun) { // end of input + x->type = HCF_END; + } else { + x->type = HCF_CHAR; + x->chr = c; } - // peek at input symbol on the right side - HCFChoice *symbol = right->head->elem; + return h_lr_lookup(engine->table, engine->state, x); +} - // table lookup - const HLRAction *action = h_lr_lookup(engine->table, engine->state, symbol); +static HParsedToken *consume_input(HLREngine *engine) +{ + HParsedToken *v; - return action; + uint8_t c = h_read_bits(&engine->input, 8, false); + + if(engine->input.overrun) { // end of input + v = NULL; + } else { + v = h_arena_malloc(engine->arena, sizeof(HParsedToken)); + v->token_type = TT_UINT; + v->uint = c; + } + + return v; } // run LR parser for one round; returns false when finished static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) { // short-hand names - HSlist *left = engine->left; - HSlist *right = engine->right; + HSlist *stack = engine->stack; HArena *arena = engine->arena; HArena *tarena = engine->tarena; @@ -278,11 +277,11 @@ static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) value->token_type = TT_SEQUENCE; value->seq = h_carray_new_sized(arena, len); - // pull values off the left stack, rewinding state accordingly + // pull values off the stack, rewinding state accordingly HParsedToken *v = NULL; for(size_t i=0; istate = (uintptr_t)h_slist_drop(left); + v = h_slist_drop(stack); + engine->state = (uintptr_t)h_slist_drop(stack); // collect values in result sequence value->seq->elements[len-1-i] = v; @@ -315,17 +314,17 @@ static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) assert(shift->type == HLR_SHIFT); // piggy-back the shift right here, never touching the input - h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_push(left, value); + h_slist_push(stack, (void *)(uintptr_t)engine->state); + h_slist_push(stack, value); engine->state = shift->nextstate; if(symbol == engine->table->start) return false; // reduced to start symbol; accept! } else { assert(action->type == HLR_SHIFT); - h_slist_push(left, (void *)(uintptr_t)engine->state); - h_slist_drop(right); // symbol (discard) - h_slist_push(left, h_slist_drop(right)); // semantic value + HParsedToken *value = consume_input(engine); + h_slist_push(stack, (void *)(uintptr_t)engine->state); + h_slist_push(stack, value); engine->state = action->nextstate; } @@ -341,9 +340,9 @@ void h_lrengine_step(HLREngine *engine, const HLRAction *action) HParseResult *h_lrengine_result(HLREngine *engine) { // parsing was successful iff after a shift the engine is back in state 0 - if(engine->state == 0 && !h_slist_empty(engine->left)) { + if(engine->state == 0 && !h_slist_empty(engine->stack)) { // on top of the stack is the start symbol's semantic value - HParsedToken *tok = engine->left->head->elem; + HParsedToken *tok = engine->stack->head->elem; return make_result(engine->arena, tok); } else { return NULL; @@ -358,11 +357,11 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - HLREngine *engine = h_lrengine_new(arena, tarena, table); + HLREngine *engine = h_lrengine_new(arena, tarena, table, stream); // iterate engine to completion while(engine->run) - h_lrengine_step(engine, h_lrengine_action(engine, stream)); + h_lrengine_step(engine, h_lrengine_action(engine)); HParseResult *result = h_lrengine_result(engine); if(!result) diff --git a/src/backends/lr.h b/src/backends/lr.h index 5e2f032..f76bd33 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -70,11 +70,8 @@ typedef struct HLREngine_ { size_t state; bool run; - // stack layout: - // on the left stack, we put pairs: (saved state, semantic value) - // on the right stack, we put pairs: (symbol, semantic value) - HSlist *left; // left stack; reductions happen here - HSlist *right; // right stack; input appears here + HSlist *stack; // holds pairs: (saved state, semantic value) + HInputStream input; HArena *arena; // will hold the results HArena *tarena; // tmp, deleted after parse @@ -108,7 +105,8 @@ HLRItem *h_lritem_new(HArena *a, HCFChoice *lhs, HCFChoice **rhs, size_t mark); HLRState *h_lrstate_new(HArena *arena); HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows); void h_lrtable_free(HLRTable *table); -HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table); +HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, + const HInputStream *stream); HLRAction *h_reduce_action(HArena *arena, const HLRItem *item); HLRAction *h_shift_action(HArena *arena, size_t nextstate); HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new); @@ -128,7 +126,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); void h_lalr_free(HParser *parser); const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); -const HLRAction *h_lrengine_action(HLREngine *engine, HInputStream *stream); +const HLRAction *h_lrengine_action(const HLREngine *engine); void h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); From 534a29b7ba6c30619954de2c20b4c83202aa2878 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 20:21:40 +0200 Subject: [PATCH 18/27] split h_lr_lookup into two (yet identical) functions --- src/backends/lr.c | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index f33aab8..cdd2a35 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -190,18 +190,6 @@ HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new) /* LR driver */ -const HLRAction * -h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) -{ - assert(state < table->nrows); - if(table->forall[state]) { - assert(h_hashtable_empty(table->rows[state])); // that would be a conflict - return table->forall[state]; - } else { - return h_hashtable_get(table->rows[state], symbol); - } -} - HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, const HInputStream *stream) { @@ -218,6 +206,36 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, return engine; } +static const HLRAction * +terminal_lookup(const HLREngine *engine, const HCFChoice *symbol) +{ + const HLRTable *table = engine->table; + size_t state = engine->state; + + assert(state < table->nrows); + if(table->forall[state]) { + assert(h_hashtable_empty(table->rows[state])); // that would be a conflict + return table->forall[state]; + } else { + return h_hashtable_get(table->rows[state], symbol); + } +} + +static const HLRAction * +nonterminal_lookup(const HLREngine *engine, const HCFChoice *symbol) +{ + const HLRTable *table = engine->table; + size_t state = engine->state; + + assert(state < table->nrows); + if(table->forall[state]) { + assert(h_hashtable_empty(table->rows[state])); // that would be a conflict + return table->forall[state]; + } else { + return h_hashtable_get(table->rows[state], symbol); + } +} + const HLRAction *h_lrengine_action(const HLREngine *engine) { HArena *tarena = engine->tarena; @@ -235,7 +253,7 @@ const HLRAction *h_lrengine_action(const HLREngine *engine) x->chr = c; } - return h_lr_lookup(engine->table, engine->state, x); + return terminal_lookup(engine, x); } static HParsedToken *consume_input(HLREngine *engine) @@ -308,7 +326,7 @@ static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we // use LALR tables, because only terminal symbols (lookahead) get reduces. - const HLRAction *shift = h_lr_lookup(engine->table, engine->state, symbol); + const HLRAction *shift = nonterminal_lookup(engine, symbol); if(shift == NULL) return false; // parse error assert(shift->type == HLR_SHIFT); From d67e12a825697290d3d41f59b9aba2c2fc0d3112 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 20:52:27 +0200 Subject: [PATCH 19/27] better factor out lr table writes --- src/backends/lalr.c | 8 ++++---- src/backends/lr.h | 2 -- src/backends/lr0.c | 13 ++++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 698b106..39c4afd 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -130,10 +130,10 @@ static inline bool has_conflicts(HLRTable *table) return !h_slist_empty(table->inadeq); } -// place a new entry in tbl; records conflicts in tbl->inadeq +// place a new terminal entry in tbl; records conflicts in tbl->inadeq // returns 0 on success, -1 on conflict // ignores forall entries -int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) +static int terminal_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) { HLRAction *prev = h_hashtable_get(tbl->rows[state], x); if(prev && prev != action) { @@ -257,7 +257,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) if(fs->end_branch) { HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); terminal->type = HCF_END; - if(h_lrtable_put(table, state, terminal, action) < 0) + if(terminal_put(table, state, terminal, action) < 0) inadeq = true; } H_FOREACH(fs->char_branches, void *key, HStringMap *m) @@ -268,7 +268,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) terminal->type = HCF_CHAR; terminal->chr = key_char((HCharKey)key); - if(h_lrtable_put(table, state, terminal, action) < 0) + if(terminal_put(table, state, terminal, action) < 0) inadeq = true; H_END_FOREACH // lookahead character } H_END_FOREACH // enhanced production diff --git a/src/backends/lr.h b/src/backends/lr.h index f76bd33..ee0c1f3 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -120,12 +120,10 @@ HHashValue h_hash_transition(const void *p); HLRDFA *h_lr0_dfa(HCFGrammar *g); HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa); -int h_lrtable_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action); int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); void h_lalr_free(HParser *parser); -const HLRAction *h_lr_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol); const HLRAction *h_lrengine_action(const HLREngine *engine); void h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); diff --git a/src/backends/lr0.c b/src/backends/lr0.c index 9f350b6..1bd63e5 100644 --- a/src/backends/lr0.c +++ b/src/backends/lr0.c @@ -161,6 +161,14 @@ HLRDFA *h_lr0_dfa(HCFGrammar *g) /* LR(0) table generation */ +static inline +void put_shift(HLRTable *table, size_t state, const HCFChoice *symbol, + size_t nextstate) +{ + HLRAction *action = h_shift_action(table->arena, nextstate); + h_hashtable_put(table->rows[state], symbol, action); +} + HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) { HAllocator *mm__ = g->mm__; @@ -174,15 +182,14 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // add dummy shift entry for the start symbol so h_lrengine_step can always // find a shift. // NB: nextstate=0 is used for the "victory condition" by h_lrengine_result. - h_hashtable_put(table->rows[0], g->start, h_shift_action(arena, 0)); + put_shift(table, 0, g->start, 0); // add shift entries for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { // for each transition x-A->y, add "shift, goto y" to table entry (x,A) HLRTransition *t = x->elem; - HLRAction *action = h_shift_action(arena, t->to); - h_hashtable_put(table->rows[t->from], t->symbol, action); + put_shift(table, t->from, t->symbol, t->to); } // add reduce entries, record inadequate states From 853e1fba4607aa5cba8fe503401312330b54d08d Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 22:55:04 +0200 Subject: [PATCH 20/27] split LR table representation by key type (terminals/nonterminals) --- src/backends/glr.c | 5 --- src/backends/lalr.c | 88 +++++++++++++++++++++++++++++---------------- src/backends/lr.c | 52 +++++++++++++++++++-------- src/backends/lr.h | 6 ++-- src/backends/lr0.c | 16 +++++++-- src/cfgrammar.c | 54 ++++++++++++++++++++-------- src/cfgrammar.h | 3 ++ 7 files changed, 156 insertions(+), 68 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 7a5f8f5..864ecaa 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -119,11 +119,6 @@ HParserBackendVTable h__glr_backend_vtable = { // XXX TODO -// - split tables into -// - one mapping input bytes to actions (shift or reduce or conflict) -// - one mapping reduced-to lhs nonterminals to shift states -// - can there still be conflicts here? -// - use HStringMap to represent lookahead sets and the "piggyback" table // - implement engine merging // - triggered when two enter the same state // - old stacks (/engines?) saved diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 39c4afd..242988e 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -13,9 +13,23 @@ static inline size_t seqsize(void *p_) return n+1; } +static HLRAction * +lrtable_lookup(const HLRTable *table, size_t state, const HCFChoice *symbol) +{ + switch(symbol->type) { + case HCF_END: + return table->tmap[state]->end_branch; + case HCF_CHAR: + return h_stringmap_get(table->tmap[state], &symbol->chr, 1, false); + default: + // nonterminal case + return h_hashtable_get(table->ntmap[state], symbol); + } +} + static size_t follow_transition(const HLRTable *table, size_t x, HCFChoice *A) { - HLRAction *action = h_hashtable_get(table->rows[x], A); + HLRAction *action = lrtable_lookup(table, x, A); assert(action != NULL); assert(action->type == HLR_SHIFT); return action->nextstate; @@ -130,21 +144,48 @@ static inline bool has_conflicts(HLRTable *table) return !h_slist_empty(table->inadeq); } -// place a new terminal entry in tbl; records conflicts in tbl->inadeq +// for each lookahead symbol (fs), put action into tmap // returns 0 on success, -1 on conflict // ignores forall entries -static int terminal_put(HLRTable *tbl, size_t state, HCFChoice *x, HLRAction *action) +static int terminals_put(HStringMap *tmap, const HStringMap *fs, HLRAction *action) { - HLRAction *prev = h_hashtable_get(tbl->rows[state], x); - if(prev && prev != action) { - // conflict - action = h_lr_conflict(tbl->arena, prev, action); - h_hashtable_put(tbl->rows[state], x, action); - return -1; - } else { - h_hashtable_put(tbl->rows[state], x, action); - return 0; + int ret = 0; + + if(fs->epsilon_branch) { + HLRAction *prev = tmap->epsilon_branch; + if(prev && prev != action) { + // conflict + tmap->epsilon_branch = h_lr_conflict(tmap->arena, prev, action); + ret = -1; + } else { + tmap->epsilon_branch = action; + } } + + if(fs->end_branch) { + HLRAction *prev = tmap->end_branch; + if(prev && prev != action) { + // conflict + tmap->end_branch = h_lr_conflict(tmap->arena, prev, action); + ret = -1; + } else { + tmap->end_branch = action; + } + } + + H_FOREACH(fs->char_branches, void *key, HStringMap *fs_) + HStringMap *tmap_ = h_hashtable_get(tmap->char_branches, key); + + if(!tmap_) { + tmap_ = h_stringmap_new(tmap->arena); + h_hashtable_put(tmap->char_branches, key, tmap_); + } + + if(terminals_put(tmap_, fs_, action) < 0) + ret = -1; + H_END_FOREACH + + return ret; } // check whether a sequence of enhanced-grammar symbols (p) matches the given @@ -254,23 +295,8 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) assert(!h_stringmap_empty(fs)); // for each lookahead symbol, put action into table cell - if(fs->end_branch) { - HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); - terminal->type = HCF_END; - if(terminal_put(table, state, terminal, action) < 0) - inadeq = true; - } - H_FOREACH(fs->char_branches, void *key, HStringMap *m) - if(!m->epsilon_branch) - continue; - - HCFChoice *terminal = h_arena_malloc(arena, sizeof(HCFChoice)); - terminal->type = HCF_CHAR; - terminal->chr = key_char((HCharKey)key); - - if(terminal_put(table, state, terminal, action) < 0) - inadeq = true; - H_END_FOREACH // lookahead character + if(terminals_put(table->tmap[state], fs, action) < 0) + inadeq = true; } H_END_FOREACH // enhanced production H_END_FOREACH // reducible item @@ -306,6 +332,8 @@ HParserBackendVTable h__lalr_backend_vtable = { // dummy! int test_lalr(void) { + HAllocator *mm__ = &system_allocator; + /* E -> E '-' T | T @@ -321,7 +349,7 @@ int test_lalr(void) HParser *p = E; printf("\n==== G R A M M A R ====\n"); - HCFGrammar *g = h_cfgrammar(&system_allocator, p); + HCFGrammar *g = h_cfgrammar_(mm__, augment(mm__, p)); if(g == NULL) { fprintf(stderr, "h_cfgrammar failed\n"); return 1; diff --git a/src/backends/lr.c b/src/backends/lr.c index cdd2a35..66a76b7 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -1,4 +1,5 @@ #include +#include #include "../parsers/parser_internal.h" #include "lr.h" @@ -118,14 +119,16 @@ HLRTable *h_lrtable_new(HAllocator *mm__, size_t nrows) HLRTable *ret = h_new(HLRTable, 1); ret->nrows = nrows; - ret->rows = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->ntmap = h_arena_malloc(arena, nrows * sizeof(HHashTable *)); + ret->tmap = h_arena_malloc(arena, nrows * sizeof(HStringMap *)); ret->forall = h_arena_malloc(arena, nrows * sizeof(HLRAction *)); ret->inadeq = h_slist_new(arena); ret->arena = arena; ret->mm__ = mm__; for(size_t i=0; irows[i] = h_hashtable_new(arena, h_eq_symbol, h_hash_symbol); + ret->ntmap[i] = h_hashtable_new(arena, h_eq_symbol, h_hash_symbol); + ret->tmap[i] = h_stringmap_new(arena); ret->forall[i] = NULL; } @@ -186,6 +189,12 @@ HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new) return action; } +bool h_lrtable_row_empty(const HLRTable *table, size_t i) +{ + return (h_hashtable_empty(table->ntmap[i]) + && h_stringmap_empty(table->tmap[i])); +} + /* LR driver */ @@ -214,10 +223,14 @@ terminal_lookup(const HLREngine *engine, const HCFChoice *symbol) assert(state < table->nrows); if(table->forall[state]) { - assert(h_hashtable_empty(table->rows[state])); // that would be a conflict + assert(h_lrtable_row_empty(table, state)); // that would be a conflict return table->forall[state]; } else { - return h_hashtable_get(table->rows[state], symbol); + // XXX use the lookahead stream directly here (cf. llk) + if(symbol->type == HCF_END) + return table->tmap[state]->end_branch; + else + return h_stringmap_get(table->tmap[state], &symbol->chr, 1, false); } } @@ -228,12 +241,9 @@ nonterminal_lookup(const HLREngine *engine, const HCFChoice *symbol) size_t state = engine->state; assert(state < table->nrows); - if(table->forall[state]) { - assert(h_hashtable_empty(table->rows[state])); // that would be a conflict - return table->forall[state]; - } else { - return h_hashtable_get(table->rows[state], symbol); - } + assert(!table->forall[state]); // contains only reduce entries + // we are only looking for shifts + return h_hashtable_get(table->ntmap[state], symbol); } const HLRAction *h_lrengine_action(const HLREngine *engine) @@ -500,6 +510,19 @@ void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) } } +static void valprint_lraction(FILE *file, void *env, void *val) +{ + const HLRAction *action = val; + const HCFGrammar *grammar = env; + pprint_lraction(file, grammar, action); +} + +static void pprint_lrtable_terminals(FILE *file, const HCFGrammar *g, + const HStringMap *map) +{ + h_pprint_stringmap(file, ' ', valprint_lraction, (void *)g, map); +} + void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, unsigned int indent) { @@ -507,18 +530,19 @@ void h_pprint_lrtable(FILE *f, const HCFGrammar *g, const HLRTable *table, for(unsigned int j=0; jforall[i]) { - fputs(" - ", f); + fputc(' ', f); pprint_lraction(f, g, table->forall[i]); - fputs(" -", f); - if(!h_hashtable_empty(table->rows[i])) + if(!h_lrtable_row_empty(table, i)) fputs(" !!", f); } - H_FOREACH(table->rows[i], HCFChoice *symbol, HLRAction *action) + H_FOREACH(table->ntmap[i], HCFChoice *symbol, HLRAction *action) fputc(' ', f); // separator h_pprint_symbol(f, g, symbol); fputc(':', f); pprint_lraction(f, g, action); H_END_FOREACH + fputc(' ', f); // separator + pprint_lrtable_terminals(f, g, table->tmap[i]); fputc('\n', f); } diff --git a/src/backends/lr.h b/src/backends/lr.h index ee0c1f3..ca8418e 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -48,8 +48,9 @@ typedef struct HLRAction_ { } HLRAction; typedef struct HLRTable_ { - size_t nrows; - HHashTable **rows; // map symbols to HLRActions + size_t nrows; // dimension of the pointer arrays below + HHashTable **ntmap; // map nonterminal symbols to HLRActions, per row + HStringMap **tmap; // map lookahead strings to HLRActions, per row HLRAction **forall; // shortcut to set an action for an entire row HCFChoice *start; // start symbol HSlist *inadeq; // indices of any inadequate states @@ -110,6 +111,7 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, HLRAction *h_reduce_action(HArena *arena, const HLRItem *item); HLRAction *h_shift_action(HArena *arena, size_t nextstate); HLRAction *h_lr_conflict(HArena *arena, HLRAction *action, HLRAction *new); +bool h_lrtable_row_empty(const HLRTable *table, size_t i); bool h_eq_symbol(const void *p, const void *q); bool h_eq_lr_itemset(const void *p, const void *q); diff --git a/src/backends/lr0.c b/src/backends/lr0.c index 1bd63e5..5add53a 100644 --- a/src/backends/lr0.c +++ b/src/backends/lr0.c @@ -166,7 +166,18 @@ void put_shift(HLRTable *table, size_t state, const HCFChoice *symbol, size_t nextstate) { HLRAction *action = h_shift_action(table->arena, nextstate); - h_hashtable_put(table->rows[state], symbol, action); + + switch(symbol->type) { + case HCF_END: + h_stringmap_put_end(table->tmap[state], action); + break; + case HCF_CHAR: + h_stringmap_put_char(table->tmap[state], symbol->chr, action); + break; + default: + // nonterminal case + h_hashtable_put(table->ntmap[state], symbol, action); + } } HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) @@ -210,7 +221,8 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // check for shift/reduce conflict with other entries // NOTE: these are not recorded as HLR_CONFLICTs at this point - if(!h_hashtable_empty(table->rows[i])) + + if(!h_lrtable_row_empty(table, i)) inadeq = true; } H_END_FOREACH diff --git a/src/cfgrammar.c b/src/cfgrammar.c index 199ef5f..b01c44c 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -813,27 +813,43 @@ void h_pprint_symbolset(FILE *file, const HCFGrammar *g, const HHashSet *set, in #define BUFSIZE 512 static bool -pprint_stringset_elems(FILE *file, bool first, char *prefix, size_t n, - const HStringMap *set) +pprint_stringmap_elems(FILE *file, bool first, char *prefix, size_t n, char sep, + void (*valprint)(FILE *f, void *env, void *val), void *env, + const HStringMap *map) { assert(n < BUFSIZE-4); - if(set->epsilon_branch) { - if(!first) fputc(',', file); first=false; - if(n==0) - fputs("''", file); - else + if(map->epsilon_branch) { + if(!first) fputc(sep, file); first=false; + if(n==0) { + fputs("\"\"", file); + } else { + fputs("\"", file); fwrite(prefix, 1, n, file); + fputs("\"", file); + } + + if(valprint) { + fputc(':', file); + valprint(file, env, map->epsilon_branch); + } } - if(set->end_branch) { - if(!first) fputc(',', file); first=false; + if(map->end_branch) { + if(!first) fputs(",\"", file); first=false; + if(n>0) fputs("\"\"", file); fwrite(prefix, 1, n, file); - fputc('$', file); + if(n>0) fputs("\"\"", file); + fputs("$", file); + + if(valprint) { + fputc(':', file); + valprint(file, env, map->end_branch); + } } - // iterate over set->char_branches - HHashTable *ht = set->char_branches; + // iterate over map->char_branches + HHashTable *ht = map->char_branches; size_t i; HHashTableEntry *hte; for(i=0; i < ht->capacity; i++) { @@ -859,20 +875,28 @@ pprint_stringset_elems(FILE *file, bool first, char *prefix, size_t n, n_ += sprintf(prefix+n_, "\\x%.2X", c); } - first = pprint_stringset_elems(file, first, prefix, n_, ends); + first = pprint_stringmap_elems(file, first, prefix, n_, + sep, valprint, env, ends); } } return first; } +void h_pprint_stringmap(FILE *file, char sep, + void (*valprint)(FILE *f, void *env, void *val), void *env, + const HStringMap *map) +{ + char buf[BUFSIZE]; + pprint_stringmap_elems(file, true, buf, 0, sep, valprint, env, map); +} + void h_pprint_stringset(FILE *file, const HStringMap *set, int indent) { int j; for(j=0; j Date: Fri, 21 Jun 2013 23:22:07 +0200 Subject: [PATCH 21/27] use lookahead stream directly for LR (terminal) lookup --- src/backends/llk.c | 27 +++------------------------ src/backends/lr.c | 25 +++---------------------- src/cfgrammar.c | 25 +++++++++++++++++++++++++ src/cfgrammar.h | 1 + 4 files changed, 32 insertions(+), 46 deletions(-) diff --git a/src/backends/llk.c b/src/backends/llk.c index 4f73c46..c0cf6af 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -21,7 +21,7 @@ typedef struct HLLkTable_ { /* Interface to look up an entry in the parse table. */ const HCFSequence *h_llk_lookup(const HLLkTable *table, const HCFChoice *x, - HInputStream lookahead) + const HInputStream *stream) { const HStringMap *row = h_hashtable_get(table->rows, x); assert(row != NULL); // the table should have one row for each nonterminal @@ -29,28 +29,7 @@ const HCFSequence *h_llk_lookup(const HLLkTable *table, const HCFChoice *x, assert(!row->epsilon_branch); // would match without looking at the input // XXX cases where this could be useful? - const HStringMap *m = row; - while(m) { - if(m->epsilon_branch) { // input matched - // assert: another lookahead would not bring a more specific match. - // this is for the table generator to ensure. - return m->epsilon_branch; - } - - // note the lookahead stream is passed by value, i.e. a copy. - // reading bits from it does not consume them from the real input. - uint8_t c = h_read_bits(&lookahead, 8, false); - - if(lookahead.overrun) { // end of input - // XXX assumption of byte-wise grammar and input - return m->end_branch; - } - - // no match yet, descend - m = h_stringmap_get_char(m, c); - } - - return NULL; + return h_stringmap_get_lookahead(row, *stream); } /* Allocate a new parse table. */ @@ -321,7 +300,7 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* seq = h_carray_new(arena); // look up applicable production in parse table - const HCFSequence *p = h_llk_lookup(table, x, *stream); + const HCFSequence *p = h_llk_lookup(table, x, stream); if(p == NULL) goto no_parse; diff --git a/src/backends/lr.c b/src/backends/lr.c index 66a76b7..ca45582 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -216,7 +216,7 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, } static const HLRAction * -terminal_lookup(const HLREngine *engine, const HCFChoice *symbol) +terminal_lookup(const HLREngine *engine, const HInputStream *stream) { const HLRTable *table = engine->table; size_t state = engine->state; @@ -226,11 +226,7 @@ terminal_lookup(const HLREngine *engine, const HCFChoice *symbol) assert(h_lrtable_row_empty(table, state)); // that would be a conflict return table->forall[state]; } else { - // XXX use the lookahead stream directly here (cf. llk) - if(symbol->type == HCF_END) - return table->tmap[state]->end_branch; - else - return h_stringmap_get(table->tmap[state], &symbol->chr, 1, false); + return h_stringmap_get_lookahead(table->tmap[state], *stream); } } @@ -248,22 +244,7 @@ nonterminal_lookup(const HLREngine *engine, const HCFChoice *symbol) const HLRAction *h_lrengine_action(const HLREngine *engine) { - HArena *tarena = engine->tarena; - - // XXX use statically-allocated terminal symbols - HCFChoice *x = h_arena_malloc(tarena, sizeof(HCFChoice)); - - HInputStream lookahead = engine->input; - uint8_t c = h_read_bits(&lookahead, 8, false); - - if(lookahead.overrun) { // end of input - x->type = HCF_END; - } else { - x->type = HCF_CHAR; - x->chr = c; - } - - return terminal_lookup(engine, x); + return terminal_lookup(engine, &engine->input); } static HParsedToken *consume_input(HLREngine *engine) diff --git a/src/cfgrammar.c b/src/cfgrammar.c index b01c44c..196d9d3 100644 --- a/src/cfgrammar.c +++ b/src/cfgrammar.c @@ -321,6 +321,31 @@ void *h_stringmap_get(const HStringMap *m, const uint8_t *str, size_t n, bool en return m->epsilon_branch; } +void *h_stringmap_get_lookahead(const HStringMap *m, HInputStream lookahead) +{ + while(m) { + if(m->epsilon_branch) { // input matched + // assert: another lookahead would not bring a more specific match. + // this is for the table generator to ensure. (LLk) + return m->epsilon_branch; + } + + // note the lookahead stream is passed by value, i.e. a copy. + // reading bits from it does not consume them from the real input. + uint8_t c = h_read_bits(&lookahead, 8, false); + + if(lookahead.overrun) { // end of input + // XXX assumption of byte-wise grammar and input + return m->end_branch; + } + + // no match yet, descend + m = h_stringmap_get_char(m, c); + } + + return NULL; +} + bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end) { return (h_stringmap_get(m, str, n, end) != NULL); diff --git a/src/cfgrammar.h b/src/cfgrammar.h index 1f52bdd..193f8ca 100644 --- a/src/cfgrammar.h +++ b/src/cfgrammar.h @@ -47,6 +47,7 @@ void h_stringmap_put_char(HStringMap *m, uint8_t c, void *v); void h_stringmap_update(HStringMap *m, const HStringMap *n); void h_stringmap_replace(HStringMap *m, void *old, void *new); void *h_stringmap_get(const HStringMap *m, const uint8_t *str, size_t n, bool end); +void *h_stringmap_get_lookahead(const HStringMap *m, HInputStream lookahead); bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end); bool h_stringmap_present_epsilon(const HStringMap *m); bool h_stringmap_empty(const HStringMap *m); From 34c6d868b9823f9c55ec9398bf33dd734dfc6bda Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 23:47:22 +0200 Subject: [PATCH 22/27] commentation --- src/backends/glr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 864ecaa..f7b6f8b 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -32,7 +32,7 @@ HLREngine *fork_engine(const HLREngine *engine) eng2->input = engine->input; // shallow-copy the stack - // this works because h_slist_push and h_slist_pop never modify + // this works because h_slist_push and h_slist_drop never modify // the underlying structure of HSlistNodes, only the head pointer. // in fact, this gives us prefix sharing for free. eng2->stack = h_arena_malloc(engine->tarena, sizeof(HSlist)); From 8bc3b93e959619708d07f468e3a7dd5037bf28c5 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 21 Jun 2013 23:53:47 +0200 Subject: [PATCH 23/27] correctly use augmented grammar for test_glr() output --- src/backends/glr.c | 4 +++- src/backends/lalr.c | 6 +++--- src/backends/lr.h | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index f7b6f8b..82ad5c0 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -129,6 +129,8 @@ HParserBackendVTable h__glr_backend_vtable = { // dummy! int test_glr(void) { + HAllocator *mm__ = &system_allocator; + /* E -> E '+' E | 'd' @@ -141,7 +143,7 @@ int test_glr(void) HParser *p = E; printf("\n==== G R A M M A R ====\n"); - HCFGrammar *g = h_cfgrammar(&system_allocator, p); + HCFGrammar *g = h_cfgrammar_(mm__, h_desugar_augmented(mm__, p)); if(g == NULL) { fprintf(stderr, "h_cfgrammar failed\n"); return 1; diff --git a/src/backends/lalr.c b/src/backends/lalr.c index 242988e..93becf3 100644 --- a/src/backends/lalr.c +++ b/src/backends/lalr.c @@ -207,7 +207,7 @@ static bool match_production(HLREnhGrammar *eg, HCFChoice **p, // desugar parser with a fresh start symbol // this guarantees that the start symbol will not occur in any productions -static HCFChoice *augment(HAllocator *mm__, HParser *parser) +HCFChoice *h_desugar_augmented(HAllocator *mm__, HParser *parser) { HCFChoice *augmented = h_new(HCFChoice, 1); @@ -231,7 +231,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params) // build LR(0) table // if necessary, resolve conflicts "by conversion to SLR" - HCFGrammar *g = h_cfgrammar_(mm__, augment(mm__, parser)); + HCFGrammar *g = h_cfgrammar_(mm__, h_desugar_augmented(mm__, parser)); if(g == NULL) // backend not suitable (language not context-free) return -1; @@ -349,7 +349,7 @@ int test_lalr(void) HParser *p = E; printf("\n==== G R A M M A R ====\n"); - HCFGrammar *g = h_cfgrammar_(mm__, augment(mm__, p)); + HCFGrammar *g = h_cfgrammar_(mm__, h_desugar_augmented(mm__, p)); if(g == NULL) { fprintf(stderr, "h_cfgrammar failed\n"); return 1; diff --git a/src/backends/lr.h b/src/backends/lr.h index ca8418e..1158542 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -123,6 +123,7 @@ HHashValue h_hash_transition(const void *p); HLRDFA *h_lr0_dfa(HCFGrammar *g); HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa); +HCFChoice *h_desugar_augmented(HAllocator *mm__, HParser *parser); int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); void h_lalr_free(HParser *parser); From bf9c9b5f7a7ffa5f81028c3c685afb7ba958a2b9 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 22 Jun 2013 12:51:25 +0200 Subject: [PATCH 24/27] refactoring in preparation for engine merging --- src/backends/glr.c | 77 +++++++++++++++++++++++++++++++--------------- src/backends/lr.c | 1 + src/backends/lr.h | 6 ++-- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 82ad5c0..2978e62 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -43,6 +43,44 @@ HLREngine *fork_engine(const HLREngine *engine) return eng2; } +static void stow_engine(HSlist *engines, HLREngine *engine) +{ + // XXX switch to one engine per state, and do the merge here + h_slist_push(engines, engine); +} + +static const HLRAction *handle_conflict(HSlist *engines, const HLREngine *engine, + const HSlist *branches) +{ + // there should be at least two conflicting actions + assert(branches->head); + assert(branches->head->next); // this is just a consistency check + + // fork a new engine for all but the first action + for(HSlistNode *x=branches->head->next; x; x=x->next) { + HLRAction *act = x->elem; + HLREngine *eng = fork_engine(engine); + + // perform one step and add to list + h_lrengine_step(eng, act); + stow_engine(engines, eng); + } + + // return first action for use with original engine + return branches->head->elem; +} + +static HLREngine *handle_demerge(HSlist *engines, HLREngine *engine, + const HLRAction *reduce) +{ + return engine; // XXX + + for(size_t i=0; iproduction.length; i++) { + // XXX if stack hits bottom, demerge + } + // XXX call step and stow on the newly-created engines +} + HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) { HLRTable *table = parser->backend_data; @@ -60,12 +98,11 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* for(HSlistNode **x = &engines->head; *x; ) { HLREngine *engine = (*x)->elem; - // check for terminated engines - if(engine->run) { - x = &(*x)->next; // advance x with no change - } else { - *x = (*x)->next; // advance x, removing the current element + // remove engine from list; it may come back in below + *x = (*x)->next; // advance x, removing the current element + // drop those engines that have terminated + if(!engine->run) { // check for parse success HParseResult *res = h_lrengine_result(engine); if(res) @@ -76,29 +113,19 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* const HLRAction *action = h_lrengine_action(engine); - // fork engine on conflicts - if(action && action->type == HLR_CONFLICT) { - const HSlist *branches = action->branches; - - // there should be at least two conflicting actions - assert(branches->head); - assert(branches->head->next); - - // save first action for use with old engine below - action = branches->head->elem; - - // fork a new engine for all the other actions - for(HSlistNode *x=branches->head->next; x; x=x->next) { - HLRAction *act = x->elem; - HLREngine *eng = fork_engine(engine); - - // perform one step and add to list - h_lrengine_step(eng, act); - h_slist_push(engines, eng); - } + // handle forks and demerges (~> spawn engines) + if(action) { + if(action->type == HLR_CONFLICT) { + // fork engine on conflicts + action = handle_conflict(engines, engine, action->branches); + } else if(action->type == HLR_REDUCE) { + // demerge as needed to ensure that stacks are deep enough + engine = handle_demerge(engines, engine, action); + } } h_lrengine_step(engine, action); + stow_engine(engines, engine); } } diff --git a/src/backends/lr.c b/src/backends/lr.c index ca45582..bb20f71 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -209,6 +209,7 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, engine->run = true; engine->stack = h_slist_new(tarena); engine->input = *stream; + engine->merged = NULL; engine->arena = arena; engine->tarena = tarena; diff --git a/src/backends/lr.h b/src/backends/lr.h index 1158542..5febc24 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -74,8 +74,10 @@ typedef struct HLREngine_ { HSlist *stack; // holds pairs: (saved state, semantic value) HInputStream input; - HArena *arena; // will hold the results - HArena *tarena; // tmp, deleted after parse + HSlist *merged; // saved ancestor engines that merged to form this one + + HArena *arena; // will hold the results + HArena *tarena; // tmp, deleted after parse } HLREngine; From 23afea4b4e6fa00262c0b0cfbd958fa3e9e44093 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 22 Jun 2013 14:21:19 +0200 Subject: [PATCH 25/27] add demerge code path (untested) --- src/backends/glr.c | 55 +++++++++++++++++++++++++++++++++++++++------- src/backends/lr.c | 1 + src/backends/lr.h | 3 ++- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 2978e62..411171b 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -70,15 +70,54 @@ static const HLRAction *handle_conflict(HSlist *engines, const HLREngine *engine return branches->head->elem; } -static HLREngine *handle_demerge(HSlist *engines, HLREngine *engine, - const HLRAction *reduce) +static HSlist *demerge_stack(HSlistNode *bottom, HSlistNode *mp, HSlist *stack) { - return engine; // XXX + HArena *arena = stack->arena; - for(size_t i=0; iproduction.length; i++) { - // XXX if stack hits bottom, demerge + HSlist *ret = h_slist_new(arena); + + // copy the stack from the top + HSlistNode **y = &ret->head; + for(HSlistNode *x=stack->head; x && x!=mp; x=x->next) { + HSlistNode *node = h_arena_malloc(arena, sizeof(HSlistNode)); + node->elem = x->elem; + node->next = NULL; + *y = node; + y = &node->next; } - // XXX call step and stow on the newly-created engines + *y = bottom; // attach the ancestor stack + + return ret; +} + +static void demerge(HSlist *engines, HLREngine *engine, + const HLRAction *action, size_t depth) +{ + // no-op on engines that are not merged + if(!engine->merged) + return; + + HSlistNode *p = engine->stack->head; + for(size_t i=0; imp) { + HLREngine *eng = engine->merged; + eng->stack = demerge_stack(eng->stack->head, engine->mp, engine->stack); + demerge(engines, eng, action, depth-i); + + // call step and stow on restored ancestor + h_lrengine_step(eng, action); + stow_engine(engines, eng); + break; + } + p = p->next; + } +} + +static inline void +handle_demerge(HSlist *engines, HLREngine *engine, const HLRAction *reduce) +{ + demerge(engines, engine, reduce, reduce->production.length); } HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) @@ -119,8 +158,8 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* // fork engine on conflicts action = handle_conflict(engines, engine, action->branches); } else if(action->type == HLR_REDUCE) { - // demerge as needed to ensure that stacks are deep enough - engine = handle_demerge(engines, engine, action); + // demerge/respawn as needed + handle_demerge(engines, engine, action); } } diff --git a/src/backends/lr.c b/src/backends/lr.c index bb20f71..c481d29 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -210,6 +210,7 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, engine->stack = h_slist_new(tarena); engine->input = *stream; engine->merged = NULL; + engine->mp = NULL; engine->arena = arena; engine->tarena = tarena; diff --git a/src/backends/lr.h b/src/backends/lr.h index 5febc24..ab48633 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -74,7 +74,8 @@ typedef struct HLREngine_ { HSlist *stack; // holds pairs: (saved state, semantic value) HInputStream input; - HSlist *merged; // saved ancestor engines that merged to form this one + struct HLREngine_ *merged; // ancestor merged into this engine at mp + HSlistNode *mp; // mergepoint: stack->head at time of merge HArena *arena; // will hold the results HArena *tarena; // tmp, deleted after parse From 67681a119afcb485249946b3ac802761fc81e026 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 22 Jun 2013 17:40:47 +0200 Subject: [PATCH 26/27] finish engine merging --- src/backends/glr.c | 240 +++++++++++++++++++++++++++------------------ src/backends/lr.c | 33 +++---- src/backends/lr.h | 8 +- src/backends/lr0.c | 6 +- 4 files changed, 168 insertions(+), 119 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 411171b..34a5633 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -1,6 +1,9 @@ #include #include "lr.h" +static bool glr_step(HParseResult **result, HLREngine **engines, + HLREngine *engine, const HLRAction *action); + /* GLR compilation (LALR w/o failing on conflict) */ @@ -22,7 +25,86 @@ void h_glr_free(HParser *parser) } -/* GLR driver */ +/* Merging engines (when they converge on the same state) */ + +static HLREngine *lrengine_merge(HLREngine *old, HLREngine *new) +{ + HArena *arena = old->arena; + + HLREngine *ret = h_arena_malloc(arena, sizeof(HLREngine)); + + assert(old->state == new->state); + assert(old->input.input == new->input.input); + + *ret = *old; + ret->stack = h_slist_new(arena); + ret->merged[0] = old; + ret->merged[1] = new; + + return ret; +} + +static HSlist *demerge_stack(HSlistNode *bottom, HSlist *stack) +{ + HArena *arena = stack->arena; + + HSlist *ret = h_slist_new(arena); + + // copy the stack from the top + HSlistNode **y = &ret->head; + for(HSlistNode *x=stack->head; x; x=x->next) { + HSlistNode *node = h_arena_malloc(arena, sizeof(HSlistNode)); + node->elem = x->elem; + node->next = NULL; + *y = node; + y = &node->next; + } + *y = bottom; // attach the ancestor stack + + return ret; +} + +static inline HLREngine *respawn(HLREngine *eng, HSlist *stack) +{ + // NB: this can be a destructive update because an engine is not used for + // anything after it is merged. + eng->stack = demerge_stack(eng->stack->head, stack); + return eng; +} + +static HLREngine * +demerge(HParseResult **result, HLREngine **engines, + HLREngine *engine, const HLRAction *action, size_t depth) +{ + // no-op on engines that are not merged + if(!engine->merged[0]) + return engine; + + HSlistNode *p = engine->stack->head; + for(size_t i=0; imerged[0], engine->stack); + HLREngine *b = respawn(engine->merged[1], engine->stack); + + // continue demerge until final depth reached + a = demerge(result, engines, a, action, depth-i); + b = demerge(result, engines, b, action, depth-i); + + // step and stow one ancestor... + glr_step(result, engines, a, action); + + // ...and return the other + return b; + } + p = p->next; + } + + return engine; // there is enough stack before the merge point +} + + +/* Forking engines (on conflicts */ HLREngine *fork_engine(const HLREngine *engine) { @@ -43,14 +125,9 @@ HLREngine *fork_engine(const HLREngine *engine) return eng2; } -static void stow_engine(HSlist *engines, HLREngine *engine) -{ - // XXX switch to one engine per state, and do the merge here - h_slist_push(engines, engine); -} - -static const HLRAction *handle_conflict(HSlist *engines, const HLREngine *engine, - const HSlist *branches) +static const HLRAction * +handle_conflict(HParseResult **result, HLREngine **engines, + const HLREngine *engine, const HSlist *branches) { // there should be at least two conflicting actions assert(branches->head); @@ -61,63 +138,46 @@ static const HLRAction *handle_conflict(HSlist *engines, const HLREngine *engine HLRAction *act = x->elem; HLREngine *eng = fork_engine(engine); - // perform one step and add to list - h_lrengine_step(eng, act); - stow_engine(engines, eng); + // perform one step and add to engines + glr_step(result, engines, eng, act); } // return first action for use with original engine return branches->head->elem; } -static HSlist *demerge_stack(HSlistNode *bottom, HSlistNode *mp, HSlist *stack) + +/* GLR driver */ + +static bool glr_step(HParseResult **result, HLREngine **engines, + HLREngine *engine, const HLRAction *action) { - HArena *arena = stack->arena; - - HSlist *ret = h_slist_new(arena); - - // copy the stack from the top - HSlistNode **y = &ret->head; - for(HSlistNode *x=stack->head; x && x!=mp; x=x->next) { - HSlistNode *node = h_arena_malloc(arena, sizeof(HSlistNode)); - node->elem = x->elem; - node->next = NULL; - *y = node; - y = &node->next; - } - *y = bottom; // attach the ancestor stack - - return ret; -} - -static void demerge(HSlist *engines, HLREngine *engine, - const HLRAction *action, size_t depth) -{ - // no-op on engines that are not merged - if(!engine->merged) - return; - - HSlistNode *p = engine->stack->head; - for(size_t i=0; imp) { - HLREngine *eng = engine->merged; - eng->stack = demerge_stack(eng->stack->head, engine->mp, engine->stack); - demerge(engines, eng, action, depth-i); - - // call step and stow on restored ancestor - h_lrengine_step(eng, action); - stow_engine(engines, eng); - break; + // handle forks and demerges (~> spawn engines) + if(action) { + if(action->type == HLR_CONFLICT) { + // fork engine on conflicts + action = handle_conflict(result, engines, engine, action->branches); + } else if(action->type == HLR_REDUCE) { + // demerge/respawn as needed + size_t depth = action->production.length; + engine = demerge(result, engines, engine, action, depth); } - p = p->next; } -} -static inline void -handle_demerge(HSlist *engines, HLREngine *engine, const HLRAction *reduce) -{ - demerge(engines, engine, reduce, reduce->production.length); + bool run = h_lrengine_step(engine, action); + + if(run) { + // store engine in the array, merge if necessary + if(engines[engine->state] == NULL) + engines[engine->state] = engine; + else + engines[engine->state] = lrengine_merge(engines[engine->state], engine); + } else if(engine->state == HLR_SUCCESS) { + // save the result + *result = h_lrengine_result(engine); + } + + return run; } HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) @@ -129,43 +189,42 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - HSlist *engines = h_slist_new(tarena); - h_slist_push(engines, h_lrengine_new(arena, tarena, table, stream)); + // allocate engine arrays (can hold one engine per state) + // these are swapped each iteration + HLREngine **engines = h_arena_malloc(tarena, table->nrows * sizeof(HLREngine *)); + HLREngine **engback = h_arena_malloc(tarena, table->nrows * sizeof(HLREngine *)); + + assert(table->nrows > 0); + for(size_t i=0; inrows; i++) { + engines[i] = NULL; + engback[i] = NULL; + } + + // create initial engine + engines[0] = h_lrengine_new(arena, tarena, table, stream); + assert(engines[0]->state == 0); HParseResult *result = NULL; - while(result == NULL && !h_slist_empty(engines)) { - for(HSlistNode **x = &engines->head; *x; ) { - HLREngine *engine = (*x)->elem; - - // remove engine from list; it may come back in below - *x = (*x)->next; // advance x, removing the current element - - // drop those engines that have terminated - if(!engine->run) { - // check for parse success - HParseResult *res = h_lrengine_result(engine); - if(res) - result = res; + size_t engines_left = 1; + while(engines_left && result == NULL) { + engines_left = 0; + for(size_t i=0; inrows; i++) { + HLREngine *engine = engines[i]; + if(engine == NULL) continue; - } + engines[i] = NULL; // cleared for next iteration - const HLRAction *action = h_lrengine_action(engine); - - // handle forks and demerges (~> spawn engines) - if(action) { - if(action->type == HLR_CONFLICT) { - // fork engine on conflicts - action = handle_conflict(engines, engine, action->branches); - } else if(action->type == HLR_REDUCE) { - // demerge/respawn as needed - handle_demerge(engines, engine, action); - } - } - - h_lrengine_step(engine, action); - stow_engine(engines, engine); + // step all engines + bool run = glr_step(&result, engback, engine, h_lrengine_action(engine)); + if(run) + engines_left++; } + + // swap the arrays + HLREngine **tmp = engines; + engines = engback; + engback = tmp; } if(!result) @@ -184,13 +243,6 @@ HParserBackendVTable h__glr_backend_vtable = { -// XXX TODO -// - implement engine merging -// - triggered when two enter the same state -// - old stacks (/engines?) saved -// - new common suffix stack created -// - when rewinding (during reduce), watch for empty stack -> demerge - // dummy! int test_glr(void) diff --git a/src/backends/lr.c b/src/backends/lr.c index c481d29..4c89d19 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -206,11 +206,10 @@ HLREngine *h_lrengine_new(HArena *arena, HArena *tarena, const HLRTable *table, engine->table = table; engine->state = 0; - engine->run = true; engine->stack = h_slist_new(tarena); engine->input = *stream; - engine->merged = NULL; - engine->mp = NULL; + engine->merged[0] = NULL; + engine->merged[1] = NULL; engine->arena = arena; engine->tarena = tarena; @@ -267,7 +266,7 @@ static HParsedToken *consume_input(HLREngine *engine) } // run LR parser for one round; returns false when finished -static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) +bool h_lrengine_step(HLREngine *engine, const HLRAction *action) { // short-hand names HSlist *stack = engine->stack; @@ -329,8 +328,11 @@ static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) h_slist_push(stack, value); engine->state = shift->nextstate; - if(symbol == engine->table->start) - return false; // reduced to start symbol; accept! + // check for success + if(engine->state == HLR_SUCCESS) { + assert(symbol == engine->table->start); + return false; + } } else { assert(action->type == HLR_SHIFT); HParsedToken *value = consume_input(engine); @@ -342,17 +344,12 @@ static bool h_lrengine_step_(HLREngine *engine, const HLRAction *action) return true; } -// run LR parser for one round; sets engine->run -void h_lrengine_step(HLREngine *engine, const HLRAction *action) -{ - engine->run = h_lrengine_step_(engine, action); -} - HParseResult *h_lrengine_result(HLREngine *engine) { - // parsing was successful iff after a shift the engine is back in state 0 - if(engine->state == 0 && !h_slist_empty(engine->stack)) { + // parsing was successful iff the engine reaches the end state + if(engine->state == HLR_SUCCESS) { // on top of the stack is the start symbol's semantic value + assert(!h_slist_empty(engine->stack)); HParsedToken *tok = engine->stack->head->elem; return make_result(engine->arena, tok); } else { @@ -371,8 +368,7 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HLREngine *engine = h_lrengine_new(arena, tarena, table, stream); // iterate engine to completion - while(engine->run) - h_lrengine_step(engine, h_lrengine_action(engine)); + while(h_lrengine_step(engine, h_lrengine_action(engine))); HParseResult *result = h_lrengine_result(engine); if(!result) @@ -464,7 +460,10 @@ void pprint_lraction(FILE *f, const HCFGrammar *g, const HLRAction *action) { switch(action->type) { case HLR_SHIFT: - fprintf(f, "s%lu", action->nextstate); + if(action->nextstate == HLR_SUCCESS) + fputs("s~", f); + else + fprintf(f, "s%lu", action->nextstate); break; case HLR_REDUCE: fputs("r(", f); diff --git a/src/backends/lr.h b/src/backends/lr.h index ab48633..8f1eadd 100644 --- a/src/backends/lr.h +++ b/src/backends/lr.h @@ -69,18 +69,18 @@ typedef struct HLREnhGrammar_ { typedef struct HLREngine_ { const HLRTable *table; size_t state; - bool run; HSlist *stack; // holds pairs: (saved state, semantic value) HInputStream input; - struct HLREngine_ *merged; // ancestor merged into this engine at mp - HSlistNode *mp; // mergepoint: stack->head at time of merge + struct HLREngine_ *merged[2]; // ancestors merged into this engine HArena *arena; // will hold the results HArena *tarena; // tmp, deleted after parse } HLREngine; +#define HLR_SUCCESS ((size_t)~0) // parser end state + // XXX move to internal.h or something // XXX replace other hashtable iterations with this @@ -131,7 +131,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params); void h_lalr_free(HParser *parser); const HLRAction *h_lrengine_action(const HLREngine *engine); -void h_lrengine_step(HLREngine *engine, const HLRAction *action); +bool h_lrengine_step(HLREngine *engine, const HLRAction *action); HParseResult *h_lrengine_result(HLREngine *engine); HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream); diff --git a/src/backends/lr0.c b/src/backends/lr0.c index 5add53a..1c86484 100644 --- a/src/backends/lr0.c +++ b/src/backends/lr0.c @@ -190,10 +190,8 @@ HLRTable *h_lr0_table(HCFGrammar *g, const HLRDFA *dfa) // remember start symbol table->start = g->start; - // add dummy shift entry for the start symbol so h_lrengine_step can always - // find a shift. - // NB: nextstate=0 is used for the "victory condition" by h_lrengine_result. - put_shift(table, 0, g->start, 0); + // shift to the accepting end state for the start symbol + put_shift(table, 0, g->start, HLR_SUCCESS); // add shift entries for(HSlistNode *x = dfa->transitions->head; x; x = x->next) { From 66809ceedad6e4ca4dfd445f166545713a444d0b Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 22 Jun 2013 18:16:41 +0200 Subject: [PATCH 27/27] go back to storing engines in lists --- src/backends/glr.c | 63 +++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/backends/glr.c b/src/backends/glr.c index 34a5633..353d0e6 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -1,7 +1,7 @@ #include #include "lr.h" -static bool glr_step(HParseResult **result, HLREngine **engines, +static bool glr_step(HParseResult **result, HSlist *engines, HLREngine *engine, const HLRAction *action); @@ -73,7 +73,7 @@ static inline HLREngine *respawn(HLREngine *eng, HSlist *stack) } static HLREngine * -demerge(HParseResult **result, HLREngine **engines, +demerge(HParseResult **result, HSlist *engines, HLREngine *engine, const HLRAction *action, size_t depth) { // no-op on engines that are not merged @@ -126,7 +126,7 @@ HLREngine *fork_engine(const HLREngine *engine) } static const HLRAction * -handle_conflict(HParseResult **result, HLREngine **engines, +handle_conflict(HParseResult **result, HSlist *engines, const HLREngine *engine, const HSlist *branches) { // there should be at least two conflicting actions @@ -149,7 +149,7 @@ handle_conflict(HParseResult **result, HLREngine **engines, /* GLR driver */ -static bool glr_step(HParseResult **result, HLREngine **engines, +static bool glr_step(HParseResult **result, HSlist *engines, HLREngine *engine, const HLRAction *action) { // handle forks and demerges (~> spawn engines) @@ -167,11 +167,17 @@ static bool glr_step(HParseResult **result, HLREngine **engines, bool run = h_lrengine_step(engine, action); if(run) { - // store engine in the array, merge if necessary - if(engines[engine->state] == NULL) - engines[engine->state] = engine; - else - engines[engine->state] = lrengine_merge(engines[engine->state], engine); + // store engine in the list, merge if necessary + HSlistNode *x; + for(x=engines->head; x; x=x->next) { + HLREngine *eng = x->elem; + if(eng->state == engine->state) { + x->elem = lrengine_merge(eng, engine); + break; + } + } + if(!x) // no merge happened + h_slist_push(engines, engine); } else if(engine->state == HLR_SUCCESS) { // save the result *result = h_lrengine_result(engine); @@ -189,40 +195,27 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse - // allocate engine arrays (can hold one engine per state) + // allocate engine lists (will hold one engine per state) // these are swapped each iteration - HLREngine **engines = h_arena_malloc(tarena, table->nrows * sizeof(HLREngine *)); - HLREngine **engback = h_arena_malloc(tarena, table->nrows * sizeof(HLREngine *)); - - assert(table->nrows > 0); - for(size_t i=0; inrows; i++) { - engines[i] = NULL; - engback[i] = NULL; - } + HSlist *engines = h_slist_new(tarena); + HSlist *engback = h_slist_new(tarena); // create initial engine - engines[0] = h_lrengine_new(arena, tarena, table, stream); - assert(engines[0]->state == 0); + h_slist_push(engines, h_lrengine_new(arena, tarena, table, stream)); HParseResult *result = NULL; - size_t engines_left = 1; - while(engines_left && result == NULL) { - engines_left = 0; + while(result == NULL && !h_slist_empty(engines)) { + assert(h_slist_empty(engback)); - for(size_t i=0; inrows; i++) { - HLREngine *engine = engines[i]; - if(engine == NULL) - continue; - engines[i] = NULL; // cleared for next iteration - - // step all engines - bool run = glr_step(&result, engback, engine, h_lrengine_action(engine)); - if(run) - engines_left++; + // step all engines + while(!h_slist_empty(engines)) { + HLREngine *engine = h_slist_pop(engines); + const HLRAction *action = h_lrengine_action(engine); + glr_step(&result, engback, engine, action); } - // swap the arrays - HLREngine **tmp = engines; + // swap the lists + HSlist *tmp = engines; engines = engback; engback = tmp; }