emileclarkb/hammer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

note why token position is not set correctly by LL(k) right now

Browse source
This commit is contained in:
Sven M. Hallberg 2015-09-17 00:33:14 +02:00
parent 6094deda96
commit a4179fca02
2 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/backends/llk.c
View file

@ -438,13 +438,12 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
// the top of stack is such that there will be a result... // the top of stack is such that there will be a result...
tok = h_arena_malloc(arena, sizeof(HParsedToken)); tok = h_arena_malloc(arena, sizeof(HParsedToken));
tok->index = stream->pos + stream->index;
tok->bit_offset = stream->bit_offset;
if(x == MARK) { if(x == MARK) {
// hit stack frame boundary... // hit stack frame boundary...
// wrap the accumulated parse result, this sequence is finished // wrap the accumulated parse result, this sequence is finished
tok->token_type = TT_SEQUENCE; tok->token_type = TT_SEQUENCE;
tok->seq = seq; tok->seq = seq;
// XXX would have to set token pos but we've forgotten pos of seq
// recover original nonterminal and result sequence // recover original nonterminal and result sequence
x = h_slist_pop(stack); x = h_slist_pop(stack);
@ -454,6 +453,9 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
else { else {
// x is a terminal or simple charset; match against input // x is a terminal or simple charset; match against input
tok->index = stream->pos + stream->index;
tok->bit_offset = stream->bit_offset;
// consume the input token // consume the input token
uint8_t input = h_read_bits(stream, 8, false); uint8_t input = h_read_bits(stream, 8, false);
@ -500,8 +502,16 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
// 'tok' has been parsed; process it // 'tok' has been parsed; process it
// perform token reshape if indicated // perform token reshape if indicated
if(x->reshape) if(x->reshape) {
tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data); HParsedToken *t = x->reshape(make_result(arena, tok), x->user_data);
if(t) {
t->index = tok->index;
t->bit_offset = tok->bit_offset;
} else {
h_arena_free(arena, tok);
}
tok = t;
}
// call validation and semantic action, if present // call validation and semantic action, if present
if(x->pred && !x->pred(make_result(tarena, tok), x->user_data)) if(x->pred && !x->pred(make_result(tarena, tok), x->user_data))

2
src/t_parser.c
View file

@ -832,7 +832,7 @@ void register_parser_tests(void) {
//g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); //g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec);
g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec);
g_test_add_data_func("/core/parser/llk/result_length", GINT_TO_POINTER(PB_LLk), test_result_length); g_test_add_data_func("/core/parser/llk/result_length", GINT_TO_POINTER(PB_LLk), test_result_length);
//XXX g_test_add_data_func("/core/parser/llk/token_position", GINT_TO_POINTER(PB_LLk), test_token_position); //g_test_add_data_func("/core/parser/llk/token_position", GINT_TO_POINTER(PB_LLk), test_token_position);
g_test_add_data_func("/core/parser/llk/iterative", GINT_TO_POINTER(PB_LLk), test_iterative); g_test_add_data_func("/core/parser/llk/iterative", GINT_TO_POINTER(PB_LLk), test_iterative);
g_test_add_data_func("/core/parser/llk/iterative/lookahead", GINT_TO_POINTER(PB_LLk), test_iterative_lookahead); g_test_add_data_func("/core/parser/llk/iterative/lookahead", GINT_TO_POINTER(PB_LLk), test_iterative_lookahead);
g_test_add_data_func("/core/parser/llk/iterative/result_length", GINT_TO_POINTER(PB_LLk), test_iterative_result_length); g_test_add_data_func("/core/parser/llk/iterative/result_length", GINT_TO_POINTER(PB_LLk), test_iterative_result_length);