From 578ac0515426ca56708c3e253323644aa9f44139 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 3 Oct 2015 14:32:54 +0200 Subject: [PATCH 01/35] add platform.h to distribution headers --- src/SConscript | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SConscript b/src/SConscript index 05ffa98..0351e1b 100644 --- a/src/SConscript +++ b/src/SConscript @@ -7,7 +7,8 @@ dist_headers = [ "allocator.h", "compiler_specifics.h", "glue.h", - "internal.h" + "internal.h", + "platform.h" ] parsers_headers = [ From 969d7682053bb73cc78503556d63dd10039eb59f Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 3 Oct 2015 17:54:11 +0200 Subject: [PATCH 02/35] Don't cache dummy results for primitive parsers. --- src/backends/packrat.c | 30 +++++++++++++++++------------- src/internal.h | 7 ++++--- src/parsers/action.c | 1 + src/parsers/and.c | 1 + src/parsers/attr_bool.c | 1 + src/parsers/bind.c | 1 + src/parsers/bits.c | 1 + src/parsers/butnot.c | 1 + src/parsers/ch.c | 1 + src/parsers/charset.c | 1 + src/parsers/choice.c | 1 + src/parsers/difference.c | 1 + src/parsers/end.c | 1 + src/parsers/endianness.c | 1 + src/parsers/epsilon.c | 1 + src/parsers/ignore.c | 1 + src/parsers/ignoreseq.c | 1 + src/parsers/indirect.c | 1 + src/parsers/int_range.c | 1 + src/parsers/many.c | 1 + src/parsers/not.c | 1 + src/parsers/nothing.c | 1 + src/parsers/optional.c | 1 + src/parsers/permutation.c | 1 + src/parsers/sequence.c | 1 + src/parsers/token.c | 1 + src/parsers/unimplemented.c | 1 + src/parsers/value.c | 2 ++ src/parsers/whitespace.c | 1 + src/parsers/xor.c | 1 + 30 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index e6f86f2..9977cc3 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -3,7 +3,7 @@ #include "../internal.h" #include "../parsers/parser_internal.h" -// short-hand for creating cache values (regular case) +// short-hand for creating lowlevel parse cache values (parse result case) static HParserCacheValue * cached_result(HParseState *state, HParseResult *result) { HParserCacheValue *ret = a_new(HParserCacheValue, 1); @@ -13,7 +13,7 @@ HParserCacheValue * cached_result(HParseState *state, HParseResult *result) { return ret; } -// short-hand for caching parse results (left recursion case) +// short-hand for creating lowlevel parse cache values (left recursion case) static HParserCacheValue *cached_lr(HParseState *state, HLeftRec *lr) { HParserCacheValue *ret = a_new(HParserCacheValue, 1); @@ -186,18 +186,22 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) { if (!m) { // It doesn't exist, so create a dummy result to cache HLeftRec *base = a_new(HLeftRec, 1); - base->seed = NULL; base->rule = parser; base->head = NULL; - h_slist_push(state->lr_stack, base); - // cache it - h_hashtable_put(state->cache, key, cached_lr(state, base)); - // parse the input + if (parser->vtable->higher) { + base->seed = NULL; base->rule = parser; base->head = NULL; + h_slist_push(state->lr_stack, base); + // cache it + h_hashtable_put(state->cache, key, cached_lr(state, base)); + // parse the input + } HParseResult *tmp_res = perform_lowlevel_parse(state, parser); - // the base variable has passed equality tests with the cache - h_slist_pop(state->lr_stack); - // update the cached value to our new position - HParserCacheValue *cached = h_hashtable_get(state->cache, key); - assert(cached != NULL); - cached->input_stream = state->input_stream; + if (parser->vtable->higher) { + // the base variable has passed equality tests with the cache + h_slist_pop(state->lr_stack); + // update the cached value to our new position + HParserCacheValue *cached = h_hashtable_get(state->cache, key); + assert(cached != NULL); + cached->input_stream = state->input_stream; + } // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one if (NULL == base->head) { h_hashtable_put(state->cache, key, cached_result(state, tmp_res)); diff --git a/src/internal.h b/src/internal.h index b35b7d5..776f636 100644 --- a/src/internal.h +++ b/src/internal.h @@ -278,9 +278,9 @@ typedef struct HRecursionHead_ { /* A left recursion. * * Members: - * seed - - * rule - - * head - + * seed - the HResult yielded by rule + * rule - the HParser that produces seed + * head - the */ typedef struct HLeftRec_ { HParseResult *seed; @@ -419,6 +419,7 @@ struct HParserVtable_ { bool (*isValidCF)(void *env); bool (*compile_to_rvm)(HRVMProg *prog, void* env); // FIXME: forgot what the bool return value was supposed to mean. void (*desugar)(HAllocator *mm__, HCFStack *stk__, void *env); + bool higher; // false if primitive }; bool h_false(void*); diff --git a/src/parsers/action.c b/src/parsers/action.c index 04eb7a4..a324333 100644 --- a/src/parsers/action.c +++ b/src/parsers/action.c @@ -81,6 +81,7 @@ static const HParserVtable action_vt = { .isValidCF = action_isValidCF, .desugar = desugar_action, .compile_to_rvm = action_ctrvm, + .higher = true, }; HParser* h_action(const HParser* p, const HAction a, void* user_data) { diff --git a/src/parsers/and.c b/src/parsers/and.c index c5c9836..e07bc9f 100644 --- a/src/parsers/and.c +++ b/src/parsers/and.c @@ -17,6 +17,7 @@ static const HParserVtable and_vt = { revision. --mlp, 18/12/12 */ .isValidCF = h_false, /* despite TODO above, this remains false. */ .compile_to_rvm = h_not_regular, + .higher = true, }; diff --git a/src/parsers/attr_bool.c b/src/parsers/attr_bool.c index e8359ab..f766774 100644 --- a/src/parsers/attr_bool.c +++ b/src/parsers/attr_bool.c @@ -79,6 +79,7 @@ static const HParserVtable attr_bool_vt = { .isValidCF = ab_isValidCF, .desugar = desugar_ab, .compile_to_rvm = ab_ctrvm, + .higher = true, }; diff --git a/src/parsers/bind.c b/src/parsers/bind.c index f024a82..7fa821d 100644 --- a/src/parsers/bind.c +++ b/src/parsers/bind.c @@ -60,6 +60,7 @@ static const HParserVtable bind_vt = { .isValidRegular = h_false, .isValidCF = h_false, .compile_to_rvm = h_not_regular, + .higher = true, }; HParser *h_bind(const HParser *p, HContinuation k, void *env) diff --git a/src/parsers/bits.c b/src/parsers/bits.c index 716524c..be8f13f 100644 --- a/src/parsers/bits.c +++ b/src/parsers/bits.c @@ -102,6 +102,7 @@ static const HParserVtable bits_vt = { .isValidCF = h_true, .desugar = desugar_bits, .compile_to_rvm = bits_ctrvm, + .higher = false, }; HParser* h_bits(size_t len, bool sign) { diff --git a/src/parsers/butnot.c b/src/parsers/butnot.c index f114a1f..24ece4b 100644 --- a/src/parsers/butnot.c +++ b/src/parsers/butnot.c @@ -40,6 +40,7 @@ static const HParserVtable butnot_vt = { .isValidRegular = h_false, .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF? .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_butnot(const HParser* p1, const HParser* p2) { diff --git a/src/parsers/ch.c b/src/parsers/ch.c index b4386cf..3da1091 100644 --- a/src/parsers/ch.c +++ b/src/parsers/ch.c @@ -47,6 +47,7 @@ static const HParserVtable ch_vt = { .isValidCF = h_true, .desugar = desugar_ch, .compile_to_rvm = ch_ctrvm, + .higher = false, }; HParser* h_ch(const uint8_t c) { diff --git a/src/parsers/charset.c b/src/parsers/charset.c index e1a910f..a4b8c89 100644 --- a/src/parsers/charset.c +++ b/src/parsers/charset.c @@ -76,6 +76,7 @@ static const HParserVtable charset_vt = { .isValidCF = h_true, .desugar = desugar_charset, .compile_to_rvm = cs_ctrvm, + .higher = false, }; HParser* h_ch_range(const uint8_t lower, const uint8_t upper) { diff --git a/src/parsers/choice.c b/src/parsers/choice.c index bfc3f90..dd3908c 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -75,6 +75,7 @@ static const HParserVtable choice_vt = { .isValidCF = choice_isValidCF, .desugar = desugar_choice, .compile_to_rvm = choice_ctrvm, + .higher = true, }; HParser* h_choice(HParser* p, ...) { diff --git a/src/parsers/difference.c b/src/parsers/difference.c index 76a2cc4..a24f5ac 100644 --- a/src/parsers/difference.c +++ b/src/parsers/difference.c @@ -39,6 +39,7 @@ static HParserVtable difference_vt = { .isValidRegular = h_false, .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF? .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_difference(const HParser* p1, const HParser* p2) { diff --git a/src/parsers/end.c b/src/parsers/end.c index 30b3ba1..85499d9 100644 --- a/src/parsers/end.c +++ b/src/parsers/end.c @@ -25,6 +25,7 @@ static const HParserVtable end_vt = { .isValidCF = h_true, .desugar = desugar_end, .compile_to_rvm = end_ctrvm, + .higher = false, }; HParser* h_end_p() { diff --git a/src/parsers/endianness.c b/src/parsers/endianness.c index e3f53ab..cb3abc3 100644 --- a/src/parsers/endianness.c +++ b/src/parsers/endianness.c @@ -46,6 +46,7 @@ static const HParserVtable endianness_vt = { .isValidCF = h_false, .desugar = NULL, .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_with_endianness(char endianness, const HParser *p) diff --git a/src/parsers/epsilon.c b/src/parsers/epsilon.c index e8ef525..bb6e8be 100644 --- a/src/parsers/epsilon.c +++ b/src/parsers/epsilon.c @@ -18,6 +18,7 @@ static const HParserVtable epsilon_vt = { .isValidCF = h_true, .desugar = desugar_epsilon, .compile_to_rvm = epsilon_ctrvm, + .higher = false, }; HParser* h_epsilon_p() { diff --git a/src/parsers/ignore.c b/src/parsers/ignore.c index af606b0..c56802a 100644 --- a/src/parsers/ignore.c +++ b/src/parsers/ignore.c @@ -49,6 +49,7 @@ static const HParserVtable ignore_vt = { .isValidCF = ignore_isValidCF, .desugar = desugar_ignore, .compile_to_rvm = ignore_ctrvm, + .higher = true, }; HParser* h_ignore(const HParser* p) { diff --git a/src/parsers/ignoreseq.c b/src/parsers/ignoreseq.c index e562136..07bdc65 100644 --- a/src/parsers/ignoreseq.c +++ b/src/parsers/ignoreseq.c @@ -103,6 +103,7 @@ static const HParserVtable ignoreseq_vt = { .isValidCF = is_isValidCF, .desugar = desugar_ignoreseq, .compile_to_rvm = is_ctrvm, + .higher = true, }; diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 026286d..b36cb94 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -31,6 +31,7 @@ static const HParserVtable indirect_vt = { .isValidCF = indirect_isValidCF, .desugar = desugar_indirect, .compile_to_rvm = h_not_regular, + .higher = true, }; void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { diff --git a/src/parsers/int_range.c b/src/parsers/int_range.c index 2937993..49b0642 100644 --- a/src/parsers/int_range.c +++ b/src/parsers/int_range.c @@ -117,6 +117,7 @@ static const HParserVtable int_range_vt = { .isValidCF = h_true, .desugar = desugar_int_range, .compile_to_rvm = ir_ctrvm, + .higher = false, }; HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper) { diff --git a/src/parsers/many.c b/src/parsers/many.c index cae2b0e..6496bbe 100644 --- a/src/parsers/many.c +++ b/src/parsers/many.c @@ -199,6 +199,7 @@ static const HParserVtable many_vt = { .isValidCF = many_isValidCF, .desugar = desugar_many, .compile_to_rvm = many_ctrvm, + .higher = true, }; HParser* h_many(const HParser* p) { diff --git a/src/parsers/not.c b/src/parsers/not.c index 6c34bad..8c2003d 100644 --- a/src/parsers/not.c +++ b/src/parsers/not.c @@ -15,6 +15,7 @@ static const HParserVtable not_vt = { .isValidRegular = h_false, /* see and.c for why */ .isValidCF = h_false, .compile_to_rvm = h_not_regular, // Is actually regular, but the generation step is currently unable to handle it. TODO: fix this. + .higher = true, }; HParser* h_not(const HParser* p) { diff --git a/src/parsers/nothing.c b/src/parsers/nothing.c index 120c1e0..0a60108 100644 --- a/src/parsers/nothing.c +++ b/src/parsers/nothing.c @@ -22,6 +22,7 @@ static const HParserVtable nothing_vt = { .isValidCF = h_true, .desugar = desugar_nothing, .compile_to_rvm = nothing_ctrvm, + .higher = false, }; HParser* h_nothing_p() { diff --git a/src/parsers/optional.c b/src/parsers/optional.c index ccee53f..7266066 100644 --- a/src/parsers/optional.c +++ b/src/parsers/optional.c @@ -84,6 +84,7 @@ static const HParserVtable optional_vt = { .isValidCF = opt_isValidCF, .desugar = desugar_optional, .compile_to_rvm = opt_ctrvm, + .higher = true, }; HParser* h_optional(const HParser* p) { diff --git a/src/parsers/permutation.c b/src/parsers/permutation.c index 564565a..b167584 100644 --- a/src/parsers/permutation.c +++ b/src/parsers/permutation.c @@ -104,6 +104,7 @@ static const HParserVtable permutation_vt = { .isValidCF = h_false, .desugar = NULL, .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_permutation(HParser* p, ...) { diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index 93c0cfb..30de34a 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -93,6 +93,7 @@ static const HParserVtable sequence_vt = { .isValidCF = sequence_isValidCF, .desugar = desugar_sequence, .compile_to_rvm = sequence_ctrvm, + .higher = true, }; HParser* h_sequence(HParser* p, ...) { diff --git a/src/parsers/token.c b/src/parsers/token.c index d36ec54..1902972 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -73,6 +73,7 @@ const HParserVtable token_vt = { .isValidCF = h_true, .desugar = desugar_token, .compile_to_rvm = token_ctrvm, + .higher = false, }; HParser* h_token(const uint8_t *str, const size_t len) { diff --git a/src/parsers/unimplemented.c b/src/parsers/unimplemented.c index e3f3039..e085858 100644 --- a/src/parsers/unimplemented.c +++ b/src/parsers/unimplemented.c @@ -18,6 +18,7 @@ static const HParserVtable unimplemented_vt = { .isValidCF = h_false, .desugar = NULL, .compile_to_rvm = h_not_regular, + .higher = true, }; static HParser unimplemented = { diff --git a/src/parsers/value.c b/src/parsers/value.c index 531db7c..7fa863a 100644 --- a/src/parsers/value.c +++ b/src/parsers/value.c @@ -26,6 +26,7 @@ static const HParserVtable put_vt = { .isValidRegular = h_false, .isValidCF = h_false, .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_put_value(const HParser* p, const char* name) { @@ -55,6 +56,7 @@ static const HParserVtable get_vt = { .isValidRegular = h_false, .isValidCF = h_false, .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_get_value(const char* name) { diff --git a/src/parsers/whitespace.c b/src/parsers/whitespace.c index 04284e8..970a32c 100644 --- a/src/parsers/whitespace.c +++ b/src/parsers/whitespace.c @@ -75,6 +75,7 @@ static const HParserVtable whitespace_vt = { .isValidCF = ws_isValidCF, .desugar = desugar_whitespace, .compile_to_rvm = ws_ctrvm, + .higher = false, }; HParser* h_whitespace(const HParser* p) { diff --git a/src/parsers/xor.c b/src/parsers/xor.c index e031d5d..3a3f21d 100644 --- a/src/parsers/xor.c +++ b/src/parsers/xor.c @@ -36,6 +36,7 @@ static const HParserVtable xor_vt = { .isValidRegular = h_false, .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF? .compile_to_rvm = h_not_regular, + .higher = true, }; HParser* h_xor(const HParser* p1, const HParser* p2) { From 755b771ee29771e90c58636da94edb128c8b0410 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 3 Oct 2015 22:26:40 +0200 Subject: [PATCH 03/35] more thorough comments --- src/backends/packrat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index 9977cc3..b771973 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -186,6 +186,7 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) { if (!m) { // It doesn't exist, so create a dummy result to cache HLeftRec *base = a_new(HLeftRec, 1); + // But only cache it now if there's some chance it could grow; primitive parsers can't if (parser->vtable->higher) { base->seed = NULL; base->rule = parser; base->head = NULL; h_slist_push(state->lr_stack, base); From 9dbd2debda252dd9c69f55877e0b5ea017583b8e Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 00:03:24 +0200 Subject: [PATCH 04/35] fix --coverage build option; building with it now creates .gcda files that gcov can use --- SConstruct | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index a8f7ce8..b7549cd 100644 --- a/SConstruct +++ b/SConstruct @@ -91,9 +91,9 @@ else: env = opt if GetOption("coverage"): - env.Append(CFLAGS=["-fprofile-arcs", "-ftest-coverage"], - CXXFLAGS=["-fprofile-arcs", "-ftest-coverage"], - LDFLAGS=["-fprofile-arcs", "-ftest-coverage"], + env.Append(CFLAGS=["--coverage"], + CXXFLAGS=["--coverage"], + LDFLAGS=["--coverage"], LIBS=['gcov']) env["CC"] = os.getenv("CC") or env["CC"] From 10997afcbb0fffc6a1ddfee767b9d6095d8e7176 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 00:42:28 +0200 Subject: [PATCH 05/35] adding code coverage to automation --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8973c57..eb7a752 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,6 +92,8 @@ matrix: env: BINDINGS=cpp CC=clang before_install: - sudo apt-get update -qq + - sudo apt-get install lcov + - gem install coveralls-lcov - if [ "$BINDINGS" != "none" ]; then sudo apt-get install -qq swig; fi - if [ "$BINDINGS" == "perl" ]; then sudo add-apt-repository ppa:dns/irc -y; sudo apt-get update -qq; sudo apt-get install -qq swig=2.0.8-1irc1~12.04; fi - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq python-dev; fi @@ -100,11 +102,13 @@ install: true before_script: - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi script: - - scons bindings=$BINDINGS test + - scons bindings=$BINDINGS test --coverage +after_success: + - lcov --capture --directory build/opt/src --output-file coverage.info + - coveralls-lcov coverage.info notifications: irc: channels: - "irc.upstandinghackers.com#hammer" use_notice: true skip_join: true - From ff65571474ea6bbff8791b873df2321d60e73412 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 01:43:44 +0200 Subject: [PATCH 06/35] this is a dirty hack, but it solves the Program builder not honoring LDFLAGS --- .travis.yml | 26 +++++++++++++------------- SConstruct | 3 +-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb7a752..fe473d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-1.9.3-p484 - env: BINDINGS=ruby CC=clang + env: BINDINGS=ruby CC="clang --coverage" - compiler: gcc language: ruby rvm: ruby-2.0.0-p353 @@ -21,7 +21,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-2.0.0-p353 - env: BINDINGS=ruby CC=clang + env: BINDINGS=ruby CC="clang --coverage" - compiler: gcc language: ruby rvm: ruby-2.1.0 @@ -29,7 +29,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-2.1.0 - env: BINDINGS=ruby CC=clang + env: BINDINGS=ruby CC="clang --coverage" - compiler: gcc language: python python: "2.7" @@ -37,7 +37,7 @@ matrix: - compiler: clang language: python python: "2.7" - env: BINDINGS=python CC=clang + env: BINDINGS=python CC="clang --coverage" - compiler: gcc language: perl perl: "5.18" @@ -45,7 +45,7 @@ matrix: - compiler: clang language: perl perl: "5.18" - env: BINDINGS=perl CC=clang + env: BINDINGS=perl CC="clang --coverage" - compiler: gcc language: perl perl: "5.14" @@ -53,7 +53,7 @@ matrix: - compiler: clang language: perl perl: "5.14" - env: BINDINGS=perl CC=clang + env: BINDINGS=perl CC="clang --coverage" - compiler: gcc language: perl perl: "5.10" @@ -61,7 +61,7 @@ matrix: - compiler: clang language: perl perl: "5.10" - env: BINDINGS=perl CC=clang + env: BINDINGS=perl CC="clang --coverage" - compiler: gcc language: php php: "5.5" @@ -69,7 +69,7 @@ matrix: - compiler: clang language: php php: "5.5" - env: BINDINGS=php CC=clang + env: BINDINGS=php CC="clang --coverage" - compiler: gcc language: php php: "5.4" @@ -77,19 +77,19 @@ matrix: - compiler: clang language: php php: "5.4" - env: BINDINGS=php CC=clang + env: BINDINGS=php CC="clang --coverage" - compiler: gcc language: dotnet env: BINDINGS=dotnet - compiler: clang language: dotnet - env: BINDINGS=dotnet CC=clang + env: BINDINGS=dotnet CC="clang --coverage" - compiler: gcc language: cpp env: BINDINGS=cpp - compiler: gcc language: cpp - env: BINDINGS=cpp CC=clang + env: BINDINGS=cpp CC="clang --coverage" before_install: - sudo apt-get update -qq - sudo apt-get install lcov @@ -102,9 +102,9 @@ install: true before_script: - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi script: - - scons bindings=$BINDINGS test --coverage + - scons bindings=$BINDINGS test --variant=debug --coverage after_success: - - lcov --capture --directory build/opt/src --output-file coverage.info + - if [ "$CC" == "clang --coverage" ]; then llvm-cov gcov -o coverage.info build/debug/src/test_suite.gcda; else lcov --capture --directory build/debug/src --output-file coverage.info; fi - coveralls-lcov coverage.info notifications: irc: diff --git a/SConstruct b/SConstruct index b7549cd..05596f3 100644 --- a/SConstruct +++ b/SConstruct @@ -93,8 +93,7 @@ else: if GetOption("coverage"): env.Append(CFLAGS=["--coverage"], CXXFLAGS=["--coverage"], - LDFLAGS=["--coverage"], - LIBS=['gcov']) + LDFLAGS=["--coverage"]) env["CC"] = os.getenv("CC") or env["CC"] env["CXX"] = os.getenv("CXX") or env["CXX"] From 2b637946fac73db06e830497413d2fd3bb161497 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 01:49:39 +0200 Subject: [PATCH 07/35] nope, guess we need -lgcov after all --- SConstruct | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 05596f3..b7549cd 100644 --- a/SConstruct +++ b/SConstruct @@ -93,7 +93,8 @@ else: if GetOption("coverage"): env.Append(CFLAGS=["--coverage"], CXXFLAGS=["--coverage"], - LDFLAGS=["--coverage"]) + LDFLAGS=["--coverage"], + LIBS=['gcov']) env["CC"] = os.getenv("CC") or env["CC"] env["CXX"] = os.getenv("CXX") or env["CXX"] From 2915a8f174e6fd71b748266d239d01116c14d090 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 01:52:26 +0200 Subject: [PATCH 08/35] dirty hack continues --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fe473d6..ff04ef5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ matrix: language: cpp env: BINDINGS=cpp CC="clang --coverage" before_install: + - if [ "$CC" == "clang" ]; then export CC="clang --coverage" - sudo apt-get update -qq - sudo apt-get install lcov - gem install coveralls-lcov From d8d106d03552c15ac60e3d96e934ea353bac1661 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 01:55:11 +0200 Subject: [PATCH 09/35] typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff04ef5..c0990f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ matrix: language: cpp env: BINDINGS=cpp CC="clang --coverage" before_install: - - if [ "$CC" == "clang" ]; then export CC="clang --coverage" + - if [ "$CC" == "clang" ]; then export CC="clang --coverage"; fi - sudo apt-get update -qq - sudo apt-get install lcov - gem install coveralls-lcov From d809bbd1a905206207334d70cbb82013988b0669 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 01:59:20 +0200 Subject: [PATCH 10/35] slightly less ugly --- .travis.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0990f7..d74cf61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-1.9.3-p484 - env: BINDINGS=ruby CC="clang --coverage" + env: BINDINGS=ruby CC=clang - compiler: gcc language: ruby rvm: ruby-2.0.0-p353 @@ -21,7 +21,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-2.0.0-p353 - env: BINDINGS=ruby CC="clang --coverage" + env: BINDINGS=ruby CC=clang - compiler: gcc language: ruby rvm: ruby-2.1.0 @@ -29,7 +29,7 @@ matrix: - compiler: clang language: ruby rvm: ruby-2.1.0 - env: BINDINGS=ruby CC="clang --coverage" + env: BINDINGS=ruby CC=clang - compiler: gcc language: python python: "2.7" @@ -37,7 +37,7 @@ matrix: - compiler: clang language: python python: "2.7" - env: BINDINGS=python CC="clang --coverage" + env: BINDINGS=python CC=clang - compiler: gcc language: perl perl: "5.18" @@ -45,7 +45,7 @@ matrix: - compiler: clang language: perl perl: "5.18" - env: BINDINGS=perl CC="clang --coverage" + env: BINDINGS=perl CC=clang - compiler: gcc language: perl perl: "5.14" @@ -53,7 +53,7 @@ matrix: - compiler: clang language: perl perl: "5.14" - env: BINDINGS=perl CC="clang --coverage" + env: BINDINGS=perl CC=clang - compiler: gcc language: perl perl: "5.10" @@ -61,7 +61,7 @@ matrix: - compiler: clang language: perl perl: "5.10" - env: BINDINGS=perl CC="clang --coverage" + env: BINDINGS=perl CC=clang - compiler: gcc language: php php: "5.5" @@ -69,7 +69,7 @@ matrix: - compiler: clang language: php php: "5.5" - env: BINDINGS=php CC="clang --coverage" + env: BINDINGS=php CC=clang - compiler: gcc language: php php: "5.4" @@ -77,19 +77,19 @@ matrix: - compiler: clang language: php php: "5.4" - env: BINDINGS=php CC="clang --coverage" + env: BINDINGS=php CC=clang - compiler: gcc language: dotnet env: BINDINGS=dotnet - compiler: clang language: dotnet - env: BINDINGS=dotnet CC="clang --coverage" + env: BINDINGS=dotnet CC=clang - compiler: gcc language: cpp env: BINDINGS=cpp - compiler: gcc language: cpp - env: BINDINGS=cpp CC="clang --coverage" + env: BINDINGS=cpp CC=clang before_install: - if [ "$CC" == "clang" ]; then export CC="clang --coverage"; fi - sudo apt-get update -qq From 4fed9327a4b87c8871677ca5b5892263022a914b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 02:53:54 +0200 Subject: [PATCH 11/35] now both clang and gcc have coverage without any dumb hacks --- .travis.yml | 1 - SConstruct | 13 ++++++++----- src/SConscript | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index d74cf61..05206db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,6 @@ matrix: language: cpp env: BINDINGS=cpp CC=clang before_install: - - if [ "$CC" == "clang" ]; then export CC="clang --coverage"; fi - sudo apt-get update -qq - sudo apt-get install lcov - gem install coveralls-lcov diff --git a/SConstruct b/SConstruct index b7549cd..bb2bb85 100644 --- a/SConstruct +++ b/SConstruct @@ -90,14 +90,17 @@ if GetOption("variant") == 'debug': else: env = opt +env["CC"] = os.getenv("CC") or env["CC"] +env["CXX"] = os.getenv("CXX") or env["CXX"] + if GetOption("coverage"): env.Append(CFLAGS=["--coverage"], CXXFLAGS=["--coverage"], - LDFLAGS=["--coverage"], - LIBS=['gcov']) - -env["CC"] = os.getenv("CC") or env["CC"] -env["CXX"] = os.getenv("CXX") or env["CXX"] + LDFLAGS=["--coverage"]) + if env["CC"] == "gcc": + env.Append(LIBS=['gcov']) + else: + env.ParseConfig('llvm-config --ldflags') if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", diff --git a/src/SConscript b/src/SConscript index 0351e1b..dd6b616 100644 --- a/src/SConscript +++ b/src/SConscript @@ -89,7 +89,7 @@ env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') testenv.Append(LIBS=['hammer'], LIBPATH=['.']) -ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c']) +ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS="--coverage" if testenv.GetOption("coverage") else None) ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path])) AlwaysBuild(ctest) testruns.append(ctest) From 9235bf793c4fbffd7423e4d1a8005e2a47dc7970 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 03:36:02 +0200 Subject: [PATCH 12/35] Only do coverage with the C builds; otherwise the bindings tests don't work --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05206db..b533da3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -101,10 +101,10 @@ before_install: install: true before_script: - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi -script: - - scons bindings=$BINDINGS test --variant=debug --coverage +script: + - if [ "$BINDINGS" == "none" ]; then scons test --variant=debug --coverage; else scons bindings=$BINDINGS test; fi after_success: - - if [ "$CC" == "clang --coverage" ]; then llvm-cov gcov -o coverage.info build/debug/src/test_suite.gcda; else lcov --capture --directory build/debug/src --output-file coverage.info; fi + - if [ "$BINDINGS" == "none" ]; then if [ "$CC" == "clang" ]; then llvm-cov gcov -o coverage.info build/debug/src/test_suite.gcda; else lcov --capture --directory build/debug/src --output-file coverage.info; fi; fi - coveralls-lcov coverage.info notifications: irc: From 99ca500d2b59dc286b405b6a278ca244a4c17fdb Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 4 Oct 2015 15:32:32 +0200 Subject: [PATCH 13/35] Don't try to recall() primitive results --- src/backends/packrat.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index b771973..abb198b 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -181,7 +181,10 @@ HParseResult* lr_answer(HParserCacheKey *k, HParseState *state, HLeftRec *growab HParseResult* h_do_parse(const HParser* parser, HParseState *state) { HParserCacheKey *key = a_new(HParserCacheKey, 1); key->input_pos = state->input_stream; key->parser = parser; - HParserCacheValue *m = recall(key, state); + HParserCacheValue *m = NULL; + if (parser->vtable->higher) { + m = recall(key, state); + } // check to see if there is already a result for this object... if (!m) { // It doesn't exist, so create a dummy result to cache From d2ade1f5b4f8c438353d79085c55b1c06436b372 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 29 Oct 2015 13:11:32 +0100 Subject: [PATCH 14/35] call h_arena_free when resizing in h_carray_append --- src/datastructures.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datastructures.c b/src/datastructures.c index 0feeb21..af8477b 100644 --- a/src/datastructures.c +++ b/src/datastructures.c @@ -25,11 +25,12 @@ HCountedArray *h_carray_new(HArena * arena) { void h_carray_append(HCountedArray *array, void* item) { if (array->used >= array->capacity) { - HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(HCountedArray*)); + HParsedToken **elements = h_arena_malloc(array->arena, (array->capacity *= 2) * sizeof(void*)); for (size_t i = 0; i < array->used; i++) elements[i] = array->elements[i]; for (size_t i = array->used; i < array->capacity; i++) elements[i] = 0; + h_arena_free(array->arena, array->elements); array->elements = elements; } array->elements[array->used++] = item; From 3765fd64e1b2d0f15ac689f5ee3af9939b26ada5 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 29 Oct 2015 13:12:16 +0100 Subject: [PATCH 15/35] don't pre-allocate all space when parsing an h_length_value --- src/parsers/many.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/parsers/many.c b/src/parsers/many.c index 6496bbe..071e3fc 100644 --- a/src/parsers/many.c +++ b/src/parsers/many.c @@ -10,7 +10,10 @@ typedef struct { static HParseResult *parse_many(void* env, HParseState *state) { HRepeat *env_ = (HRepeat*) env; - HCountedArray *seq = h_carray_new_sized(state->arena, (env_->count > 0 ? env_->count : 4)); + size_t size = env_->count; + if(size <= 0) size = 4; + if(size > 1024) size = 1024; // let's try parsing some elements first... + HCountedArray *seq = h_carray_new_sized(state->arena, size); size_t count = 0; HInputStream bak; while (env_->min_p || env_->count > count) { From d5e79aa4cb0f932a1f4b4b913d0b8f55678316f3 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Thu, 29 Oct 2015 22:03:05 +0100 Subject: [PATCH 16/35] fail an assert on h_sequence(NULL) instead of segfaulting later --- src/parsers/sequence.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index 30de34a..6377589 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -120,6 +120,8 @@ HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) { va_list ap; size_t len = 0; const HParser *arg; + + assert(p != NULL); va_copy(ap, ap_); do { len++; From 9ef70f2f2d40b973c81cce4ca9e65bafa6c1ba4a Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 30 Oct 2015 21:23:45 +0100 Subject: [PATCH 17/35] never return if h_arena_malloc fails, call errx() or longjmp() --- src/allocator.c | 34 ++++++++++++++++++++++++---------- src/allocator.h | 2 ++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/allocator.c b/src/allocator.c index 258edfa..4646dd8 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "hammer.h" #include "internal.h" @@ -42,6 +43,8 @@ struct HArena_ { size_t block_size; size_t used; size_t wasted; + + jmp_buf *except; }; HArena *h_new_arena(HAllocator* mm__, size_t block_size) { @@ -62,9 +65,26 @@ HArena *h_new_arena(HAllocator* mm__, size_t block_size) { ret->used = 0; ret->mm__ = mm__; ret->wasted = sizeof(struct arena_link) + sizeof(struct HArena_) + block_size; + ret->except = NULL; return ret; } +void h_arena_set_except(HArena *arena, jmp_buf *except) +{ + arena->except = except; +} + +static void *alloc_block(HArena *arena, size_t size) +{ + void *block = arena->mm__->alloc(arena->mm__, size); + if (!block) { + if (arena->except) + longjmp(*arena->except, 1); + h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size); + } + return block; +} + void* h_arena_malloc(HArena *arena, size_t size) { if (size <= arena->head->free) { // fast path.. @@ -79,22 +99,16 @@ void* h_arena_malloc(HArena *arena, size_t size) { // This involves some annoying casting... arena->used += size; arena->wasted += sizeof(struct arena_link*); - void* link = arena->mm__->alloc(arena->mm__, size + sizeof(struct arena_link*)); - if (!link) { - // TODO: error-reporting -- let user know that arena link couldn't be allocated - return NULL; - } + void* link = alloc_block(arena, size + sizeof(struct arena_link*)); + assert(link != NULL); memset(link, 0, size + sizeof(struct arena_link*)); *(struct arena_link**)link = arena->head->next; arena->head->next = (struct arena_link*)link; return (void*)(((uint8_t*)link) + sizeof(struct arena_link*)); } else { // we just need to allocate an ordinary new block. - struct arena_link *link = (struct arena_link*)arena->mm__->alloc(arena->mm__, sizeof(struct arena_link) + arena->block_size); - if (!link) { - // TODO: error-reporting -- let user know that arena link couldn't be allocated - return NULL; - } + struct arena_link *link = alloc_block(arena, sizeof(struct arena_link) + arena->block_size); + assert(link != NULL); memset(link, 0, sizeof(struct arena_link) + arena->block_size); link->free = arena->block_size - size; link->used = size; diff --git a/src/allocator.h b/src/allocator.h index 4a48693..d80209d 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -18,6 +18,7 @@ #ifndef HAMMER_ALLOCATOR__H__ #define HAMMER_ALLOCATOR__H__ #include +#include #ifdef __cplusplus extern "C" { @@ -51,6 +52,7 @@ HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for def void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2); void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); +void h_arena_set_except(HArena *arena, jmp_buf *except); typedef struct { size_t used; From e8b1962005e32156a722c9189e2431c7d3a50a2b Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 30 Oct 2015 21:26:08 +0100 Subject: [PATCH 18/35] out of memory handling and leak fixes in regex backend --- src/backends/regex.c | 45 ++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/backends/regex.c b/src/backends/regex.c index f6494fa..3c2e803 100644 --- a/src/backends/regex.c +++ b/src/backends/regex.c @@ -56,7 +56,16 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_ *heads_p = h_sarray_new(mm__, prog->length); HRVMTrace *ret_trace = NULL; + HParseResult *ret = NULL; + // out of memory handling + if(!arena || !heads_n || !heads_p) + goto end; + jmp_buf except; + h_arena_set_except(arena, &except); + if(setjmp(except)) + goto end; + uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued HRVMThread *ip_queue = a_new(HRVMThread, prog->length); size_t ipq_top; @@ -164,18 +173,19 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_ } // No accept was reached. match_fail: - if (ret_trace == NULL) { - // No match found; definite failure. - h_delete_arena(arena); - return NULL; + + h_arena_set_except(arena, NULL); // there should be no more allocs from this + if (ret_trace) { + // Invert the direction of the trace linked list. + ret_trace = invert_trace(ret_trace); + ret = run_trace(mm__, prog, ret_trace, input, len); + // NB: ret is in its own arena } - // Invert the direction of the trace linked list. - - ret_trace = invert_trace(ret_trace); - HParseResult *ret = run_trace(mm__, prog, ret_trace, input, len); - // ret is in its own arena - h_delete_arena(arena); + end: + if (arena) h_delete_arena(arena); + if (heads_n) h_sarray_free(heads_n); + if (heads_p) h_sarray_free(heads_p); return ret; } #undef PUSH_SVM @@ -203,6 +213,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace, ctx.stack_capacity = 16; ctx.stack = h_new(HParsedToken*, ctx.stack_capacity); + // out of memory handling + if(!arena || !ctx.stack) + goto fail; + jmp_buf except; + h_arena_set_except(arena, &except); + if(setjmp(except)) + goto fail; + HParsedToken *tmp_res; HRVMTrace *cur; for (cur = trace; cur; cur = cur->next) { @@ -242,7 +260,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace, break; case SVM_ACCEPT: assert(ctx.stack_count <= 1); - HParseResult *res = a_new(HParseResult, 1); + HParseResult *res = a_new(HParseResult, 1); if (ctx.stack_count == 1) { res->ast = ctx.stack[0]; } else { @@ -250,11 +268,14 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace, } res->bit_length = cur->input_pos * 8; res->arena = arena; + h_arena_set_except(arena, NULL); + h_free(ctx.stack); return res; } } fail: - h_delete_arena(arena); + if (arena) h_delete_arena(arena); + if (ctx.stack) h_free(ctx.stack); return NULL; } From 25acd90a481357942cd3a7f3add6b8b996345305 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sun, 1 Nov 2015 17:49:53 +0100 Subject: [PATCH 19/35] allow h_sequence(NULL) as parser for the empty sequence --- src/parsers/sequence.c | 43 +++++++++++++++++++++++------------------- src/t_parser.c | 2 ++ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index 6377589..55c0c88 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -117,28 +117,33 @@ HParser* h_sequence__v(HParser* p, va_list ap) { } HParser* h_sequence__mv(HAllocator* mm__, HParser *p, va_list ap_) { - va_list ap; - size_t len = 0; - const HParser *arg; - - assert(p != NULL); - va_copy(ap, ap_); - do { - len++; - arg = va_arg(ap, HParser *); - } while (arg); - va_end(ap); HSequence *s = h_new(HSequence, 1); - s->p_array = h_new(HParser *, len); + s->len = 0; - va_copy(ap, ap_); - s->p_array[0] = p; - for (size_t i = 1; i < len; i++) { - s->p_array[i] = va_arg(ap, HParser *); - } while (arg); - va_end(ap); + if(p) { + // non-empty sequence + const HParser *arg; + size_t len = 0; + va_list ap; + + va_copy(ap, ap_); + do { + len++; + arg = va_arg(ap, HParser *); + } while (arg); + va_end(ap); + s->p_array = h_new(HParser *, len); + + va_copy(ap, ap_); + s->p_array[0] = p; + for (size_t i = 1; i < len; i++) { + s->p_array[i] = va_arg(ap, HParser *); + } while (arg); + va_end(ap); + + s->len = len; + } - s->len = len; return h_new_parser(mm__, &sequence_vt, s); } diff --git a/src/t_parser.c b/src/t_parser.c index c42eca9..331d262 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -241,6 +241,7 @@ static void test_nothing_p(gconstpointer backend) { static void test_sequence(gconstpointer backend) { const HParser *sequence_1 = h_sequence(h_ch('a'), h_ch('b'), NULL); const HParser *sequence_2 = h_sequence(h_ch('a'), h_whitespace(h_ch('b')), NULL); + const HParser *sequence_3 = h_sequence(NULL, NULL); // second NULL is to silence GCC g_check_parse_match(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)"); g_check_parse_failed(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1); @@ -248,6 +249,7 @@ static void test_sequence(gconstpointer backend) { g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x61 u0x62)"); g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", 3, "(u0x61 u0x62)"); g_check_parse_match(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", 4, "(u0x61 u0x62)"); + g_check_parse_match(sequence_3, (HParserBackend)GPOINTER_TO_INT(backend), "", 0, "()"); } static void test_choice(gconstpointer backend) { From 9e37fbe51312ff515150579d07c71676d048b205 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 14 Nov 2015 18:44:31 +0100 Subject: [PATCH 20/35] fix LIBPATH order for building test_suite --- src/SConscript | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SConscript b/src/SConscript index dd6b616..7a1b9d4 100644 --- a/src/SConscript +++ b/src/SConscript @@ -88,7 +88,8 @@ env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') -testenv.Append(LIBS=['hammer'], LIBPATH=['.']) +testenv.Append(LIBS=['hammer']) +testenv.Prepend(LIBPATH=['.']) ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS="--coverage" if testenv.GetOption("coverage") else None) ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path])) AlwaysBuild(ctest) From 5b3cb46c968a5960378b6d31412943422e3e207d Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sat, 14 Nov 2015 23:26:04 +0100 Subject: [PATCH 21/35] avoid GCC warning about potential longjmp clobbers --- src/backends/regex.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/backends/regex.c b/src/backends/regex.c index 3c2e803..f7dd98a 100644 --- a/src/backends/regex.c +++ b/src/backends/regex.c @@ -52,20 +52,21 @@ HRVMTrace *invert_trace(HRVMTrace *trace) { void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_t len) { HArena *arena = h_new_arena(mm__, 0); - HSArray *heads_n = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s - *heads_p = h_sarray_new(mm__, prog->length); + HSArray *heads_a = h_sarray_new(mm__, prog->length), // Both of these contain HRVMTrace*'s + *heads_b = h_sarray_new(mm__, prog->length); HRVMTrace *ret_trace = NULL; HParseResult *ret = NULL; // out of memory handling - if(!arena || !heads_n || !heads_p) + if(!arena || !heads_a || !heads_b) goto end; jmp_buf except; h_arena_set_except(arena, &except); if(setjmp(except)) goto end; + HSArray *heads_n = heads_a, *heads_p = heads_b; uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued HRVMThread *ip_queue = a_new(HRVMThread, prog->length); size_t ipq_top; @@ -184,8 +185,8 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_ end: if (arena) h_delete_arena(arena); - if (heads_n) h_sarray_free(heads_n); - if (heads_p) h_sarray_free(heads_p); + if (heads_a) h_sarray_free(heads_a); + if (heads_b) h_sarray_free(heads_b); return ret; } #undef PUSH_SVM From e26a8ff572e71b0e2690e7497f015aad09cf0711 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 27 Nov 2015 17:55:18 +0100 Subject: [PATCH 22/35] add libhammer.pc to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 721dcf9..570fbf8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ Session.vim *.gcov cscope.out build/ +libhammer.pc .sconsign.dblite *.os *.pyc From ca1d8df06cdead8f9de1ba81d1de8cda363c16b4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 30 Nov 2015 14:19:40 +0100 Subject: [PATCH 23/35] don't allocate a new arena in h_bind, use the existing one Rationale: If memory allocation fails in the inner parse and we longjump up the stack, the temporary arena will be missed and leak. NB: This change means that any allocations done by the continuation (in the form of new parsers, probably) will persist for the lifetime of the parse result. Beware of wasting too much memory this way! The bind continuation should generally keep dynamic allocations to a minimum. --- src/parsers/bind.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/parsers/bind.c b/src/parsers/bind.c index 7fa821d..808df97 100644 --- a/src/parsers/bind.c +++ b/src/parsers/bind.c @@ -4,7 +4,6 @@ typedef struct { const HParser *p; HContinuation k; void *env; - HAllocator *mm__; } BindEnv; // an HAllocator backed by an HArena @@ -39,20 +38,15 @@ static HParseResult *parse_bind(void *be_, HParseState *state) { if(!res) return NULL; - // create a temporary arena allocator for the continuation - HArena *arena = h_new_arena(be->mm__, 0); - ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, arena}; + // create a wrapper arena allocator for the continuation + ArenaAllocator aa = {{aa_alloc, aa_realloc, aa_free}, state->arena}; HParser *kx = be->k((HAllocator *)&aa, res->ast, be->env); if(!kx) { - h_delete_arena(arena); return NULL; } - res = h_do_parse(kx, state); - - h_delete_arena(arena); - return res; + return h_do_parse(kx, state); } static const HParserVtable bind_vt = { @@ -76,7 +70,6 @@ HParser *h_bind__m(HAllocator *mm__, be->p = p; be->k = k; be->env = env; - be->mm__ = mm__; return h_new_parser(mm__, &bind_vt, be); } From 3fc56a0dc3390e5a5ac94bacb2110ba4dcf673ee Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 30 Nov 2015 16:37:00 +0100 Subject: [PATCH 24/35] add h_alloc() which calls errx() on failure and use it for all basic allocation Rationale: "Basic allocation" refers to things outside of parsing proper, mostly initialization. If such allocations fail, the system is globally emory-starved from which it will likely not recover by returning failure. In this case, terminating the process is in fact the most robust strategy as it may mean the difference between a permanent hang and a temporary crash. --- src/allocator.c | 15 ++++++++++----- src/allocator.h | 24 +++++++++++++----------- src/bitwriter.c | 6 ++---- src/internal.h | 2 +- src/pprint.c | 6 ++---- src/registry.c | 6 ++---- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/allocator.c b/src/allocator.c index 4646dd8..cc259e6 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -47,15 +47,20 @@ struct HArena_ { jmp_buf *except; }; +void* h_alloc(HAllocator* mm__, size_t size) { + void *p = mm__->alloc(mm__, size); + if(!p) + h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size); + return p; +} + HArena *h_new_arena(HAllocator* mm__, size_t block_size) { if (block_size == 0) block_size = 4096; struct HArena_ *ret = h_new(struct HArena_, 1); - struct arena_link *link = (struct arena_link*)mm__->alloc(mm__, sizeof(struct arena_link) + block_size); - if (!link) { - // TODO: error-reporting -- let user know that arena link couldn't be allocated - return NULL; - } + struct arena_link *link = (struct arena_link*)h_alloc(mm__, sizeof(struct arena_link) + block_size); + assert(ret != NULL); + assert(link != NULL); memset(link, 0, sizeof(struct arena_link) + block_size); link->free = block_size; link->used = 0; diff --git a/src/allocator.h b/src/allocator.h index d80209d..dc88af6 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -24,17 +24,6 @@ extern "C" { #endif -// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer. -typedef struct HAllocator_ { - void* (*alloc)(struct HAllocator_* allocator, size_t size); - void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size); - void (*free)(struct HAllocator_* allocator, void* ptr); -} HAllocator; - -typedef struct HArena_ HArena ; // hidden implementation - -HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... - #if defined __llvm__ # if __has_attribute(malloc) # define ATTR_MALLOC(n) __attribute__((malloc)) @@ -49,6 +38,19 @@ HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for def # define ATTR_MALLOC(n) #endif +// TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer. +typedef struct HAllocator_ { + void* (*alloc)(struct HAllocator_* allocator, size_t size); + void* (*realloc)(struct HAllocator_* allocator, void* ptr, size_t size); + void (*free)(struct HAllocator_* allocator, void* ptr); +} HAllocator; + +void* h_alloc(HAllocator* allocator, size_t size) ATTR_MALLOC(2); + +typedef struct HArena_ HArena ; // hidden implementation + +HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... + void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2); void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bitwriter.c b/src/bitwriter.c index 74e2734..bcc21dd 100644 --- a/src/bitwriter.c +++ b/src/bitwriter.c @@ -12,10 +12,8 @@ HBitWriter *h_bit_writer_new(HAllocator* mm__) { HBitWriter *writer = h_new(HBitWriter, 1); memset(writer, 0, sizeof(*writer)); - writer->buf = mm__->alloc(mm__, writer->capacity = 8); - if (!writer) { - return NULL; - } + writer->buf = h_alloc(mm__, writer->capacity = 8); + assert(writer != NULL); memset(writer->buf, 0, writer->capacity); writer->mm__ = mm__; writer->flags = BYTE_BIG_ENDIAN | BIT_BIG_ENDIAN; diff --git a/src/internal.h b/src/internal.h index 776f636..10db4b2 100644 --- a/src/internal.h +++ b/src/internal.h @@ -49,7 +49,7 @@ rtype_t name##__m(HAllocator* mm__) // Functions with arguments are difficult to forward cleanly. Alas, we will need to forward them manually. -#define h_new(type, count) ((type*)(mm__->alloc(mm__, sizeof(type)*(count)))) +#define h_new(type, count) ((type*)(h_alloc(mm__, sizeof(type)*(count)))) #define h_free(addr) (mm__->free(mm__, (addr))) #ifndef __cplusplus diff --git a/src/pprint.c b/src/pprint.c index 11ec3d6..9c7c652 100644 --- a/src/pprint.c +++ b/src/pprint.c @@ -186,13 +186,11 @@ static void unamb_sub(const HParsedToken* tok, struct result_buf *buf) { char* h_write_result_unamb(const HParsedToken* tok) { struct result_buf buf = { - .output = (&system_allocator)->alloc(&system_allocator, 16), + .output = h_alloc(&system_allocator, 16), .len = 0, .capacity = 16 }; - if (!buf.output) { - return NULL; - } + assert(buf.output != NULL); unamb_sub(tok, &buf); append_buf_c(&buf, 0); return buf.output; diff --git a/src/registry.c b/src/registry.c index 2ebac1a..d905320 100644 --- a/src/registry.c +++ b/src/registry.c @@ -46,10 +46,8 @@ static int compare_entries(const void* v1, const void* v2) { } HTokenType h_allocate_token_type(const char* name) { - Entry* new_entry = (&system_allocator)->alloc(&system_allocator, sizeof(*new_entry)); - if (!new_entry) { - return TT_INVALID; - } + Entry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry)); + assert(new_entry != NULL); new_entry->name = name; new_entry->value = 0; Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries); From 5996477a5e96086501fd9140aa877c08d65edc3c Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 30 Nov 2015 17:50:29 +0100 Subject: [PATCH 25/35] out-of-memory handling in packrat backend --- src/backends/packrat.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index abb198b..b7e47ae 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -254,6 +254,15 @@ static bool pos_equal(const void* key1, const void* key2) { HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStream *input_stream) { HArena * arena = h_new_arena(mm__, 0); + + // out-of-memory handling + jmp_buf except; + h_arena_set_except(arena, &except); + if(setjmp(except)) { + h_delete_arena(arena); + return NULL; + } + HParseState *parse_state = a_new_(arena, HParseState, 1); parse_state->cache = h_hashtable_new(arena, cache_key_equal, // key_equal_func cache_key_hash); // hash_func From d5f3e133095bb93e5557c95976f9e48476dfcc88 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Mon, 30 Nov 2015 18:07:33 +0100 Subject: [PATCH 26/35] out-of-memory handling in LL(k), LR, and GLR backends --- src/backends/glr.c | 10 ++++++++++ src/backends/llk.c | 11 +++++++++++ src/backends/lr.c | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/src/backends/glr.c b/src/backends/glr.c index e753ea5..535dc28 100644 --- a/src/backends/glr.c +++ b/src/backends/glr.c @@ -198,6 +198,16 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *arena = h_new_arena(mm__, 0); // will hold the results HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse + // out-of-memory handling + jmp_buf except; + h_arena_set_except(arena, &except); + h_arena_set_except(tarena, &except); + if(setjmp(except)) { + h_delete_arena(arena); + h_delete_arena(tarena); + return NULL; + } + // allocate engine lists (will hold one engine per state) // these are swapped each iteration HSlist *engines = h_slist_new(tarena); diff --git a/src/backends/llk.c b/src/backends/llk.c index 0ab4610..2f41f91 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -561,6 +561,17 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* { HLLkState *s = llk_parse_start_(mm__, parser); + // out-of-memory handling + jmp_buf except; + h_arena_set_except(s->arena, &except); + h_arena_set_except(s->tarena, &except); + if(setjmp(except)) { + h_delete_arena(s->arena); + h_delete_arena(s->tarena); + h_free(s); + return NULL; + } + assert(stream->last_chunk); s->seq = llk_parse_chunk_(s, parser, stream); diff --git a/src/backends/lr.c b/src/backends/lr.c index fb256c0..8f2a0ea 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -388,6 +388,16 @@ HParseResult *h_lr_parse(HAllocator* mm__, const HParser* parser, HInputStream* HArena *tarena = h_new_arena(mm__, 0); // tmp, deleted after parse HLREngine *engine = h_lrengine_new(arena, tarena, table, stream); + // out-of-memory handling + jmp_buf except; + h_arena_set_except(arena, &except); + h_arena_set_except(tarena, &except); + if(setjmp(except)) { + h_delete_arena(arena); + h_delete_arena(tarena); + return NULL; + } + // iterate engine to completion while(h_lrengine_step(engine, h_lrengine_action(engine))); From 2309bd6da901c2a8287463e5d4e408aa6f456e77 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 16:48:01 +0100 Subject: [PATCH 27/35] out-of-memory support for iterative LL(k) --- src/backends/llk.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/backends/llk.c b/src/backends/llk.c index 2f41f91..3780cf1 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -389,6 +389,13 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, if(!seq) return NULL; // parse already failed + // out-of-memory handling + jmp_buf except; + h_arena_set_except(arena, &except); + h_arena_set_except(tarena, &except); + if(setjmp(except)) + goto no_parse; + if(s->win.length > 0) { append_win(kmax, s, chunk); stream = &s->win; @@ -530,8 +537,6 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, return seq; no_parse: - h_delete_arena(arena); - s->arena = NULL; return NULL; need_input: @@ -550,6 +555,8 @@ static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s) if(s->seq) { assert(s->seq->used == 1); res = make_result(s->arena, s->seq->elements[0]); + } else { + h_delete_arena(s->arena); } h_delete_arena(s->tarena); @@ -561,17 +568,6 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* { HLLkState *s = llk_parse_start_(mm__, parser); - // out-of-memory handling - jmp_buf except; - h_arena_set_except(s->arena, &except); - h_arena_set_except(s->tarena, &except); - if(setjmp(except)) { - h_delete_arena(s->arena); - h_delete_arena(s->tarena); - h_free(s); - return NULL; - } - assert(stream->last_chunk); s->seq = llk_parse_chunk_(s, parser, stream); From 3ce4f61cf9253c6f746626a36e66501f94557523 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 16:49:40 +0100 Subject: [PATCH 28/35] add test for out-of-memory handling --- src/t_misc.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/t_misc.c b/src/t_misc.c index 92c2b32..47afbe4 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -1,5 +1,6 @@ #include #include +#include #include "test_suite.h" #include "hammer.h" @@ -29,7 +30,43 @@ static void test_tt_registry(void) { g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, 0); } +// perform a big allocation during parsing to trigger out-of-memory handling +static HParsedToken *act_big_alloc(const HParseResult *r, void *user) { + void *buf = h_arena_malloc(r->arena, 1024*1024*1024); + assert(buf != NULL); + return NULL; +} +static void test_oom(void) { + HParser *p = h_action(h_ch('x'), act_big_alloc, NULL); + // this should always fail, but never crash + + struct rlimit bak, lim; + int i; + i = getrlimit(RLIMIT_DATA, &bak); + assert(i == 0); + lim.rlim_cur = 1000*1024*1024; // never enough + if(lim.rlim_cur > bak.rlim_max) + lim.rlim_cur = bak.rlim_max; + lim.rlim_max = bak.rlim_max; + i = setrlimit(RLIMIT_DATA, &lim); + assert(i == 0); + + g_check_parse_failed(p, PB_PACKRAT, "x",1); + g_check_parse_failed(p, PB_REGULAR, "x",1); + g_check_parse_failed(p, PB_LLk, "x",1); + g_check_parse_failed(p, PB_LALR, "x",1); + g_check_parse_failed(p, PB_GLR, "x",1); + + g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1); + //g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1); + //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1); + + i = setrlimit(RLIMIT_DATA, &bak); + assert(i == 0); +} + void register_misc_tests(void) { g_test_add_func("/core/misc/tt_user", test_tt_user); g_test_add_func("/core/misc/tt_registry", test_tt_registry); + g_test_add_func("/core/misc/oom", test_oom); } From 384a7b939039a8409849e41df1b5d17144152e5c Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 17:07:04 +0100 Subject: [PATCH 29/35] reset arena jmp_bufs at end of h_llk_parse_chunk --- src/backends/llk.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backends/llk.c b/src/backends/llk.c index 3780cf1..6954e60 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -589,6 +589,9 @@ bool h_llk_parse_chunk(HSuspendedParser *s, HInputStream *input) state->seq = llk_parse_chunk_(state, s->parser, input); + h_arena_set_except(state->arena, NULL); + h_arena_set_except(state->tarena, NULL); + return (state->seq == NULL || h_slist_empty(state->stack)); } From 22b5611cdfab7b460d03abadb2951d1eeedc19dd Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 17:08:02 +0100 Subject: [PATCH 30/35] add oom handling to iterative LR engine --- src/backends/lr.c | 13 +++++++++++++ src/t_misc.c | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/backends/lr.c b/src/backends/lr.c index 8f2a0ea..f2ac495 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -426,6 +426,16 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream) engine->input = *stream; bool run = true; + + // out-of-memory handling + jmp_buf except; + h_arena_set_except(engine->arena, &except); + h_arena_set_except(engine->tarena, &except); + if(setjmp(except)) { + run = false; // done immediately + assert(engine->state != HLR_SUCCESS); // h_parse_finish will return NULL + } + while(run) { // check input against table to determine which action to take const HLRAction *action = h_lrengine_action(engine); @@ -441,6 +451,9 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream) break; } + h_arena_set_except(engine->arena, NULL); + h_arena_set_except(engine->tarena, NULL); + *stream = engine->input; return !run; // done if engine no longer running } diff --git a/src/t_misc.c b/src/t_misc.c index 47afbe4..8cd73dc 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -57,8 +57,9 @@ static void test_oom(void) { g_check_parse_failed(p, PB_LALR, "x",1); g_check_parse_failed(p, PB_GLR, "x",1); + //g_check_parse_chunks_failed(p, PB_REGULAR, "",0, "x",1); g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1); - //g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1); + g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1); //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1); i = setrlimit(RLIMIT_DATA, &bak); From e89d9f9134efb82d524bdc6defd6d09b2c526260 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 17:47:37 +0100 Subject: [PATCH 31/35] delete results properly in parser test macros --- src/test_suite.h | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/test_suite.h b/src/test_suite.h index 49f13cf..d36820f 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -78,15 +78,6 @@ } while(0) -// TODO: replace uses of this with g_check_parse_failed -#define g_check_failed(res) do { \ - const HParseResult *result = (res); \ - if (NULL != result) { \ - g_test_message("Check failed: shouldn't have succeeded, but did"); \ - g_test_fail(); \ - } \ - } while(0) - #define g_check_parse_failed(parser, backend, input, inp_len) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \ if(skip != 0) { \ @@ -94,8 +85,9 @@ g_test_fail(); \ break; \ } \ - const HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \ + HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \ if (NULL != result) { \ + h_parse_result_free(result); \ g_test_message("Check failed: shouldn't have succeeded, but did"); \ g_test_fail(); \ } \ @@ -119,7 +111,7 @@ "Inefficiency: %5f%%", \ stats.used, stats.wasted, \ stats.wasted * 100. / (stats.used+stats.wasted)); \ - h_delete_arena(res->arena); \ + h_parse_result_free(res); \ } \ } while(0) @@ -144,7 +136,7 @@ "Inefficiency: %5f%%", \ stats.used, stats.wasted, \ stats.wasted * 100. / (stats.used+stats.wasted)); \ - h_delete_arena(res->arena); \ + h_parse_result_free(res); \ } \ } while(0) @@ -167,8 +159,9 @@ } \ h_parse_chunk(s, (const uint8_t*)chunk1, c1_len); \ h_parse_chunk(s, (const uint8_t*)chunk2, c2_len); \ - const HParseResult *res = h_parse_finish(s); \ + HParseResult *res = h_parse_finish(s); \ if (NULL != res) { \ + h_parse_result_free(res); \ g_test_message("Check failed: shouldn't have succeeded, but did"); \ g_test_fail(); \ } \ @@ -207,7 +200,7 @@ "Inefficiency: %5f%%", \ stats.used, stats.wasted, \ stats.wasted * 100. / (stats.used+stats.wasted)); \ - h_delete_arena(res->arena); \ + h_parse_result_free(res); \ } \ } while(0) From db4fd66eaf6f56a0fc90734ede702c3c92ffda9f Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 1 Dec 2015 18:12:21 +0100 Subject: [PATCH 32/35] try it with a smaller alloc --- src/t_misc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/t_misc.c b/src/t_misc.c index 8cd73dc..59d6260 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -32,8 +32,9 @@ static void test_tt_registry(void) { // perform a big allocation during parsing to trigger out-of-memory handling static HParsedToken *act_big_alloc(const HParseResult *r, void *user) { - void *buf = h_arena_malloc(r->arena, 1024*1024*1024); + void *buf = h_arena_malloc(r->arena, 500*1024*1024); assert(buf != NULL); + g_test_message("Memory allocation was supposed to fail"); return NULL; } static void test_oom(void) { @@ -44,7 +45,7 @@ static void test_oom(void) { int i; i = getrlimit(RLIMIT_DATA, &bak); assert(i == 0); - lim.rlim_cur = 1000*1024*1024; // never enough + lim.rlim_cur = 499*1024*1024; // never enough if(lim.rlim_cur > bak.rlim_max) lim.rlim_cur = bak.rlim_max; lim.rlim_max = bak.rlim_max; From 7b13a82851f78a118e3014612ea8640fa49c6670 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 2 Dec 2015 13:28:24 +0100 Subject: [PATCH 33/35] add __m variants to some check macros --- src/test_suite.h | 59 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/src/test_suite.h b/src/test_suite.h index d36820f..83359f9 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -78,14 +78,14 @@ } while(0) -#define g_check_parse_failed(parser, backend, input, inp_len) do { \ - int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \ +#define g_check_parse_failed__m(mm__, parser, backend, input, inp_len) do { \ + int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \ if(skip != 0) { \ g_test_message("Compile failed"); \ g_test_fail(); \ break; \ } \ - HParseResult *result = h_parse(parser, (const uint8_t*)input, inp_len); \ + HParseResult *result = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \ if (NULL != result) { \ h_parse_result_free(result); \ g_test_message("Check failed: shouldn't have succeeded, but did"); \ @@ -93,6 +93,9 @@ } \ } while(0) +#define g_check_parse_failed(p, be, input, len) \ + g_check_parse_failed__m(&system_allocator, p, be, input, len) + #define g_check_parse_ok(parser, backend, input, inp_len) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ if(skip) { \ @@ -140,18 +143,18 @@ } \ } while(0) -#define g_check_parse_chunks_failed(parser, backend, chunk1, c1_len, chunk2, c2_len) do { \ - int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \ +#define g_check_parse_chunks_failed__m(mm__, parser, backend, chunk1, c1_len, chunk2, c2_len) do { \ + int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \ if(skip) { \ g_test_message("Compile failed"); \ g_test_fail(); \ break; \ } \ - g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len); \ + g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len); \ } while(0) -#define g_check_parse_chunks_failed_(parser, chunk1, c1_len, chunk2, c2_len) do { \ - HSuspendedParser *s = h_parse_start(parser); \ +#define g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len) do { \ + HSuspendedParser *s = h_parse_start__m(mm__, parser); \ if(!s) { \ g_test_message("Chunk-wise parsing not available"); \ g_test_fail(); \ @@ -167,6 +170,46 @@ } \ } while(0) +#define g_check_parse_chunks_failed(p, be, c1, c1_len, c2, c2_len) \ + g_check_parse_chunks_failed__m(&system_allocator, p, be, c1, c1_len, c2, c2_len) + +#define g_check_parse_chunks_failed_(p, c1, c1_len, c2, c2_len) \ + g_check_parse_chunks_failed___m(&system_allocator, p, c1, c1_len, c2, c2_len) + +#define g_check_parse_chunks_ok(parser, backend, chunk1, c1_len, chunk2, c2_len) do { \ + int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \ + if(skip) { \ + g_test_message("Compile failed"); \ + g_test_fail(); \ + break; \ + } \ + g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len); \ + } while(0) + +#define g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len) do { \ + HSuspendedParser *s = h_parse_start(parser); \ + if(!s) { \ + g_test_message("Chunk-wise parsing not available"); \ + g_test_fail(); \ + break; \ + } \ + h_parse_chunk(s, (const uint8_t*)chunk1, c1_len); \ + h_parse_chunk(s, (const uint8_t*)chunk2, c2_len); \ + HParseResult *res = h_parse_finish(s); \ + if (!res) { \ + g_test_message("Parse failed on line %d", __LINE__); \ + g_test_fail(); \ + } else { \ + HArenaStats stats; \ + h_allocator_stats(res->arena, &stats); \ + g_test_message("Parse used %zd bytes, wasted %zd bytes. " \ + "Inefficiency: %5f%%", \ + stats.used, stats.wasted, \ + stats.wasted * 100. / (stats.used+stats.wasted)); \ + h_parse_result_free(res); \ + } \ + } while(0) + #define g_check_parse_chunks_match(parser, backend, chunk1, c1_len, chunk2, c2_len, result) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ if(skip) { \ From 9602caf64f055e3ac312d9c8ab01c7d8dc573b97 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 2 Dec 2015 13:28:54 +0100 Subject: [PATCH 34/35] test out-of-memory handling with a mock allocator --- src/t_misc.c | 67 +++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/src/t_misc.c b/src/t_misc.c index 59d6260..04ce96c 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -30,41 +30,50 @@ static void test_tt_registry(void) { g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, 0); } -// perform a big allocation during parsing to trigger out-of-memory handling -static HParsedToken *act_big_alloc(const HParseResult *r, void *user) { - void *buf = h_arena_malloc(r->arena, 500*1024*1024); +// test out-of-memory handling with a selectively failing allocator +static void *fail_alloc(HAllocator *mm__, size_t size) { + if(size - 0xdead <= 0x30) // allow for overhead of arena link structure + return NULL; + return system_allocator.alloc(&system_allocator, size); +} +static void *fail_realloc(HAllocator *mm__, void *ptr, size_t size) { + return system_allocator.realloc(&system_allocator, ptr, size); +} +static void fail_free(HAllocator *mm__, void *ptr) { + return system_allocator.free(&system_allocator, ptr); +} +static HAllocator fail_allocator = {fail_alloc, fail_realloc, fail_free}; +static HParsedToken *act_oom(const HParseResult *r, void *user) { + void *buf = h_arena_malloc(r->arena, 0xdead); assert(buf != NULL); - g_test_message("Memory allocation was supposed to fail"); - return NULL; + return NULL; // succeed with null token } static void test_oom(void) { - HParser *p = h_action(h_ch('x'), act_big_alloc, NULL); + HParser *p = h_action(h_ch('x'), act_oom, NULL); // this should always fail, but never crash - struct rlimit bak, lim; - int i; - i = getrlimit(RLIMIT_DATA, &bak); - assert(i == 0); - lim.rlim_cur = 499*1024*1024; // never enough - if(lim.rlim_cur > bak.rlim_max) - lim.rlim_cur = bak.rlim_max; - lim.rlim_max = bak.rlim_max; - i = setrlimit(RLIMIT_DATA, &lim); - assert(i == 0); + // sanity-check: parses should succeed with the normal allocator... + g_check_parse_ok(p, PB_PACKRAT, "x",1); + g_check_parse_ok(p, PB_REGULAR, "x",1); + g_check_parse_ok(p, PB_LLk, "x",1); + g_check_parse_ok(p, PB_LALR, "x",1); + g_check_parse_ok(p, PB_GLR, "x",1); + //XXX g_check_parse_chunks_ok(p, PB_REGULAR, "",0, "x",1); + g_check_parse_chunks_ok(p, PB_LLk, "",0, "x",1); + g_check_parse_chunks_ok(p, PB_LALR, "",0, "x",1); + //XXX g_check_parse_chunks_ok(p, PB_GLR, "",0, "x",1); - g_check_parse_failed(p, PB_PACKRAT, "x",1); - g_check_parse_failed(p, PB_REGULAR, "x",1); - g_check_parse_failed(p, PB_LLk, "x",1); - g_check_parse_failed(p, PB_LALR, "x",1); - g_check_parse_failed(p, PB_GLR, "x",1); - - //g_check_parse_chunks_failed(p, PB_REGULAR, "",0, "x",1); - g_check_parse_chunks_failed(p, PB_LLk, "",0, "x",1); - g_check_parse_chunks_failed(p, PB_LALR, "",0, "x",1); - //g_check_parse_chunks_failed(p, PB_GLR, "",0, "x",1); - - i = setrlimit(RLIMIT_DATA, &bak); - assert(i == 0); + // ...and fail gracefully with the broken one + HAllocator *mm__ = &fail_allocator; + g_check_parse_failed__m(mm__, p, PB_PACKRAT, "x",1); + g_check_parse_failed__m(mm__, p, PB_REGULAR, "x",1); + g_check_parse_failed__m(mm__, p, PB_LLk, "x",1); + g_check_parse_failed__m(mm__, p, PB_LALR, "x",1); + g_check_parse_failed__m(mm__, p, PB_GLR, "x",1); + //XXX g_check_parse_chunks_failed__m(mm__, p, PB_REGULAR, "",0, "x",1); + g_check_parse_chunks_failed__m(mm__, p, PB_LLk, "",0, "x",1); + g_check_parse_chunks_failed__m(mm__, p, PB_LALR, "",0, "x",1); + //XXX g_check_parse_chunks_failed__m(mm__, p, PB_GLR, "",0, "x",1); } void register_misc_tests(void) { From d28f182c4f62990899609e674239c4f134e834b8 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Fri, 4 Dec 2015 13:13:06 +0100 Subject: [PATCH 35/35] clear arena exception handler when exiting llk_parse_chunk_ --- src/backends/llk.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/backends/llk.c b/src/backends/llk.c index 6954e60..4e8209b 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -383,10 +383,9 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, HArena *arena = s->arena; HArena *tarena = s->tarena; HSlist *stack = s->stack; - HCountedArray *seq = s->seq; size_t kmax = table->kmax; - if(!seq) + if(!s->seq) return NULL; // parse already failed // out-of-memory handling @@ -396,6 +395,8 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, if(setjmp(except)) goto no_parse; + HCountedArray *seq = s->seq; + if(s->win.length > 0) { append_win(kmax, s, chunk); stream = &s->win; @@ -534,10 +535,15 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, // since we started with a single nonterminal on the stack, seq should // contain exactly the parse result. assert(seq->used == 1); + + end: + h_arena_set_except(arena, NULL); + h_arena_set_except(tarena, NULL); return seq; no_parse: - return NULL; + seq = NULL; + goto end; need_input: if(stream->last_chunk) @@ -545,7 +551,7 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser, if(tok) h_arena_free(arena, tok); // no result, yet h_slist_push(stack, x); // try this symbol again next time - return seq; + goto end; } static HParseResult *llk_parse_finish_(HAllocator *mm__, HLLkState *s)