From 2408106191956f20360c3663e4ba0d11ee6cb0d8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Fri, 9 Aug 2013 18:45:26 +0200 Subject: [PATCH 001/103] C++ bindings getting underway. --- src/hammer.cxx | 152 +++++++++++++++ src/hammer.h | 10 + src/hammer.hxx | 493 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 655 insertions(+) create mode 100644 src/hammer.cxx create mode 100644 src/hammer.hxx diff --git a/src/hammer.cxx b/src/hammer.cxx new file mode 100644 index 0000000..dc2b104 --- /dev/null +++ b/src/hammer.cxx @@ -0,0 +1,152 @@ +#include "hammer.hxx" + +namespace hammer { + + typedef variant AnyResult; + + const BytesResult::result_type BytesResult::result() { return _bytes; } + const UintResult::result_type UintResult::result() { return _uint; } + const IntResult::result_type IntResult::result() { return _sint; } + const SequenceResult::result_type SequenceResult::result() { return _seq; } + + template<> + BytesResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); + } + + template<> + BytesResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); + } + + template<> + UintResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return UintResult(res->ast->uint); + } + + template<> + UintResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return UintResult(res->ast->uint); + } + + template<> + IntResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return IntResult(res->ast->sint); + } + + template<> + IntResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return IntResult(res->ast->sint); + } + + template<> + NullResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return NullResult(); + } + + template<> + NullResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return NullResult(); + } + + vector make_seq(HCountedArray *seq) { + vector ret; + for (size_t i=0; iused; ++i) { + switch(seq->elements[i]->token_type) { + case TT_NONE: + ret.push_back(NullResult()); + break; + case TT_BYTES: + ret.push_back(BytesResult(vector(seq->elements[i]->bytes.token, seq->elements[i]->bytes.token+seq->elements[i]->bytes.len))); + break; + case TT_SINT: + ret.push_back(IntResult(seq->elements[i]->sint)); + break; + case TT_UINT: + ret.push_back(UintResult(seq->elements[i]->uint)); + break; + case TT_SEQUENCE: + ret.push_back(make_seq(seq->elements[i]->seq)); + break; + default: + //TODO some kind of error + break; + } + } + return ret; + } + + template<> + SequenceResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return SequenceResult(make_seq(res->ast->seq)); + } + + template<> + SequenceResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return SequenceResult(make_seq(res->ast->seq)); + } + + template + Many Parser::many() { + return Many(this); + } + + template + RepeatN Parser::many(size_t n) { + return RepeatN(this, n); + } + + template + Optional Parser::optional() { + return Optional(this); + } + + template + RepeatN Parser::operator[](size_t n) { + return RepeatN(this, n); + } + + /* + Int64 Int64::in_range(const int64_t lower, const int64_t upper) { + return IntRange(Int64(), lower, upper); + } + + Int32 Int32::in_range(const int32_t lower, const int32_t upper) { + return IntRange(Int32(), lower, upper); + } + + Int16 Int16::in_range(const int16_t lower, const int16_t upper) { + return IntRange(Int16(), lower, upper); + } + + Int8 Int8::in_range(const int8_t lower, const int8_t upper) { + return IntRange(Int8(), lower, upper); + } + + Uint64 Uint64::in_range(const uint64_t lower, const uint64_t upper) { + return IntRange(Uint64(), lower, upper); + } + + Uint32 Uint32::in_range(const uint32_t lower, const uint32_t upper) { + return IntRange(Uint32(), lower, upper); + } + + Uint16 Uint16::in_range(const uint16_t lower, const uint16_t upper) { + return IntRange(Uint16(), lower, upper); + } + + Uint8 Uint8::in_range(const uint8_t lower, const uint8_t upper) { + return IntRange(Uint8(), lower, upper); + } + */ +} diff --git a/src/hammer.h b/src/hammer.h index 67fb8e4..89a9570 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -27,7 +27,13 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __cplusplus typedef int bool; +#endif typedef struct HParseState_ HParseState; @@ -617,4 +623,8 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results); void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} +#ifdef __cplusplus +} +#endif + #endif // #ifndef HAMMER_HAMMER__H diff --git a/src/hammer.hxx b/src/hammer.hxx new file mode 100644 index 0000000..0baee86 --- /dev/null +++ b/src/hammer.hxx @@ -0,0 +1,493 @@ +#ifndef HAMMER_HAMMER__HXX +#define HAMMER_HAMMER__HXX + +#include "hammer.h" +#include +#include +#include +#include + +using std::list; using std::string; using std::vector; +using boost::variant; + +namespace hammer { + + template + class ParseResult { + public: + typedef T result_type; + protected: + ParseResult() { } + }; + + class BytesResult : public ParseResult > { + public: + typedef vector result_type; + BytesResult(const vector res) : _bytes(res) { } + const result_type result(); + private: + BytesResult() { } + result_type _bytes; + }; + + class UintResult : public ParseResult { + public: + typedef uint64_t result_type; + UintResult(const uint64_t res) : _uint(res) { } + const result_type result(); + private: + UintResult() { } + result_type _uint; + }; + + class IntResult : public ParseResult { + public: + typedef int64_t result_type; + IntResult(const int64_t res) : _sint(res) { } + const result_type result(); + private: + IntResult() { } + result_type _sint; + }; + + class NullResult: public ParseResult { + public: + NullResult() { } + typedef void* result_type; + const result_type result() { return NULL; } + }; + + class SequenceResult : public ParseResult > > { + public: + typedef vector > result_type; + SequenceResult(result_type res) : _seq(res) { } + const result_type result(); + private: + SequenceResult() { } + result_type _seq; + }; + + /* forward declarations */ + template class Many; + template class Many1; + template class Optional; + template class RepeatN; + + template + class Parser { + public: + typedef T result_type; + result_type parse(const string &input); + result_type parse(const uint8_t *input, size_t length); + Many many(); + RepeatN many(size_t n); + Optional optional(); + RepeatN operator[](size_t n); + HParser* parser() { return _parser; } + HParser* _parser; + Parser() { } + }; + + class Token : public Parser { + public: + Token(string &str) : _tok(str) { + _parser = h_token(reinterpret_cast(str.c_str()), str.size()); + } + Token(const uint8_t *str, size_t length) : _tok(reinterpret_cast(str), length) { + _parser = h_token(str, length); + } + private: + string _tok; + }; + + class Ch : public Parser { + public: + Ch(const uint8_t c) : _c(c) { + _parser = h_ch(c); + } + private: + uint8_t _c; + }; + + class ChRange : public Parser { + public: + ChRange(const uint8_t lower, const uint8_t upper) : _lower(lower), _upper(upper) { + _parser = h_ch_range(lower, upper); + } + private: + uint8_t _lower, _upper; + }; + + class IntRange : public Parser { + public: + IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { + _parser = h_int_range(p.parser(), lower, upper); + } + private: + Parser _p; + int64_t _lower, _upper; + }; + + class SignedBits : public Parser { + public: + SignedBits(size_t len) : _len(len) { + _parser = h_bits(len, true); + } + private: + size_t _len; + }; + + class UnsignedBits : public Parser { + public: + UnsignedBits(size_t len) : _len(len) { + _parser = h_bits(len, false); + } + private: + size_t _len; + }; + + class Int64 : public Parser { + public: + Int64() { + _parser = h_int64(); + } + Int64 in_range(const int64_t lower, const int64_t upper); + }; + + class Int32 : public Parser { + public: + Int32() { + _parser = h_int32(); + } + Int32 in_range(const int32_t lower, const int32_t upper); + }; + + class Int16 : public Parser { + public: + Int16() { + _parser = h_int16(); + } + Int16 in_range(const int16_t lower, const int16_t upper); + }; + + class Int8 : public Parser { + public: + Int8() { + _parser = h_int8(); + } + Int8 in_range(const int8_t lower, const int8_t upper); + }; + + class Uint64 : public Parser { + public: + Uint64() { + _parser = h_uint64(); + } + Uint64 in_range(const uint64_t lower, const uint64_t upper); + }; + + class Uint32 : public Parser { + public: + Uint32() { + _parser = h_uint32(); + } + Uint32 in_range(const uint32_t lower, const uint32_t upper); + }; + + class Uint16 : public Parser { + public: + Uint16() { + _parser = h_uint16(); + } + Uint16 in_range(const uint16_t lower, const uint16_t upper); + }; + + class Uint8 : public Parser { + public: + Uint8() { + _parser = h_uint8(); + } + Uint8 in_range(const uint8_t lower, const uint8_t upper); + }; + + template + class Whitespace : public Parser { + public: + typedef typename T::result_type result_type; + Whitespace(Parser &p) : _p(p) { + this->_parser = h_whitespace(p.parser()); + } + private: + Parser _p; + }; + + template + class Left : public Parser { + public: + typedef typename T::result_type result_type; + Left(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_left(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Right : public Parser { + public: + typedef typename U::result_type result_type; + Right(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_right(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Middle : public Parser { + public: + typedef typename U::result_type result_type; + Middle(Parser &p, Parser &x, Parser &q) : _p(p), _x(x), _q(q) { + this->_parser = h_middle(p.parser(), x.parser(), q.parser()); + } + private: + Parser _p; + Parser _x; + Parser _q; + }; + + /* what are we doing about h_action? */ + + class In : public Parser { + public: + In(string &charset) : _charset(charset) { + _parser = h_in(reinterpret_cast(charset.c_str()), charset.size()); + } + In(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { + _parser = h_in(charset, length); + } + private: + string _charset; + }; + + class NotIn : public Parser { + public: + NotIn(string &charset) : _charset(charset) { + _parser = h_not_in(reinterpret_cast(charset.c_str()), charset.size()); + } + NotIn(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { + _parser = h_not_in(charset, length); + } + private: + string _charset; + }; + + class End : public Parser { + public: + End() { + _parser = h_end_p(); + } + }; + + class Nothing : public Parser { + public: + Nothing() { + _parser = h_nothing_p(); + } + }; + + class Sequence : public Parser { + public: + Sequence(list &ps) : _ps(ps) { + void *parsers[ps.size()]; + size_t i = 0; + for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { + parsers[i] = it->parser(); + } + _parser = h_sequence__a(parsers); + } + // maybe also a begin and end iterator version + private: + list _ps; + }; + + class Choice : public Parser { + public: + Choice(list > &ps) : _ps(ps) { + void *parsers[ps.size()]; + size_t i = 0; + for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { + parsers[i] = it->parser(); + } + _parser = h_choice__a(parsers); + } + private: + list > _ps; + }; + + template + class ButNot : public Parser { + public: + typedef typename T::result_type result_type; + ButNot(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_butnot(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Difference : public Parser { + public: + typedef typename T::result_type result_type; + Difference(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_difference(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Xor : public Parser > { + public: + typedef variant result_type; + Xor(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_xor(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Many : public Parser { + public: + Many(Parser &p) : _p(p) { + _parser = h_many(p.parser()); + } + private: + Parser _p; + }; + + template + class Many1 : public Parser { + public: + Many1(Parser &p) : _p(p) { + _parser = h_many1(p.parser()); + } + private: + Parser _p; + }; + + template + class RepeatN: public Parser { + public: + RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { + _parser = h_repeat_n(p.parser(), n); + } + private: + Parser _p; + size_t _n; + }; + + template + class Optional : public Parser { + public: + typedef typename T::result_type result_type; + Optional(Parser &p) : _p(p) { + this->_parser = h_optional(p.parser()); + } + private: + Parser _p; + }; + + template + class Ignore : public Parser { + public: + Ignore(Parser &p) : _p(p) { + _parser = h_ignore(p.parser()); + } + private: + Parser _p; + }; + + template + class SepBy : public Parser { + public: + SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { + _parser = h_sepBy(p.parser(), sep.parser()); + } + private: + Parser _p; + Parser _sep; + }; + + template + class SepBy1 : public Parser { + public: + SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { + _parser = h_sepBy1(p.parser(), sep.parser()); + } + private: + Parser _p; + Parser _sep; + }; + + class Epsilon : public Parser { + public: + Epsilon() { + _parser = h_epsilon_p(); + } + }; + + template + class LengthValue : public Parser { + public: + LengthValue(Parser &length, Parser &value) : _length(length), _value(value) { + _parser = h_length_value(length.parser(), value.parser()); + } + private: + Parser _length; + Parser _value; + }; + + /* FIXME attr_bool */ + + template + class And : public Parser { + public: + And(Parser &p) : _p(p) { + _parser = h_and(p.parser()); + } + private: + Parser _p; + }; + + template + class Not : public Parser { + public: + Not(Parser &p) : _p(p) { + _parser = h_not(p.parser()); + } + private: + Parser _p; + }; + + template + class Indirect : public Parser { + public: + Indirect(Parser &p) : _p(p) { + this->_parser = h_indirect(); + h_bind_indirect(this->_parser, p.parser()); + } + private: + Parser _p; + }; + +} + +#endif From 2e76cf2acfd8df306f1ef67f7c5b850aaa0d3f96 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Fri, 9 Aug 2013 22:49:16 +0200 Subject: [PATCH 002/103] and now in_range works too. YEAH. --- src/hammer.cxx | 43 +++++++++++++++++++++++++------------------ src/hammer.hxx | 49 +++++++++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/hammer.cxx b/src/hammer.cxx index dc2b104..1cf38c2 100644 --- a/src/hammer.cxx +++ b/src/hammer.cxx @@ -116,37 +116,44 @@ namespace hammer { return RepeatN(this, n); } - /* - Int64 Int64::in_range(const int64_t lower, const int64_t upper) { - return IntRange(Int64(), lower, upper); + IntRange Int64::in_range(const int64_t lower, const int64_t upper) { + Int64 p = Int64(); + return IntRange(p, lower, upper); } - Int32 Int32::in_range(const int32_t lower, const int32_t upper) { - return IntRange(Int32(), lower, upper); + IntRange Int32::in_range(const int32_t lower, const int32_t upper) { + Int32 p = Int32(); + return IntRange(p, lower, upper); } - Int16 Int16::in_range(const int16_t lower, const int16_t upper) { - return IntRange(Int16(), lower, upper); + IntRange Int16::in_range(const int16_t lower, const int16_t upper) { + Int16 p = Int16(); + return IntRange(p, lower, upper); } - Int8 Int8::in_range(const int8_t lower, const int8_t upper) { - return IntRange(Int8(), lower, upper); + IntRange Int8::in_range(const int8_t lower, const int8_t upper) { + Int8 p = Int8(); + return IntRange(p, lower, upper); } - Uint64 Uint64::in_range(const uint64_t lower, const uint64_t upper) { - return IntRange(Uint64(), lower, upper); + IntRange Uint64::in_range(const uint64_t lower, const uint64_t upper) { + Uint64 p = Uint64(); + return IntRange(p, lower, upper); } - Uint32 Uint32::in_range(const uint32_t lower, const uint32_t upper) { - return IntRange(Uint32(), lower, upper); + IntRange Uint32::in_range(const uint32_t lower, const uint32_t upper) { + Uint32 p = Uint32(); + return IntRange(p, lower, upper); } - Uint16 Uint16::in_range(const uint16_t lower, const uint16_t upper) { - return IntRange(Uint16(), lower, upper); + IntRange Uint16::in_range(const uint16_t lower, const uint16_t upper) { + Uint16 p = Uint16(); + return IntRange(p, lower, upper); } - Uint8 Uint8::in_range(const uint8_t lower, const uint8_t upper) { - return IntRange(Uint8(), lower, upper); + IntRange Uint8::in_range(const uint8_t lower, const uint8_t upper) { + Uint8 p = Uint8(); + return IntRange(p, lower, upper); } - */ + } diff --git a/src/hammer.hxx b/src/hammer.hxx index 0baee86..287ca21 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,6 +72,7 @@ namespace hammer { template class Many1; template class Optional; template class RepeatN; + template class IntRange; template class Parser { @@ -83,9 +84,11 @@ namespace hammer { RepeatN many(size_t n); Optional optional(); RepeatN operator[](size_t n); - HParser* parser() { return _parser; } - HParser* _parser; + const HParser* parser() { return _parser; } + protected: + const HParser* _parser; Parser() { } + // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... }; class Token : public Parser { @@ -118,16 +121,6 @@ namespace hammer { uint8_t _lower, _upper; }; - class IntRange : public Parser { - public: - IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { - _parser = h_int_range(p.parser(), lower, upper); - } - private: - Parser _p; - int64_t _lower, _upper; - }; - class SignedBits : public Parser { public: SignedBits(size_t len) : _len(len) { @@ -151,7 +144,7 @@ namespace hammer { Int64() { _parser = h_int64(); } - Int64 in_range(const int64_t lower, const int64_t upper); + IntRange in_range(const int64_t lower, const int64_t upper); }; class Int32 : public Parser { @@ -159,7 +152,7 @@ namespace hammer { Int32() { _parser = h_int32(); } - Int32 in_range(const int32_t lower, const int32_t upper); + IntRange in_range(const int32_t lower, const int32_t upper); }; class Int16 : public Parser { @@ -167,7 +160,7 @@ namespace hammer { Int16() { _parser = h_int16(); } - Int16 in_range(const int16_t lower, const int16_t upper); + IntRange in_range(const int16_t lower, const int16_t upper); }; class Int8 : public Parser { @@ -175,7 +168,7 @@ namespace hammer { Int8() { _parser = h_int8(); } - Int8 in_range(const int8_t lower, const int8_t upper); + IntRange in_range(const int8_t lower, const int8_t upper); }; class Uint64 : public Parser { @@ -183,7 +176,7 @@ namespace hammer { Uint64() { _parser = h_uint64(); } - Uint64 in_range(const uint64_t lower, const uint64_t upper); + IntRange in_range(const uint64_t lower, const uint64_t upper); }; class Uint32 : public Parser { @@ -191,7 +184,7 @@ namespace hammer { Uint32() { _parser = h_uint32(); } - Uint32 in_range(const uint32_t lower, const uint32_t upper); + IntRange in_range(const uint32_t lower, const uint32_t upper); }; class Uint16 : public Parser { @@ -199,7 +192,7 @@ namespace hammer { Uint16() { _parser = h_uint16(); } - Uint16 in_range(const uint16_t lower, const uint16_t upper); + IntRange in_range(const uint16_t lower, const uint16_t upper); }; class Uint8 : public Parser { @@ -207,7 +200,18 @@ namespace hammer { Uint8() { _parser = h_uint8(); } - Uint8 in_range(const uint8_t lower, const uint8_t upper); + IntRange in_range(const uint8_t lower, const uint8_t upper); + }; + + template + class IntRange : public Parser { + public: + IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { + this->_parser = h_int_range(p.parser(), lower, upper); + } + private: + Parser _p; + int64_t _lower, _upper; }; template @@ -299,12 +303,13 @@ namespace hammer { }; class Sequence : public Parser { + friend class Parser; public: Sequence(list &ps) : _ps(ps) { void *parsers[ps.size()]; size_t i = 0; for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = it->parser(); + parsers[i] = const_cast(it->parser()); } _parser = h_sequence__a(parsers); } @@ -319,7 +324,7 @@ namespace hammer { void *parsers[ps.size()]; size_t i = 0; for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = it->parser(); + parsers[i] = const_cast(it->parser()); } _parser = h_choice__a(parsers); } From 691f3fb5472a2e58974d079d1a004d3900025d01 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 11 Aug 2013 01:13:07 +0200 Subject: [PATCH 003/103] Many apparently requires a template specialization. This is absurd, but whatever. I'll ask stackoverflow. --- src/hammer.cxx | 15 ++++++++++----- src/hammer.hxx | 49 +++++++++++++++++++++---------------------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/hammer.cxx b/src/hammer.cxx index 1cf38c2..e7f3cb7 100644 --- a/src/hammer.cxx +++ b/src/hammer.cxx @@ -98,12 +98,17 @@ namespace hammer { template Many Parser::many() { - return Many(this); + return Many(*this); + } + + template<> + Many Parser::many() { + return Many(*this); } template - RepeatN Parser::many(size_t n) { - return RepeatN(this, n); + RepeatN Parser::many(size_t n) { + return RepeatN(this, n); } template @@ -112,8 +117,8 @@ namespace hammer { } template - RepeatN Parser::operator[](size_t n) { - return RepeatN(this, n); + RepeatN Parser::operator[](size_t n) { + return RepeatN(this, n); } IntRange Int64::in_range(const int64_t lower, const int64_t upper) { diff --git a/src/hammer.hxx b/src/hammer.hxx index 287ca21..8f3200c 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -69,9 +69,9 @@ namespace hammer { /* forward declarations */ template class Many; - template class Many1; + class Many1; template class Optional; - template class RepeatN; + class RepeatN; template class IntRange; template @@ -81,12 +81,12 @@ namespace hammer { result_type parse(const string &input); result_type parse(const uint8_t *input, size_t length); Many many(); - RepeatN many(size_t n); + RepeatN many(size_t n); Optional optional(); - RepeatN operator[](size_t n); - const HParser* parser() { return _parser; } + RepeatN operator[](size_t n); + HParser* parser() { return _parser; } protected: - const HParser* _parser; + HParser* _parser; Parser() { } // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... }; @@ -105,6 +105,7 @@ namespace hammer { class Ch : public Parser { public: + friend class Parser; Ch(const uint8_t c) : _c(c) { _parser = h_ch(c); } @@ -378,24 +379,22 @@ namespace hammer { Parser _p; }; - template class Many1 : public Parser { public: - Many1(Parser &p) : _p(p) { + Many1(Parser &p) : _p(p) { _parser = h_many1(p.parser()); } private: - Parser _p; + Parser _p; }; - template class RepeatN: public Parser { public: - RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { + RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { _parser = h_repeat_n(p.parser(), n); } private: - Parser _p; + Parser _p; size_t _n; }; @@ -410,36 +409,31 @@ namespace hammer { Parser _p; }; - template class Ignore : public Parser { public: - Ignore(Parser &p) : _p(p) { + Ignore(Parser &p) : _p(p) { _parser = h_ignore(p.parser()); } private: - Parser _p; + Parser _p; }; - template class SepBy : public Parser { public: - SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { + SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { _parser = h_sepBy(p.parser(), sep.parser()); } private: - Parser _p; - Parser _sep; + Parser _p, _sep; }; - template class SepBy1 : public Parser { public: - SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { + SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { _parser = h_sepBy1(p.parser(), sep.parser()); } private: - Parser _p; - Parser _sep; + Parser _p, _sep; }; class Epsilon : public Parser { @@ -462,29 +456,28 @@ namespace hammer { /* FIXME attr_bool */ - template class And : public Parser { public: - And(Parser &p) : _p(p) { + And(Parser &p) : _p(p) { _parser = h_and(p.parser()); } private: - Parser _p; + Parser _p; }; - template class Not : public Parser { public: Not(Parser &p) : _p(p) { _parser = h_not(p.parser()); } private: - Parser _p; + Parser _p; }; template class Indirect : public Parser { public: + typedef typename T::result_type result_type; Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); From 6a29277db492eccfd3c2e9c75507352002c82d51 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 3 Sep 2013 05:32:21 +0200 Subject: [PATCH 004/103] C++ tests set up. They probably don't compile; need to write iostream ops for prettyprinting and set up C++ build in scons. --- src/hammer.hxx | 13 + src/pprint.c | 20 +- src/t_cplusplus.cxx | 565 ++++++++++++++++++++++++++++++++++++++++++++ src/t_parser.c | 362 ++++++++++++++-------------- src/test_suite.c | 6 + src/test_suite.h | 70 +++++- 6 files changed, 851 insertions(+), 185 deletions(-) create mode 100644 src/t_cplusplus.cxx diff --git a/src/hammer.hxx b/src/hammer.hxx index 8f3200c..c0c39c5 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,6 +72,8 @@ namespace hammer { class Many1; template class Optional; class RepeatN; + class Ignore; + class Indirect; template class IntRange; template @@ -83,8 +85,12 @@ namespace hammer { Many many(); RepeatN many(size_t n); Optional optional(); + Ignore ignore(); RepeatN operator[](size_t n); HParser* parser() { return _parser; } + int compile(HParserBackend backend, const void* params) { + return h_compile(_parser, backend, params); + } protected: HParser* _parser; Parser() { } @@ -478,10 +484,17 @@ namespace hammer { class Indirect : public Parser { public: typedef typename T::result_type result_type; + /* Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); } + */ + Indirect() : _p(0) {} + bind(Parser &p) { + this->_parser = h_indirect(); + h_bind_indirect(this->_parser, p.parser()); + } private: Parser _p; }; diff --git a/src/pprint.c b/src/pprint.c index d8b22e2..969476c 100644 --- a/src/pprint.c +++ b/src/pprint.c @@ -22,6 +22,14 @@ #include "internal.h" #include +#ifdef __cplusplus +#include "hammer.hxx" +#include +#include + +using std::string; using std::stringstream; +#endif + typedef struct pp_state { int delta; int indent_amt; @@ -172,7 +180,13 @@ char* h_write_result_unamb__m(HAllocator* mm__, const HParsedToken* tok) { unamb_sub(tok, &buf); append_buf_c(&buf, 0); return buf.output; + +#ifdef __cplusplus +template +string& h_write_result_unamb(ParseResult tok) { + stringstream s; + s << boost::get(tok.result()[0]); + return s.str(); +} +#endif } - - - diff --git a/src/t_cplusplus.cxx b/src/t_cplusplus.cxx new file mode 100644 index 0000000..88e0c29 --- /dev/null +++ b/src/t_cplusplus.cxx @@ -0,0 +1,565 @@ +#include "hammer.hxx" +#include "test_suite.h" +#include +#include +#include +//#include +//#include + +static void test_token_cxx(gconstpointer backend) { + hammer::Token token_ = hammer::Token("95\xa2"); + + g_check_parse_ok_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95\xa2", 3, "<39.35.a2>"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95", 2); +} + +static void test_ch_cxx(gconstpointer backend) { + hammer::Ch ch_ = hammer::Ch(0xa2); + + g_check_parse_ok_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa2", "u0xa2"); // FIXME this will fail, c++ pretty-printing doesn't do this yet + g_check_parse_failed_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa3"); +} + +static void test_ch_range_cxx(gconstpointer backend) { + hammer::ChRange range_ = hammer::ChRange('a', 'c'); + + g_check_parse_ok_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "b"); + g_check_parse_failed_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); +} + +static void test_int64_cxx(gconstpointer backend) { + hammer::Int64 int64_ = hammer::Int64(); + + g_check_parse_ok_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00\x00", "s-0x200000000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00"); +} + +static void test_int32_cxx(gconstpointer backend) { + hammer::Int32 int32_ = hammer::Int32(); + + g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00\x00", "s-0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00"); + + g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "s0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00"); +} + +static void test_int16_cxx(gconstpointer backend) { + hammer::Int16 int16_ = hammer::Int16(); + + g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe\x00", "s-0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe"); + + g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "s0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); +} + +static void test_int8_cxx(gconstpointer backend) { + hammer::Int8 int8_ = hammer::Int8(); + + g_check_parse_ok_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x88", "s-0x78"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_uint64_cxx(gconstpointer backend) { + hammer::Uint64 uint64_ = hammer::Uint64(); + + g_check_parse_ok_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00\x00", "u0x200000000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00"); +} + +static void test_uint32_cxx(gconstpointer backend) { + hammer::Uint32 uint32_ = hammer::Uint32(); + + g_check_parse_ok_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "u0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00", 3); +} + +static void test_uint16_cxx(gconstpointer backend) { + hammer::Uint16 uint16_ = hammer::Uint16(); + + g_check_parse_ok_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "u0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); +} + +static void test_uint8_cxx(gconstpointer backend) { + hammer::Uint8 uint8_ = hammer::Uint8(); + + g_check_parse_ok_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x78", "u0x78"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_int_range_cxx(gconstpointer backend) { + hammer::IntRange int_range_ = hammer::Uint8().in_range(3, 10); + + g_check_parse_ok_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\x05", "u0x5"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\xb"); +} + +static void test_whitespace_cxx(gconstpointer backend) { + hammer::Whitespace whitespace_ = hammer::Whitespace(hammer::Ch('a')); + hammer::Whitespace whitespace_end = hammer::Whitespace(hammer::End()); + + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "\ta", "u0x61"); + g_check_parse_failed_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "_a"); + + g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend), "", "NULL"); + g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," ", "NULL"); + g_check_parse_failed_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," x"); +} + +static void test_left_cxx(gconstpointer backend) { + hammer::Left left_ = hammer::Left(hammer::Ch('a'), hammer::Ch(' ')); + + g_check_parse_ok_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a ", "u0x61"); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); +} + +static void test_right_cxx(gconstpointer backend) { + hammer::Right right_ = hammer::Right(hammer::Ch(' '), hammer::Ch('a')); + + g_check_parse_ok_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "ba"); +} + +static void test_middle_cxx(gconstpointer backend) { + hammer::Middle middle_ = hammer::Middle(hammer::Ch(' '), hammer::Ch('a'), hammer::Ch(' ')); + + g_check_parse_ok_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a ", "u0x61"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " b "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "ba "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " ab"); +} + +static void test_in_cxx(gconstpointer backend) { + std::string options = "abc"; + hammer::In in_ = hammer::In(options); + g_check_parse_ok_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); + g_check_parse_failed_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); + +} + +static void test_not_in_cxx(gconstpointer backend) { + std::string options = "abc"; + hammer::NotIn not_in_ = hammer::NotIn(options); + g_check_parse_ok_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "u0x64"); + g_check_parse_failed_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + +} + +static void test_end_cxx(gconstpointer backend) { + hammer::End end_ = hammer::Sequence(hammer::Ch('a'), hammer::End()); + g_check_parse_ok_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_failed_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "aa"); +} + +static void test_nothing_cxx(gconstpointer backend) { + hammer::Nothing nothing_ = hammer::Nothing(); + g_check_parse_failed_cxx(nothing_, (HParserBackend)GPOINTER_TO_INT(backend),"a"); +} + +static void test_sequence_cxx(gconstpointer backend) { + hammer::Sequence sequence_1 = hammer::Sequence(hammer::Ch('a'), hammer::Ch('b')); + hammer::Sequence sequence_2 = hammer::Sequence(hammer::Ch('a'), hammer::Whitespace(hammer::Ch('b'))); + + g_check_parse_ok_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "b"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); +} + +static void test_choice_cxx(gconstpointer backend) { + hammer::Choice choice_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')); + + g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); + g_check_parse_failed_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "c"); +} + +static void test_butnot_cxx(gconstpointer backend) { + hammer::ButNot butnot_1 = hammer::ButNot(hammer::Ch('a'), hammer::Token("ab")); + hammer::ButNot butnot_2 = hammer::ButNot(hammer::ChRange('0', '9'), hammer::Ch('6')); + + g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_failed_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); + g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "u0x61"); + g_check_parse_failed_cxx(butnot_2, (HParserBackend)GPOINTER_TO_INT(backend), "6"); +} + +static void test_difference_cxx(gconstpointer backend) { + hammer::Difference difference_ = hammer::Difference(hammer::Token("ab"), hammer::Ch('a')); + + g_check_parse_ok_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "<61.62>"); + g_check_parse_failed_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); +} + +static void test_xor_cxx(gconstpointer backend) { + hammer::Xor xor_ = hammer::Xor(hammer::ChRange('0', '6'), hammer::ChRange('5', '9')); + + g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "0", "u0x30"); + g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "9", "u0x39"); + g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "5"); + g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); +} + +static void test_many_cxx(gconstpointer backend) { + hammer::Many many_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many(); + + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); +} + +static void test_many1_cxx(gconstpointer backend) { + hammer::Many1 many1_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many1(); + + g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); + g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "daabbabadef"); +} + +static void test_repeat_n_cxx(gconstpointer backend) { + hammer::RepeatN repeat_n_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b'))[2]; + + g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "adef"); + g_check_parse_ok_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "abdef", "(u0x61 u0x62)"); + g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "dabdef"); +} + +static void test_optional_cxx(gconstpointer backend) { + hammer::Optional optional_ = hammer::Sequence(hammer::Ch('a'), + hammer::Choice(hammer::Ch('b'), hammer::Ch('c')).optional(), + hammer::ch('d')); + + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "abd", "(u0x61 u0x62 u0x64)"); + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "acd", "(u0x61 u0x63 u0x64)"); + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ad", "(u0x61 null u0x64)"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "aed"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); +} + +static void test_ignore_cxx(gconstpointer backend) { + hammer::Ignore ignore_ = hammer::Sequence(hammer::Ch('a'), + hammer::ch('b').ignore(), + hammer::ch('c')); + + g_check_parse_ok_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "abc", "(u0x61 u0x63)"); + g_check_parse_failed_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); +} + +static void test_sepBy_cxx(gconstpointer backend) { + hammer::SepBy sepBy_ = hammer::SepBy(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); + + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); +} + +static void test_sepBy1_cxx(gconstpointer backend) { + hammer::SepBy1 sepBy1_ = hammer::SepBy1(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); + + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); + g_check_parse_failed_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_epsilon_cxx(gconstpointer backend) { + hammer::Epsilon epsilon_p_1 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon(), hammer::Ch('b')); + hammer::Epsilon epsilon_p_2 = hammer::Sequence(hammer::Epsilon(), hammer::Ch('a')); + hammer::Epsilon epsilon_p_3 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon()); + + g_check_parse_ok_cxx(epsilon_p_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(epsilon_p_2, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); +} + +static void test_and_cxx(gconstpointer backend) { + hammer::And and_1 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('0')); + hammer::And and_2 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('1')); + hammer::And and_3 = hammer::Sequence(hammer::Ch('1'), hammer::And(hammer::Ch('2'))); + + g_check_parse_ok_cxx(and_1, (HParserBackend)GPOINTER_TO_INT(backend), "0", "(u0x30)"); + g_check_parse_failed_cxx(and_2, (HParserBackend)GPOINTER_TO_INT(backend), "0"); + g_check_parse_ok_cxx(and_3, (HParserBackend)GPOINTER_TO_INT(backend), "12", "(u0x31)"); +} + +static void test_not_cxx(gconstpointer backend) { + hammer::Not not_1 = hammer::Sequence(hammer::Ch('a'), hammer::Choice(hammer::Ch('+'), hammer::Token("++")), hammer::Ch('b')); + hammer::Not not_2 = hammer::Sequence(hammer::Ch('a'), + hammer::Choice(hammer::Sequence(hammer::Ch('+'), + hammer::Not(hammer::Ch('+'))), + hammer::Token("++")), + hammer::Ch('b')); + + g_check_parse_ok_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 u0x2b u0x62)"); + g_check_parse_failed_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a++b"); + g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 (u0x2b) u0x62)"); + g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", "(u0x61 <2b.2b> u0x62)"); +} + +static void test_leftrec(gconstpointer backend) { + hammer::Ch a_ = hammer::Ch('a'); + + hammer::Indirect lr_ = hammer::Indirect(); + + lr_.bind(hammer::Choice(hammer::Sequence(lr_, a_), a_)); + + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 u0x61)"); + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "((u0x61 u0x61) u0x61)"); +} + +static void test_rightrec(gconstpointer backend) { + hammer::Ch a_ = hammer::Ch('a'); + + hammer::Indirect rr_ = hammer::Indirect(); + rr_.bind(hammer::Choice(hammer::Sequence(a_, rr_), hammer::Epsilon())); + + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 (u0x61))"); + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "(u0x61 (u0x61 (u0x61)))"); +} + +// FIXME won't be ready until action is ready +/* +static void test_ambiguous(gconstpointer backend) { + hammer::Ch d_ = hammer::Ch('d'); + hammer::Ch p_ = hammer::Ch('+'); + hammer::Indirect E_ = hammer::Indirect(); + E_.bind(hammer::Choice(hammer::Sequence(E_, p_, E_), d_)); + hammer::Action expr_ = hammer::Action(E_, act_flatten); + + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "(u0x64)"); + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", "(u0x64 u0x2b u0x64)"); + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", "(u0x64 u0x2b u0x64 u0x2b u0x64)"); + g_check_parse_failed_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+"); +} +*/ + +void register_cxx_tests(void) { + g_test_add_data_func("/core/c++/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range_cxx); + g_test_add_data_func("/core/c++/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left_cxx); + g_test_add_data_func("/core/c++/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right_cxx); + g_test_add_data_func("/core/c++/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle_cxx); + g_test_add_data_func("/core/c++/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in_cxx); + g_test_add_data_func("/core/c++/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/packrat/end", GINT_TO_POINTER(PB_PACKRAT), test_end_cxx); + g_test_add_data_func("/core/c++/parser/packrat/nothing", GINT_TO_POINTER(PB_PACKRAT), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot_cxx); + g_test_add_data_func("/core/c++/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference_cxx); + g_test_add_data_func("/core/c++/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor_cxx); + g_test_add_data_func("/core/c++/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many_cxx); + g_test_add_data_func("/core/c++/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n_cxx); + g_test_add_data_func("/core/c++/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/packrat/epsilon", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_cxx); + g_test_add_data_func("/core/c++/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and_cxx); + g_test_add_data_func("/core/c++/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not_cxx); + // g_test_add_data_func("/core/c++/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec_cxx); + + g_test_add_data_Func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); + g_test_add_data_func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); + g_test_add_data_func("/core/c++/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left_cxx); + g_test_add_data_func("/core/c++/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right_cxx); + g_test_add_data_func("/core/c++/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle_cxx); + //g_test_add_data_func("/core/c++/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action_cxx); + g_test_add_data_func("/core/c++/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in_cxx); + g_test_add_data_func("/core/c++/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/llk/end", GINT_TO_POINTER(PB_LLk), test_end_cxx); + g_test_add_data_func("/core/c++/parser/llk/nothing", GINT_TO_POINTER(PB_LLk), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many_cxx); + g_test_add_data_func("/core/c++/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/llk/epsilon", GINT_TO_POINTER(PB_LLk), test_epsilon_cxx); + //g_test_add_data_func("/core/c++/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore_cxx); + //g_test_add_data_func("/core/c++/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec_cxx); + + g_test_add_data_func("/core/c++/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle_cxx); + // g_test_add_data_func("/core/c++/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/regex/end", GINT_TO_POINTER(PB_REGULAR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/regex/nothing", GINT_TO_POINTER(PB_REGULAR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n_cxx); + g_test_add_data_func("/core/c++/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/regex/epsilon", GINT_TO_POINTER(PB_REGULAR), test_epsilon_cxx); + // g_test_add_data_func("/core/c++/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore_cxx); + + g_test_add_data_func("/core/c++/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle_cxx); + // g_test_add_data_func("/core/c++/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/lalr/end", GINT_TO_POINTER(PB_LALR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/lalr/nothing", GINT_TO_POINTER(PB_LALR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/lalr/epsilon", GINT_TO_POINTER(PB_LALR), test_epsilon_cxx); + // g_test_add_data_func("/core/c++/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec_cxx); + + g_test_add_data_func("/core/c++/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle_cxx); + g_test_add_data_func("/core/c++/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/glr/end", GINT_TO_POINTER(PB_GLR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/glr/nothing", GINT_TO_POINTER(PB_GLR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/glr/epsilon", GINT_TO_POINTER(PB_GLR), test_epsilon_cxx); + g_test_add_data_func("/core/c++/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec_cxx); + g_test_add_data_func("/core/c++/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous_cxx); + +} + +/* +int main(int argc, char **argv) { + hammer::Ch p = hammer::Ch('a'); + std::string test = "a"; + hammer::UintResult r = p.parse(test); + std::cerr << r.result() << std::endl; + hammer::SequenceResult s = p.many().parse(test); + hammer::UintResult u = boost::get(s.result()[0]); + std::cerr << u.result() << std::endl; + return 0; +} +*/ diff --git a/src/t_parser.c b/src/t_parser.c index 59adf36..c49e443 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -442,199 +442,199 @@ static void test_ambiguous(gconstpointer backend) { } void register_parser_tests(void) { - g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); - g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); - g_test_add_data_func("/core/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); - g_test_add_data_func("/core/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); - g_test_add_data_func("/core/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); - g_test_add_data_func("/core/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); - g_test_add_data_func("/core/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); - g_test_add_data_func("/core/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); - g_test_add_data_func("/core/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); - g_test_add_data_func("/core/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); - g_test_add_data_func("/core/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); - g_test_add_data_func("/core/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); + g_test_add_data_func("/core/c/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); + g_test_add_data_func("/core/c/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); + g_test_add_data_func("/core/c/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); + g_test_add_data_func("/core/c/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); + g_test_add_data_func("/core/c/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); + g_test_add_data_func("/core/c/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); + g_test_add_data_func("/core/c/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); + g_test_add_data_func("/core/c/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); + g_test_add_data_func("/core/c/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); + g_test_add_data_func("/core/c/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); + g_test_add_data_func("/core/c/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); + g_test_add_data_func("/core/c/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); #if 0 - g_test_add_data_func("/core/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); - g_test_add_data_func("/core/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); + g_test_add_data_func("/core/c/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); + g_test_add_data_func("/core/c/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); #endif - g_test_add_data_func("/core/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); - g_test_add_data_func("/core/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); - g_test_add_data_func("/core/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); - g_test_add_data_func("/core/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); - g_test_add_data_func("/core/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); - g_test_add_data_func("/core/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); - g_test_add_data_func("/core/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); - g_test_add_data_func("/core/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); - g_test_add_data_func("/core/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); - g_test_add_data_func("/core/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); - g_test_add_data_func("/core/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); - g_test_add_data_func("/core/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); - g_test_add_data_func("/core/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); - g_test_add_data_func("/core/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); - g_test_add_data_func("/core/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); - g_test_add_data_func("/core/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); - g_test_add_data_func("/core/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); - g_test_add_data_func("/core/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); - g_test_add_data_func("/core/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); - g_test_add_data_func("/core/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); - g_test_add_data_func("/core/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); - g_test_add_data_func("/core/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); - g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); - g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); - g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); - g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); + g_test_add_data_func("/core/c/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); + g_test_add_data_func("/core/c/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); + g_test_add_data_func("/core/c/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); + g_test_add_data_func("/core/c/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); + g_test_add_data_func("/core/c/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); + g_test_add_data_func("/core/c/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); + g_test_add_data_func("/core/c/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); + g_test_add_data_func("/core/c/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); + g_test_add_data_func("/core/c/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); + g_test_add_data_func("/core/c/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); + g_test_add_data_func("/core/c/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); + g_test_add_data_func("/core/c/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); + g_test_add_data_func("/core/c/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); + g_test_add_data_func("/core/c/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); + g_test_add_data_func("/core/c/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); + g_test_add_data_func("/core/c/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); + g_test_add_data_func("/core/c/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); + g_test_add_data_func("/core/c/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); + g_test_add_data_func("/core/c/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); + g_test_add_data_func("/core/c/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); + g_test_add_data_func("/core/c/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); + g_test_add_data_func("/core/c/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); + g_test_add_data_func("/core/c/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); + g_test_add_data_func("/core/c/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); + g_test_add_data_func("/core/c/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); + // g_test_add_data_func("/core/c/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + g_test_add_data_func("/core/c/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); - g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); - g_test_add_data_func("/core/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); - g_test_add_data_func("/core/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); - g_test_add_data_func("/core/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); - g_test_add_data_func("/core/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); - g_test_add_data_func("/core/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); - g_test_add_data_func("/core/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); - g_test_add_data_func("/core/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); - g_test_add_data_func("/core/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); - g_test_add_data_func("/core/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); - g_test_add_data_func("/core/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); - g_test_add_data_func("/core/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); + g_test_add_data_func("/core/c/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); + g_test_add_data_func("/core/c/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); + g_test_add_data_func("/core/c/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); + g_test_add_data_func("/core/c/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); + g_test_add_data_func("/core/c/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); + g_test_add_data_func("/core/c/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); + g_test_add_data_func("/core/c/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); + g_test_add_data_func("/core/c/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); + g_test_add_data_func("/core/c/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); + g_test_add_data_func("/core/c/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); + g_test_add_data_func("/core/c/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); + g_test_add_data_func("/core/c/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); #if 0 - g_test_add_data_func("/core/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); - g_test_add_data_func("/core/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); + g_test_add_data_func("/core/c/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); + g_test_add_data_func("/core/c/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); #endif - g_test_add_data_func("/core/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); - g_test_add_data_func("/core/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); - g_test_add_data_func("/core/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); - g_test_add_data_func("/core/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); - g_test_add_data_func("/core/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); - g_test_add_data_func("/core/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); - g_test_add_data_func("/core/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); - g_test_add_data_func("/core/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); - g_test_add_data_func("/core/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); - g_test_add_data_func("/core/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); - g_test_add_data_func("/core/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); - g_test_add_data_func("/core/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); - g_test_add_data_func("/core/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); - g_test_add_data_func("/core/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); - g_test_add_data_func("/core/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); - g_test_add_data_func("/core/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); - g_test_add_data_func("/core/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); - g_test_add_data_func("/core/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); - g_test_add_data_func("/core/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); - //g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); - g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); + g_test_add_data_func("/core/c/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); + g_test_add_data_func("/core/c/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); + g_test_add_data_func("/core/c/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); + g_test_add_data_func("/core/c/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); + g_test_add_data_func("/core/c/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); + g_test_add_data_func("/core/c/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); + g_test_add_data_func("/core/c/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); + g_test_add_data_func("/core/c/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); + g_test_add_data_func("/core/c/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); + g_test_add_data_func("/core/c/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); + g_test_add_data_func("/core/c/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); + g_test_add_data_func("/core/c/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); + g_test_add_data_func("/core/c/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); + g_test_add_data_func("/core/c/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); + g_test_add_data_func("/core/c/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); + g_test_add_data_func("/core/c/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); + g_test_add_data_func("/core/c/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); + g_test_add_data_func("/core/c/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); + g_test_add_data_func("/core/c/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); + //g_test_add_data_func("/core/c/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); + g_test_add_data_func("/core/c/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); - g_test_add_data_func("/core/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); - g_test_add_data_func("/core/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); - g_test_add_data_func("/core/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); - g_test_add_data_func("/core/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); - g_test_add_data_func("/core/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); - g_test_add_data_func("/core/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); - g_test_add_data_func("/core/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); - g_test_add_data_func("/core/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); - g_test_add_data_func("/core/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); - g_test_add_data_func("/core/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); - g_test_add_data_func("/core/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); - g_test_add_data_func("/core/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); + g_test_add_data_func("/core/c/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); + g_test_add_data_func("/core/c/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); + g_test_add_data_func("/core/c/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); + g_test_add_data_func("/core/c/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); + g_test_add_data_func("/core/c/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); + g_test_add_data_func("/core/c/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); + g_test_add_data_func("/core/c/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); + g_test_add_data_func("/core/c/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); + g_test_add_data_func("/core/c/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); + g_test_add_data_func("/core/c/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); + g_test_add_data_func("/core/c/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); + g_test_add_data_func("/core/c/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); - g_test_add_data_func("/core/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); + g_test_add_data_func("/core/c/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); + g_test_add_data_func("/core/c/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); #endif - g_test_add_data_func("/core/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); - g_test_add_data_func("/core/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); - g_test_add_data_func("/core/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); - g_test_add_data_func("/core/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); - g_test_add_data_func("/core/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); - g_test_add_data_func("/core/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); - g_test_add_data_func("/core/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); - g_test_add_data_func("/core/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); - g_test_add_data_func("/core/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); - g_test_add_data_func("/core/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); - g_test_add_data_func("/core/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); - g_test_add_data_func("/core/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); - g_test_add_data_func("/core/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); - g_test_add_data_func("/core/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); - g_test_add_data_func("/core/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); - g_test_add_data_func("/core/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); - g_test_add_data_func("/core/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); - g_test_add_data_func("/core/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); - g_test_add_data_func("/core/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); - g_test_add_data_func("/core/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); + g_test_add_data_func("/core/c/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); + g_test_add_data_func("/core/c/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); + g_test_add_data_func("/core/c/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); + g_test_add_data_func("/core/c/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); + g_test_add_data_func("/core/c/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); + g_test_add_data_func("/core/c/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); + g_test_add_data_func("/core/c/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); + g_test_add_data_func("/core/c/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); + g_test_add_data_func("/core/c/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); + g_test_add_data_func("/core/c/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); + g_test_add_data_func("/core/c/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); + g_test_add_data_func("/core/c/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); + g_test_add_data_func("/core/c/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); + g_test_add_data_func("/core/c/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); + g_test_add_data_func("/core/c/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); + g_test_add_data_func("/core/c/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); + g_test_add_data_func("/core/c/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); + g_test_add_data_func("/core/c/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); + g_test_add_data_func("/core/c/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); - g_test_add_data_func("/core/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); - g_test_add_data_func("/core/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); - g_test_add_data_func("/core/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); - g_test_add_data_func("/core/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); - g_test_add_data_func("/core/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); - g_test_add_data_func("/core/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); - g_test_add_data_func("/core/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); - g_test_add_data_func("/core/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); - g_test_add_data_func("/core/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); - g_test_add_data_func("/core/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); - g_test_add_data_func("/core/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); - g_test_add_data_func("/core/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); + g_test_add_data_func("/core/c/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); + g_test_add_data_func("/core/c/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); + g_test_add_data_func("/core/c/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); + g_test_add_data_func("/core/c/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); + g_test_add_data_func("/core/c/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); + g_test_add_data_func("/core/c/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); + g_test_add_data_func("/core/c/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); + g_test_add_data_func("/core/c/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); + g_test_add_data_func("/core/c/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); + g_test_add_data_func("/core/c/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); + g_test_add_data_func("/core/c/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); + g_test_add_data_func("/core/c/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); - g_test_add_data_func("/core/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); + g_test_add_data_func("/core/c/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); + g_test_add_data_func("/core/c/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); #endif - g_test_add_data_func("/core/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); - g_test_add_data_func("/core/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); - g_test_add_data_func("/core/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); - g_test_add_data_func("/core/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); - g_test_add_data_func("/core/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); - g_test_add_data_func("/core/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); - g_test_add_data_func("/core/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); - g_test_add_data_func("/core/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); - g_test_add_data_func("/core/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); - g_test_add_data_func("/core/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); - g_test_add_data_func("/core/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); - g_test_add_data_func("/core/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); - g_test_add_data_func("/core/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); - g_test_add_data_func("/core/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); - g_test_add_data_func("/core/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); - g_test_add_data_func("/core/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); - g_test_add_data_func("/core/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); - g_test_add_data_func("/core/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); - g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); - g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); - g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); + g_test_add_data_func("/core/c/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); + g_test_add_data_func("/core/c/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); + g_test_add_data_func("/core/c/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); + g_test_add_data_func("/core/c/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); + g_test_add_data_func("/core/c/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); + g_test_add_data_func("/core/c/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); + g_test_add_data_func("/core/c/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); + g_test_add_data_func("/core/c/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); + g_test_add_data_func("/core/c/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); + g_test_add_data_func("/core/c/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); + g_test_add_data_func("/core/c/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); + g_test_add_data_func("/core/c/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); + g_test_add_data_func("/core/c/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); + g_test_add_data_func("/core/c/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); + g_test_add_data_func("/core/c/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); + g_test_add_data_func("/core/c/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); + g_test_add_data_func("/core/c/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); + g_test_add_data_func("/core/c/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); + g_test_add_data_func("/core/c/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); + g_test_add_data_func("/core/c/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); - g_test_add_data_func("/core/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); - g_test_add_data_func("/core/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); - g_test_add_data_func("/core/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); - g_test_add_data_func("/core/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); - g_test_add_data_func("/core/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); - g_test_add_data_func("/core/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); - g_test_add_data_func("/core/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); - g_test_add_data_func("/core/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); - g_test_add_data_func("/core/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); - g_test_add_data_func("/core/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); - g_test_add_data_func("/core/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); - g_test_add_data_func("/core/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); + g_test_add_data_func("/core/c/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); + g_test_add_data_func("/core/c/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); + g_test_add_data_func("/core/c/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); + g_test_add_data_func("/core/c/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); + g_test_add_data_func("/core/c/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); + g_test_add_data_func("/core/c/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); + g_test_add_data_func("/core/c/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); + g_test_add_data_func("/core/c/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); + g_test_add_data_func("/core/c/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); + g_test_add_data_func("/core/c/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); + g_test_add_data_func("/core/c/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); + g_test_add_data_func("/core/c/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); - g_test_add_data_func("/core/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); + g_test_add_data_func("/core/c/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); + g_test_add_data_func("/core/c/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); #endif - g_test_add_data_func("/core/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); - g_test_add_data_func("/core/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); - g_test_add_data_func("/core/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); - g_test_add_data_func("/core/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); - g_test_add_data_func("/core/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); - g_test_add_data_func("/core/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); - g_test_add_data_func("/core/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); - g_test_add_data_func("/core/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); - g_test_add_data_func("/core/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); - g_test_add_data_func("/core/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); - g_test_add_data_func("/core/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); - g_test_add_data_func("/core/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); - g_test_add_data_func("/core/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); - g_test_add_data_func("/core/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); - g_test_add_data_func("/core/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); - g_test_add_data_func("/core/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); - g_test_add_data_func("/core/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); - g_test_add_data_func("/core/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); - g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); - g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); - g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); - g_test_add_data_func("/core/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); + g_test_add_data_func("/core/c/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); + g_test_add_data_func("/core/c/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); + g_test_add_data_func("/core/c/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); + g_test_add_data_func("/core/c/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); + g_test_add_data_func("/core/c/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); + g_test_add_data_func("/core/c/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); + g_test_add_data_func("/core/c/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); + g_test_add_data_func("/core/c/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); + g_test_add_data_func("/core/c/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); + g_test_add_data_func("/core/c/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); + g_test_add_data_func("/core/c/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); + g_test_add_data_func("/core/c/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); + g_test_add_data_func("/core/c/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); + g_test_add_data_func("/core/c/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); + g_test_add_data_func("/core/c/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); + g_test_add_data_func("/core/c/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); + g_test_add_data_func("/core/c/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); + g_test_add_data_func("/core/c/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); + g_test_add_data_func("/core/c/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); + g_test_add_data_func("/core/c/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); + g_test_add_data_func("/core/c/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); } diff --git a/src/test_suite.c b/src/test_suite.c index e065f13..9191f93 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -25,6 +25,9 @@ extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); extern void register_benchmark_tests(); +#ifdef __cplusplus +extern void register_cxx_tests(); +#endif int main(int argc, char** argv) { g_test_init(&argc, &argv, NULL); @@ -36,6 +39,9 @@ int main(int argc, char** argv) { register_grammar_tests(); register_misc_tests(); register_benchmark_tests(); + #ifdef __cplusplus + register_cxx_tests(); + #endif g_test_run(); } diff --git a/src/test_suite.h b/src/test_suite.h index fc008e7..76c6d78 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -19,6 +19,16 @@ #define HAMMER_TEST_SUITE__H #include +#ifdef __cplusplus +extern "C" { +#include +#include +#endif + +#ifndef __cplusplus +typedef int bool; +#endif + // Equivalent to g_assert_*, but not using g_assert... #define g_check_inttype(fmt, typ, n1, op, n2) do { \ typ _n1 = (n1); \ @@ -52,6 +62,19 @@ } \ } while(0) +#ifdef __cplusplus +#define g_check_string_cxx(n1, op, n2) do { \ + std::string _n1 = (n1); \ + std::string _n2 = (n2); \ + if (!(_n1.compare(_n2) op 0)) { \ + g_test_message("Check failed: (%s) (%s %s %s)", \ + #n1 " " #op " " #n2, \ + _n1.c_str(), #op, _n2.c_str()); \ + g_test_fail(); \ + } \ + } while(0) +#endif + #define g_check_regular(lang) do { \ if (!lang->isValidRegular(lang->env)) { \ g_test_message("Language is not regular"); \ @@ -95,6 +118,21 @@ g_test_fail(); \ } \ } while(0) + +#ifdef __cplusplus +#define g_check_parse_failed_cxx(parser, backend, input) do { \ + int skip = parser.compile((HParserBackend)backend, NULL); \ + if(skip != 0) { \ + g_test_message("Backend not applicable, skipping test"); \ + break; \ + } \ + parser::result_type result = parser.parse(input); \ + if (NULL != result) { \ + g_test_message("Check failed: shouldn't have succeeded, but did"); \ + g_test_fail(); \ + } \ + } while(0) +#endif #define g_check_parse_ok(parser, backend, input, inp_len, result) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ @@ -120,6 +158,34 @@ } \ } while(0) +#ifdef __cplusplus +#define g_check_parse_ok_cxx(parser, backend, input, result) do { \ + int skip = parser.compile((HParserBackend)backend, NULL); \ + if(skip) { \ + g_test_message("Backend not applicable, skipping test"); \ + break; \ + } \ + parser::result_type res = parser.parse((std::string)input); \ + if (!res) { \ + g_test_message("Parse failed on line %d", __LINE__); \ + g_test_fail(); \ + } else { \ + std::string& cres = h_write_result_unamb(res); \ + g_check_string_cxx(cres, ==, (std::string)result); \ + /* + system_allocator.free(&system_allocator, cres); \ + HArenaStats stats; \ + h_allocator_stats(res->arena, &stats); \ + g_test_message("Parse used %zd bytes, wasted %zd bytes. " \ + "Inefficiency: %5f%%", \ + stats.used, stats.wasted, \ + stats.wasted * 100. / (stats.used+stats.wasted)); \ + h_delete_arena(res->arena); \ + */ // FIXME might've made a memory leak there + } \ + } while(0) +#endif + #define g_check_hashtable_present(table, key) do { \ if(!h_hashtable_present(table, key)) { \ g_test_message("Check failed: key should have been in table, but wasn't"); \ @@ -197,6 +263,8 @@ #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) - +#ifdef __cplusplus +} +#endif #endif // #ifndef HAMMER_TEST_SUITE__H From 0db4925469927323e0013199aec67069ff8b2923 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 3 Sep 2013 05:33:30 +0200 Subject: [PATCH 005/103] Revert "C++ tests set up. They probably don't compile; need to write iostream ops for prettyprinting and set up C++ build in scons." Should have branched it. This reverts commit 6a29277db492eccfd3c2e9c75507352002c82d51. --- src/hammer.hxx | 13 - src/pprint.c | 20 +- src/t_cplusplus.cxx | 565 -------------------------------------------- src/t_parser.c | 362 ++++++++++++++-------------- src/test_suite.c | 6 - src/test_suite.h | 70 +----- 6 files changed, 185 insertions(+), 851 deletions(-) delete mode 100644 src/t_cplusplus.cxx diff --git a/src/hammer.hxx b/src/hammer.hxx index c0c39c5..8f3200c 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,8 +72,6 @@ namespace hammer { class Many1; template class Optional; class RepeatN; - class Ignore; - class Indirect; template class IntRange; template @@ -85,12 +83,8 @@ namespace hammer { Many many(); RepeatN many(size_t n); Optional optional(); - Ignore ignore(); RepeatN operator[](size_t n); HParser* parser() { return _parser; } - int compile(HParserBackend backend, const void* params) { - return h_compile(_parser, backend, params); - } protected: HParser* _parser; Parser() { } @@ -484,17 +478,10 @@ namespace hammer { class Indirect : public Parser { public: typedef typename T::result_type result_type; - /* Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); } - */ - Indirect() : _p(0) {} - bind(Parser &p) { - this->_parser = h_indirect(); - h_bind_indirect(this->_parser, p.parser()); - } private: Parser _p; }; diff --git a/src/pprint.c b/src/pprint.c index 969476c..d8b22e2 100644 --- a/src/pprint.c +++ b/src/pprint.c @@ -22,14 +22,6 @@ #include "internal.h" #include -#ifdef __cplusplus -#include "hammer.hxx" -#include -#include - -using std::string; using std::stringstream; -#endif - typedef struct pp_state { int delta; int indent_amt; @@ -180,13 +172,7 @@ char* h_write_result_unamb__m(HAllocator* mm__, const HParsedToken* tok) { unamb_sub(tok, &buf); append_buf_c(&buf, 0); return buf.output; +} + + -#ifdef __cplusplus -template -string& h_write_result_unamb(ParseResult tok) { - stringstream s; - s << boost::get(tok.result()[0]); - return s.str(); -} -#endif -} diff --git a/src/t_cplusplus.cxx b/src/t_cplusplus.cxx deleted file mode 100644 index 88e0c29..0000000 --- a/src/t_cplusplus.cxx +++ /dev/null @@ -1,565 +0,0 @@ -#include "hammer.hxx" -#include "test_suite.h" -#include -#include -#include -//#include -//#include - -static void test_token_cxx(gconstpointer backend) { - hammer::Token token_ = hammer::Token("95\xa2"); - - g_check_parse_ok_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95\xa2", 3, "<39.35.a2>"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95", 2); -} - -static void test_ch_cxx(gconstpointer backend) { - hammer::Ch ch_ = hammer::Ch(0xa2); - - g_check_parse_ok_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa2", "u0xa2"); // FIXME this will fail, c++ pretty-printing doesn't do this yet - g_check_parse_failed_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa3"); -} - -static void test_ch_range_cxx(gconstpointer backend) { - hammer::ChRange range_ = hammer::ChRange('a', 'c'); - - g_check_parse_ok_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "b"); - g_check_parse_failed_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); -} - -static void test_int64_cxx(gconstpointer backend) { - hammer::Int64 int64_ = hammer::Int64(); - - g_check_parse_ok_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00\x00", "s-0x200000000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00"); -} - -static void test_int32_cxx(gconstpointer backend) { - hammer::Int32 int32_ = hammer::Int32(); - - g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00\x00", "s-0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00"); - - g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "s0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00"); -} - -static void test_int16_cxx(gconstpointer backend) { - hammer::Int16 int16_ = hammer::Int16(); - - g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe\x00", "s-0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe"); - - g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "s0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); -} - -static void test_int8_cxx(gconstpointer backend) { - hammer::Int8 int8_ = hammer::Int8(); - - g_check_parse_ok_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x88", "s-0x78"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_uint64_cxx(gconstpointer backend) { - hammer::Uint64 uint64_ = hammer::Uint64(); - - g_check_parse_ok_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00\x00", "u0x200000000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00"); -} - -static void test_uint32_cxx(gconstpointer backend) { - hammer::Uint32 uint32_ = hammer::Uint32(); - - g_check_parse_ok_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "u0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00", 3); -} - -static void test_uint16_cxx(gconstpointer backend) { - hammer::Uint16 uint16_ = hammer::Uint16(); - - g_check_parse_ok_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "u0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); -} - -static void test_uint8_cxx(gconstpointer backend) { - hammer::Uint8 uint8_ = hammer::Uint8(); - - g_check_parse_ok_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x78", "u0x78"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_int_range_cxx(gconstpointer backend) { - hammer::IntRange int_range_ = hammer::Uint8().in_range(3, 10); - - g_check_parse_ok_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\x05", "u0x5"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\xb"); -} - -static void test_whitespace_cxx(gconstpointer backend) { - hammer::Whitespace whitespace_ = hammer::Whitespace(hammer::Ch('a')); - hammer::Whitespace whitespace_end = hammer::Whitespace(hammer::End()); - - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "\ta", "u0x61"); - g_check_parse_failed_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "_a"); - - g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend), "", "NULL"); - g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," ", "NULL"); - g_check_parse_failed_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," x"); -} - -static void test_left_cxx(gconstpointer backend) { - hammer::Left left_ = hammer::Left(hammer::Ch('a'), hammer::Ch(' ')); - - g_check_parse_ok_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a ", "u0x61"); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); -} - -static void test_right_cxx(gconstpointer backend) { - hammer::Right right_ = hammer::Right(hammer::Ch(' '), hammer::Ch('a')); - - g_check_parse_ok_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "ba"); -} - -static void test_middle_cxx(gconstpointer backend) { - hammer::Middle middle_ = hammer::Middle(hammer::Ch(' '), hammer::Ch('a'), hammer::Ch(' ')); - - g_check_parse_ok_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a ", "u0x61"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " b "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "ba "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " ab"); -} - -static void test_in_cxx(gconstpointer backend) { - std::string options = "abc"; - hammer::In in_ = hammer::In(options); - g_check_parse_ok_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); - g_check_parse_failed_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); - -} - -static void test_not_in_cxx(gconstpointer backend) { - std::string options = "abc"; - hammer::NotIn not_in_ = hammer::NotIn(options); - g_check_parse_ok_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "u0x64"); - g_check_parse_failed_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - -} - -static void test_end_cxx(gconstpointer backend) { - hammer::End end_ = hammer::Sequence(hammer::Ch('a'), hammer::End()); - g_check_parse_ok_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_failed_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "aa"); -} - -static void test_nothing_cxx(gconstpointer backend) { - hammer::Nothing nothing_ = hammer::Nothing(); - g_check_parse_failed_cxx(nothing_, (HParserBackend)GPOINTER_TO_INT(backend),"a"); -} - -static void test_sequence_cxx(gconstpointer backend) { - hammer::Sequence sequence_1 = hammer::Sequence(hammer::Ch('a'), hammer::Ch('b')); - hammer::Sequence sequence_2 = hammer::Sequence(hammer::Ch('a'), hammer::Whitespace(hammer::Ch('b'))); - - g_check_parse_ok_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "b"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); -} - -static void test_choice_cxx(gconstpointer backend) { - hammer::Choice choice_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')); - - g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); - g_check_parse_failed_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "c"); -} - -static void test_butnot_cxx(gconstpointer backend) { - hammer::ButNot butnot_1 = hammer::ButNot(hammer::Ch('a'), hammer::Token("ab")); - hammer::ButNot butnot_2 = hammer::ButNot(hammer::ChRange('0', '9'), hammer::Ch('6')); - - g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_failed_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); - g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "u0x61"); - g_check_parse_failed_cxx(butnot_2, (HParserBackend)GPOINTER_TO_INT(backend), "6"); -} - -static void test_difference_cxx(gconstpointer backend) { - hammer::Difference difference_ = hammer::Difference(hammer::Token("ab"), hammer::Ch('a')); - - g_check_parse_ok_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "<61.62>"); - g_check_parse_failed_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); -} - -static void test_xor_cxx(gconstpointer backend) { - hammer::Xor xor_ = hammer::Xor(hammer::ChRange('0', '6'), hammer::ChRange('5', '9')); - - g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "0", "u0x30"); - g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "9", "u0x39"); - g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "5"); - g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); -} - -static void test_many_cxx(gconstpointer backend) { - hammer::Many many_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many(); - - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); -} - -static void test_many1_cxx(gconstpointer backend) { - hammer::Many1 many1_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many1(); - - g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); - g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "daabbabadef"); -} - -static void test_repeat_n_cxx(gconstpointer backend) { - hammer::RepeatN repeat_n_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b'))[2]; - - g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "adef"); - g_check_parse_ok_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "abdef", "(u0x61 u0x62)"); - g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "dabdef"); -} - -static void test_optional_cxx(gconstpointer backend) { - hammer::Optional optional_ = hammer::Sequence(hammer::Ch('a'), - hammer::Choice(hammer::Ch('b'), hammer::Ch('c')).optional(), - hammer::ch('d')); - - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "abd", "(u0x61 u0x62 u0x64)"); - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "acd", "(u0x61 u0x63 u0x64)"); - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ad", "(u0x61 null u0x64)"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "aed"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); -} - -static void test_ignore_cxx(gconstpointer backend) { - hammer::Ignore ignore_ = hammer::Sequence(hammer::Ch('a'), - hammer::ch('b').ignore(), - hammer::ch('c')); - - g_check_parse_ok_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "abc", "(u0x61 u0x63)"); - g_check_parse_failed_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); -} - -static void test_sepBy_cxx(gconstpointer backend) { - hammer::SepBy sepBy_ = hammer::SepBy(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); - - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); -} - -static void test_sepBy1_cxx(gconstpointer backend) { - hammer::SepBy1 sepBy1_ = hammer::SepBy1(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); - - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); - g_check_parse_failed_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_epsilon_cxx(gconstpointer backend) { - hammer::Epsilon epsilon_p_1 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon(), hammer::Ch('b')); - hammer::Epsilon epsilon_p_2 = hammer::Sequence(hammer::Epsilon(), hammer::Ch('a')); - hammer::Epsilon epsilon_p_3 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon()); - - g_check_parse_ok_cxx(epsilon_p_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(epsilon_p_2, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); -} - -static void test_and_cxx(gconstpointer backend) { - hammer::And and_1 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('0')); - hammer::And and_2 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('1')); - hammer::And and_3 = hammer::Sequence(hammer::Ch('1'), hammer::And(hammer::Ch('2'))); - - g_check_parse_ok_cxx(and_1, (HParserBackend)GPOINTER_TO_INT(backend), "0", "(u0x30)"); - g_check_parse_failed_cxx(and_2, (HParserBackend)GPOINTER_TO_INT(backend), "0"); - g_check_parse_ok_cxx(and_3, (HParserBackend)GPOINTER_TO_INT(backend), "12", "(u0x31)"); -} - -static void test_not_cxx(gconstpointer backend) { - hammer::Not not_1 = hammer::Sequence(hammer::Ch('a'), hammer::Choice(hammer::Ch('+'), hammer::Token("++")), hammer::Ch('b')); - hammer::Not not_2 = hammer::Sequence(hammer::Ch('a'), - hammer::Choice(hammer::Sequence(hammer::Ch('+'), - hammer::Not(hammer::Ch('+'))), - hammer::Token("++")), - hammer::Ch('b')); - - g_check_parse_ok_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 u0x2b u0x62)"); - g_check_parse_failed_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a++b"); - g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 (u0x2b) u0x62)"); - g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", "(u0x61 <2b.2b> u0x62)"); -} - -static void test_leftrec(gconstpointer backend) { - hammer::Ch a_ = hammer::Ch('a'); - - hammer::Indirect lr_ = hammer::Indirect(); - - lr_.bind(hammer::Choice(hammer::Sequence(lr_, a_), a_)); - - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 u0x61)"); - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "((u0x61 u0x61) u0x61)"); -} - -static void test_rightrec(gconstpointer backend) { - hammer::Ch a_ = hammer::Ch('a'); - - hammer::Indirect rr_ = hammer::Indirect(); - rr_.bind(hammer::Choice(hammer::Sequence(a_, rr_), hammer::Epsilon())); - - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 (u0x61))"); - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "(u0x61 (u0x61 (u0x61)))"); -} - -// FIXME won't be ready until action is ready -/* -static void test_ambiguous(gconstpointer backend) { - hammer::Ch d_ = hammer::Ch('d'); - hammer::Ch p_ = hammer::Ch('+'); - hammer::Indirect E_ = hammer::Indirect(); - E_.bind(hammer::Choice(hammer::Sequence(E_, p_, E_), d_)); - hammer::Action expr_ = hammer::Action(E_, act_flatten); - - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "(u0x64)"); - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", "(u0x64 u0x2b u0x64)"); - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", "(u0x64 u0x2b u0x64 u0x2b u0x64)"); - g_check_parse_failed_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+"); -} -*/ - -void register_cxx_tests(void) { - g_test_add_data_func("/core/c++/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range_cxx); - g_test_add_data_func("/core/c++/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left_cxx); - g_test_add_data_func("/core/c++/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right_cxx); - g_test_add_data_func("/core/c++/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle_cxx); - g_test_add_data_func("/core/c++/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in_cxx); - g_test_add_data_func("/core/c++/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/packrat/end", GINT_TO_POINTER(PB_PACKRAT), test_end_cxx); - g_test_add_data_func("/core/c++/parser/packrat/nothing", GINT_TO_POINTER(PB_PACKRAT), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot_cxx); - g_test_add_data_func("/core/c++/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference_cxx); - g_test_add_data_func("/core/c++/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor_cxx); - g_test_add_data_func("/core/c++/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many_cxx); - g_test_add_data_func("/core/c++/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n_cxx); - g_test_add_data_func("/core/c++/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/packrat/epsilon", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_cxx); - g_test_add_data_func("/core/c++/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and_cxx); - g_test_add_data_func("/core/c++/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not_cxx); - // g_test_add_data_func("/core/c++/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec_cxx); - - g_test_add_data_Func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); - g_test_add_data_func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); - g_test_add_data_func("/core/c++/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left_cxx); - g_test_add_data_func("/core/c++/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right_cxx); - g_test_add_data_func("/core/c++/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle_cxx); - //g_test_add_data_func("/core/c++/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action_cxx); - g_test_add_data_func("/core/c++/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in_cxx); - g_test_add_data_func("/core/c++/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/llk/end", GINT_TO_POINTER(PB_LLk), test_end_cxx); - g_test_add_data_func("/core/c++/parser/llk/nothing", GINT_TO_POINTER(PB_LLk), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many_cxx); - g_test_add_data_func("/core/c++/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/llk/epsilon", GINT_TO_POINTER(PB_LLk), test_epsilon_cxx); - //g_test_add_data_func("/core/c++/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore_cxx); - //g_test_add_data_func("/core/c++/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec_cxx); - - g_test_add_data_func("/core/c++/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle_cxx); - // g_test_add_data_func("/core/c++/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/regex/end", GINT_TO_POINTER(PB_REGULAR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/regex/nothing", GINT_TO_POINTER(PB_REGULAR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n_cxx); - g_test_add_data_func("/core/c++/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/regex/epsilon", GINT_TO_POINTER(PB_REGULAR), test_epsilon_cxx); - // g_test_add_data_func("/core/c++/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore_cxx); - - g_test_add_data_func("/core/c++/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle_cxx); - // g_test_add_data_func("/core/c++/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/lalr/end", GINT_TO_POINTER(PB_LALR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/lalr/nothing", GINT_TO_POINTER(PB_LALR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/lalr/epsilon", GINT_TO_POINTER(PB_LALR), test_epsilon_cxx); - // g_test_add_data_func("/core/c++/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec_cxx); - - g_test_add_data_func("/core/c++/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle_cxx); - g_test_add_data_func("/core/c++/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/glr/end", GINT_TO_POINTER(PB_GLR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/glr/nothing", GINT_TO_POINTER(PB_GLR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/glr/epsilon", GINT_TO_POINTER(PB_GLR), test_epsilon_cxx); - g_test_add_data_func("/core/c++/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec_cxx); - g_test_add_data_func("/core/c++/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous_cxx); - -} - -/* -int main(int argc, char **argv) { - hammer::Ch p = hammer::Ch('a'); - std::string test = "a"; - hammer::UintResult r = p.parse(test); - std::cerr << r.result() << std::endl; - hammer::SequenceResult s = p.many().parse(test); - hammer::UintResult u = boost::get(s.result()[0]); - std::cerr << u.result() << std::endl; - return 0; -} -*/ diff --git a/src/t_parser.c b/src/t_parser.c index c49e443..59adf36 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -442,199 +442,199 @@ static void test_ambiguous(gconstpointer backend) { } void register_parser_tests(void) { - g_test_add_data_func("/core/c/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); - g_test_add_data_func("/core/c/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); - g_test_add_data_func("/core/c/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); - g_test_add_data_func("/core/c/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); - g_test_add_data_func("/core/c/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); - g_test_add_data_func("/core/c/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); - g_test_add_data_func("/core/c/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); - g_test_add_data_func("/core/c/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); - g_test_add_data_func("/core/c/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); - g_test_add_data_func("/core/c/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); - g_test_add_data_func("/core/c/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); - g_test_add_data_func("/core/c/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); + g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); + g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); + g_test_add_data_func("/core/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); + g_test_add_data_func("/core/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); + g_test_add_data_func("/core/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); + g_test_add_data_func("/core/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); + g_test_add_data_func("/core/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); + g_test_add_data_func("/core/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); + g_test_add_data_func("/core/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); + g_test_add_data_func("/core/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); + g_test_add_data_func("/core/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); + g_test_add_data_func("/core/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); - g_test_add_data_func("/core/c/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); + g_test_add_data_func("/core/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); + g_test_add_data_func("/core/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); #endif - g_test_add_data_func("/core/c/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); - g_test_add_data_func("/core/c/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); - g_test_add_data_func("/core/c/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); - g_test_add_data_func("/core/c/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); - g_test_add_data_func("/core/c/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); - g_test_add_data_func("/core/c/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); - g_test_add_data_func("/core/c/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); - g_test_add_data_func("/core/c/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); - g_test_add_data_func("/core/c/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); - g_test_add_data_func("/core/c/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); - g_test_add_data_func("/core/c/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); - g_test_add_data_func("/core/c/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); - g_test_add_data_func("/core/c/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); - g_test_add_data_func("/core/c/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); - g_test_add_data_func("/core/c/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); - g_test_add_data_func("/core/c/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); - g_test_add_data_func("/core/c/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); - g_test_add_data_func("/core/c/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); - g_test_add_data_func("/core/c/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); - g_test_add_data_func("/core/c/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); - g_test_add_data_func("/core/c/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); - g_test_add_data_func("/core/c/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); - g_test_add_data_func("/core/c/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); - g_test_add_data_func("/core/c/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); - g_test_add_data_func("/core/c/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/c/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); - g_test_add_data_func("/core/c/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); + g_test_add_data_func("/core/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); + g_test_add_data_func("/core/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); + g_test_add_data_func("/core/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); + g_test_add_data_func("/core/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); + g_test_add_data_func("/core/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); + g_test_add_data_func("/core/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); + g_test_add_data_func("/core/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); + g_test_add_data_func("/core/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); + g_test_add_data_func("/core/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); + g_test_add_data_func("/core/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); + g_test_add_data_func("/core/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); + g_test_add_data_func("/core/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); + g_test_add_data_func("/core/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); + g_test_add_data_func("/core/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); + g_test_add_data_func("/core/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); + g_test_add_data_func("/core/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); + g_test_add_data_func("/core/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); + g_test_add_data_func("/core/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); + g_test_add_data_func("/core/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); + g_test_add_data_func("/core/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); + g_test_add_data_func("/core/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); + g_test_add_data_func("/core/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); + g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); + g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); + g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); + // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); - g_test_add_data_func("/core/c/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); - g_test_add_data_func("/core/c/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); - g_test_add_data_func("/core/c/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); - g_test_add_data_func("/core/c/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); - g_test_add_data_func("/core/c/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); - g_test_add_data_func("/core/c/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); - g_test_add_data_func("/core/c/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); - g_test_add_data_func("/core/c/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); - g_test_add_data_func("/core/c/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); - g_test_add_data_func("/core/c/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); - g_test_add_data_func("/core/c/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); - g_test_add_data_func("/core/c/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); + g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); + g_test_add_data_func("/core/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); + g_test_add_data_func("/core/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); + g_test_add_data_func("/core/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); + g_test_add_data_func("/core/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); + g_test_add_data_func("/core/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); + g_test_add_data_func("/core/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); + g_test_add_data_func("/core/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); + g_test_add_data_func("/core/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); + g_test_add_data_func("/core/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); + g_test_add_data_func("/core/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); + g_test_add_data_func("/core/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); - g_test_add_data_func("/core/c/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); + g_test_add_data_func("/core/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); + g_test_add_data_func("/core/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); #endif - g_test_add_data_func("/core/c/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); - g_test_add_data_func("/core/c/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); - g_test_add_data_func("/core/c/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); - g_test_add_data_func("/core/c/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); - g_test_add_data_func("/core/c/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); - g_test_add_data_func("/core/c/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); - g_test_add_data_func("/core/c/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); - g_test_add_data_func("/core/c/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); - g_test_add_data_func("/core/c/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); - g_test_add_data_func("/core/c/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); - g_test_add_data_func("/core/c/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); - g_test_add_data_func("/core/c/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); - g_test_add_data_func("/core/c/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); - g_test_add_data_func("/core/c/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); - g_test_add_data_func("/core/c/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); - g_test_add_data_func("/core/c/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); - g_test_add_data_func("/core/c/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); - g_test_add_data_func("/core/c/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); - g_test_add_data_func("/core/c/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); - //g_test_add_data_func("/core/c/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); - g_test_add_data_func("/core/c/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); + g_test_add_data_func("/core/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); + g_test_add_data_func("/core/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); + g_test_add_data_func("/core/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); + g_test_add_data_func("/core/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); + g_test_add_data_func("/core/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); + g_test_add_data_func("/core/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); + g_test_add_data_func("/core/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); + g_test_add_data_func("/core/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); + g_test_add_data_func("/core/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); + g_test_add_data_func("/core/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); + g_test_add_data_func("/core/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); + g_test_add_data_func("/core/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); + g_test_add_data_func("/core/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); + g_test_add_data_func("/core/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); + g_test_add_data_func("/core/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); + g_test_add_data_func("/core/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); + g_test_add_data_func("/core/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); + g_test_add_data_func("/core/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); + g_test_add_data_func("/core/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); + //g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); + g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); - g_test_add_data_func("/core/c/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); - g_test_add_data_func("/core/c/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); - g_test_add_data_func("/core/c/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); - g_test_add_data_func("/core/c/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); - g_test_add_data_func("/core/c/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); - g_test_add_data_func("/core/c/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); - g_test_add_data_func("/core/c/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); - g_test_add_data_func("/core/c/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); - g_test_add_data_func("/core/c/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); - g_test_add_data_func("/core/c/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); - g_test_add_data_func("/core/c/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); - g_test_add_data_func("/core/c/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); + g_test_add_data_func("/core/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); + g_test_add_data_func("/core/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); + g_test_add_data_func("/core/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); + g_test_add_data_func("/core/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); + g_test_add_data_func("/core/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); + g_test_add_data_func("/core/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); + g_test_add_data_func("/core/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); + g_test_add_data_func("/core/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); + g_test_add_data_func("/core/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); + g_test_add_data_func("/core/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); + g_test_add_data_func("/core/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); + g_test_add_data_func("/core/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); - g_test_add_data_func("/core/c/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); + g_test_add_data_func("/core/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); + g_test_add_data_func("/core/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); #endif - g_test_add_data_func("/core/c/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); - g_test_add_data_func("/core/c/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); - g_test_add_data_func("/core/c/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); - g_test_add_data_func("/core/c/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); - g_test_add_data_func("/core/c/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); - g_test_add_data_func("/core/c/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); - g_test_add_data_func("/core/c/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); - g_test_add_data_func("/core/c/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); - g_test_add_data_func("/core/c/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); - g_test_add_data_func("/core/c/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); - g_test_add_data_func("/core/c/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); - g_test_add_data_func("/core/c/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); - g_test_add_data_func("/core/c/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); - g_test_add_data_func("/core/c/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); - g_test_add_data_func("/core/c/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); - g_test_add_data_func("/core/c/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); - g_test_add_data_func("/core/c/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); - g_test_add_data_func("/core/c/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); - g_test_add_data_func("/core/c/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); + g_test_add_data_func("/core/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); + g_test_add_data_func("/core/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); + g_test_add_data_func("/core/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); + g_test_add_data_func("/core/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); + g_test_add_data_func("/core/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); + g_test_add_data_func("/core/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); + g_test_add_data_func("/core/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); + g_test_add_data_func("/core/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); + g_test_add_data_func("/core/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); + g_test_add_data_func("/core/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); + g_test_add_data_func("/core/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); + g_test_add_data_func("/core/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); + g_test_add_data_func("/core/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); + g_test_add_data_func("/core/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); + g_test_add_data_func("/core/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); + g_test_add_data_func("/core/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); + g_test_add_data_func("/core/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); + g_test_add_data_func("/core/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); + g_test_add_data_func("/core/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); + g_test_add_data_func("/core/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); - g_test_add_data_func("/core/c/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); - g_test_add_data_func("/core/c/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); - g_test_add_data_func("/core/c/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); - g_test_add_data_func("/core/c/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); - g_test_add_data_func("/core/c/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); - g_test_add_data_func("/core/c/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); - g_test_add_data_func("/core/c/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); - g_test_add_data_func("/core/c/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); - g_test_add_data_func("/core/c/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); - g_test_add_data_func("/core/c/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); - g_test_add_data_func("/core/c/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); - g_test_add_data_func("/core/c/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); + g_test_add_data_func("/core/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); + g_test_add_data_func("/core/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); + g_test_add_data_func("/core/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); + g_test_add_data_func("/core/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); + g_test_add_data_func("/core/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); + g_test_add_data_func("/core/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); + g_test_add_data_func("/core/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); + g_test_add_data_func("/core/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); + g_test_add_data_func("/core/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); + g_test_add_data_func("/core/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); + g_test_add_data_func("/core/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); + g_test_add_data_func("/core/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); - g_test_add_data_func("/core/c/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); + g_test_add_data_func("/core/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); + g_test_add_data_func("/core/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); #endif - g_test_add_data_func("/core/c/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); - g_test_add_data_func("/core/c/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); - g_test_add_data_func("/core/c/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); - g_test_add_data_func("/core/c/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); - g_test_add_data_func("/core/c/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); - g_test_add_data_func("/core/c/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); - g_test_add_data_func("/core/c/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); - g_test_add_data_func("/core/c/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); - g_test_add_data_func("/core/c/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); - g_test_add_data_func("/core/c/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); - g_test_add_data_func("/core/c/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); - g_test_add_data_func("/core/c/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); - g_test_add_data_func("/core/c/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); - g_test_add_data_func("/core/c/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); - g_test_add_data_func("/core/c/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); - g_test_add_data_func("/core/c/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); - g_test_add_data_func("/core/c/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); - g_test_add_data_func("/core/c/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); - g_test_add_data_func("/core/c/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); - g_test_add_data_func("/core/c/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); + g_test_add_data_func("/core/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); + g_test_add_data_func("/core/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); + g_test_add_data_func("/core/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); + g_test_add_data_func("/core/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); + g_test_add_data_func("/core/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); + g_test_add_data_func("/core/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); + g_test_add_data_func("/core/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); + g_test_add_data_func("/core/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); + g_test_add_data_func("/core/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); + g_test_add_data_func("/core/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); + g_test_add_data_func("/core/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); + g_test_add_data_func("/core/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); + g_test_add_data_func("/core/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); + g_test_add_data_func("/core/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); + g_test_add_data_func("/core/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); + g_test_add_data_func("/core/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); + g_test_add_data_func("/core/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); + g_test_add_data_func("/core/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); + g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); + g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); + g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); - g_test_add_data_func("/core/c/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); - g_test_add_data_func("/core/c/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); - g_test_add_data_func("/core/c/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); - g_test_add_data_func("/core/c/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); - g_test_add_data_func("/core/c/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); - g_test_add_data_func("/core/c/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); - g_test_add_data_func("/core/c/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); - g_test_add_data_func("/core/c/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); - g_test_add_data_func("/core/c/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); - g_test_add_data_func("/core/c/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); - g_test_add_data_func("/core/c/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); - g_test_add_data_func("/core/c/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); + g_test_add_data_func("/core/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); + g_test_add_data_func("/core/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); + g_test_add_data_func("/core/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); + g_test_add_data_func("/core/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); + g_test_add_data_func("/core/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); + g_test_add_data_func("/core/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); + g_test_add_data_func("/core/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); + g_test_add_data_func("/core/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); + g_test_add_data_func("/core/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); + g_test_add_data_func("/core/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); + g_test_add_data_func("/core/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); + g_test_add_data_func("/core/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); - g_test_add_data_func("/core/c/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); + g_test_add_data_func("/core/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); + g_test_add_data_func("/core/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); #endif - g_test_add_data_func("/core/c/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); - g_test_add_data_func("/core/c/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); - g_test_add_data_func("/core/c/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); - g_test_add_data_func("/core/c/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); - g_test_add_data_func("/core/c/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); - g_test_add_data_func("/core/c/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); - g_test_add_data_func("/core/c/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); - g_test_add_data_func("/core/c/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); - g_test_add_data_func("/core/c/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); - g_test_add_data_func("/core/c/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); - g_test_add_data_func("/core/c/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); - g_test_add_data_func("/core/c/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); - g_test_add_data_func("/core/c/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); - g_test_add_data_func("/core/c/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); - g_test_add_data_func("/core/c/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); - g_test_add_data_func("/core/c/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); - g_test_add_data_func("/core/c/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); - g_test_add_data_func("/core/c/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); - g_test_add_data_func("/core/c/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); - g_test_add_data_func("/core/c/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); - g_test_add_data_func("/core/c/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); + g_test_add_data_func("/core/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); + g_test_add_data_func("/core/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); + g_test_add_data_func("/core/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); + g_test_add_data_func("/core/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); + g_test_add_data_func("/core/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); + g_test_add_data_func("/core/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); + g_test_add_data_func("/core/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); + g_test_add_data_func("/core/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); + g_test_add_data_func("/core/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); + g_test_add_data_func("/core/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); + g_test_add_data_func("/core/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); + g_test_add_data_func("/core/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); + g_test_add_data_func("/core/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); + g_test_add_data_func("/core/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); + g_test_add_data_func("/core/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); + g_test_add_data_func("/core/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); + g_test_add_data_func("/core/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); + g_test_add_data_func("/core/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); + g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); + g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); + g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); + g_test_add_data_func("/core/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); } diff --git a/src/test_suite.c b/src/test_suite.c index 9191f93..e065f13 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -25,9 +25,6 @@ extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); extern void register_benchmark_tests(); -#ifdef __cplusplus -extern void register_cxx_tests(); -#endif int main(int argc, char** argv) { g_test_init(&argc, &argv, NULL); @@ -39,9 +36,6 @@ int main(int argc, char** argv) { register_grammar_tests(); register_misc_tests(); register_benchmark_tests(); - #ifdef __cplusplus - register_cxx_tests(); - #endif g_test_run(); } diff --git a/src/test_suite.h b/src/test_suite.h index 76c6d78..fc008e7 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -19,16 +19,6 @@ #define HAMMER_TEST_SUITE__H #include -#ifdef __cplusplus -extern "C" { -#include -#include -#endif - -#ifndef __cplusplus -typedef int bool; -#endif - // Equivalent to g_assert_*, but not using g_assert... #define g_check_inttype(fmt, typ, n1, op, n2) do { \ typ _n1 = (n1); \ @@ -62,19 +52,6 @@ typedef int bool; } \ } while(0) -#ifdef __cplusplus -#define g_check_string_cxx(n1, op, n2) do { \ - std::string _n1 = (n1); \ - std::string _n2 = (n2); \ - if (!(_n1.compare(_n2) op 0)) { \ - g_test_message("Check failed: (%s) (%s %s %s)", \ - #n1 " " #op " " #n2, \ - _n1.c_str(), #op, _n2.c_str()); \ - g_test_fail(); \ - } \ - } while(0) -#endif - #define g_check_regular(lang) do { \ if (!lang->isValidRegular(lang->env)) { \ g_test_message("Language is not regular"); \ @@ -118,21 +95,6 @@ typedef int bool; g_test_fail(); \ } \ } while(0) - -#ifdef __cplusplus -#define g_check_parse_failed_cxx(parser, backend, input) do { \ - int skip = parser.compile((HParserBackend)backend, NULL); \ - if(skip != 0) { \ - g_test_message("Backend not applicable, skipping test"); \ - break; \ - } \ - parser::result_type result = parser.parse(input); \ - if (NULL != result) { \ - g_test_message("Check failed: shouldn't have succeeded, but did"); \ - g_test_fail(); \ - } \ - } while(0) -#endif #define g_check_parse_ok(parser, backend, input, inp_len, result) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ @@ -158,34 +120,6 @@ typedef int bool; } \ } while(0) -#ifdef __cplusplus -#define g_check_parse_ok_cxx(parser, backend, input, result) do { \ - int skip = parser.compile((HParserBackend)backend, NULL); \ - if(skip) { \ - g_test_message("Backend not applicable, skipping test"); \ - break; \ - } \ - parser::result_type res = parser.parse((std::string)input); \ - if (!res) { \ - g_test_message("Parse failed on line %d", __LINE__); \ - g_test_fail(); \ - } else { \ - std::string& cres = h_write_result_unamb(res); \ - g_check_string_cxx(cres, ==, (std::string)result); \ - /* - system_allocator.free(&system_allocator, cres); \ - HArenaStats stats; \ - h_allocator_stats(res->arena, &stats); \ - g_test_message("Parse used %zd bytes, wasted %zd bytes. " \ - "Inefficiency: %5f%%", \ - stats.used, stats.wasted, \ - stats.wasted * 100. / (stats.used+stats.wasted)); \ - h_delete_arena(res->arena); \ - */ // FIXME might've made a memory leak there - } \ - } while(0) -#endif - #define g_check_hashtable_present(table, key) do { \ if(!h_hashtable_present(table, key)) { \ g_test_message("Check failed: key should have been in table, but wasn't"); \ @@ -263,8 +197,6 @@ typedef int bool; #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) -#ifdef __cplusplus -} -#endif + #endif // #ifndef HAMMER_TEST_SUITE__H From 78b7904c5a74fd8044f557355e2e2485060dc414 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 006/103] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 487fa97..37a273b 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -124,6 +124,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -570,7 +571,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From f8604f5c7658d1bc051267ed36c0b12e67350da1 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 007/103] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From 8928c04456e8bb190d02d06869fc13278aa9f3bf Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 008/103] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..7a05636 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From 390bbd32f12ef45fd801fcb633ff836e26c5e565 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 009/103] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From 06aa19e206ea0155aa17825dbcf5645d1be57685 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 010/103] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From c9c282dcb468daebc45073193b568f5f55ae4f01 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 011/103] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 9061318dd691359578322736fa131c90305c7005 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 012/103] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index f5a9e13..e435587 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -29,7 +29,9 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H typedef struct HParseState_ HParseState; @@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From 529aa3e1a3d67272e2c8c0dea4d620252e5c8447 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 013/103] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index e435587..a7046d0 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -70,8 +70,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -81,6 +94,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -144,12 +160,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From c54f63871af927d238f361bf3a3565a32e8ff0a6 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 014/103] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From fc63ee5d1bd1451db8bb80829be07772e452422f Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 015/103] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index a7046d0..d8c613d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} #endif // #ifndef HAMMER_HAMMER__H From a5da5804570a5b2449fcd2b3d06ffbff5b55efe9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 016/103] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From 804b8f7bd1f0c4667e63d1e93de98bd0f6a69e66 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 017/103] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 9b5c868..cb5d7f6 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From 6effa7ae3ebfc8acdd7858facdf456cb1ec50a52 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 018/103] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1458ecd..dd66860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 0eab9112587315b7e9089fd022e434412cb22e7a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 019/103] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From f1e6e1fa9532043ed515fff6523252e9e5e6a5c9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 020/103] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From 83fdfd93fda0d583562e35c62ebbe2726efa3da5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 021/103] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From e8b2c17026070e47d00835da3f30a29235d91bb4 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 022/103] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From d1b71779e130b4d1dfeab98f2958fa300cc589b5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 023/103] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From f3ce2fc7de48a7be27f7c9cabfa710ec56a9fe41 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 024/103] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From cec4659baa148de18403db5c305655453b1b019b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 025/103] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From 8681600365ae02ed3ff5c209520419a8be5d81b8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:29:44 -0600 Subject: [PATCH 026/103] working on fixing leftrec for packrat --- src/t_parser.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/t_parser.c b/src/t_parser.c index 59adf36..0a7aede 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -410,11 +410,11 @@ static void test_leftrec(gconstpointer backend) { HParser *a_ = h_ch('a'); HParser *lr_ = h_indirect(); - h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL)); + h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), h_epsilon_p(), NULL)); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "u0x61"); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)"); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "((u0x61 u0x61) u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "((u0x61) u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "(((u0x61) u0x61) u0x61)"); } static void test_rightrec(gconstpointer backend) { @@ -483,7 +483,7 @@ void register_parser_tests(void) { g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + //g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); From 656f4546ddbdfb405ae66845484d1b43cd5fe250 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 027/103] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dd66860..2276989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 9e44bea92035b91c4e2764ad432b5f25c04ed81a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 12:37:30 -0600 Subject: [PATCH 028/103] commenting out assert for debugging purposes --- src/backends/packrat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index 8aa1f8e..87f166d 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -83,7 +83,7 @@ void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) { some->eval_set = NULL; rec_detect->head = some; } - assert(state->lr_stack->head != NULL); + //assert(state->lr_stack->head != NULL); HSlistNode *head = state->lr_stack->head; HLeftRec *lr; while (head && (lr = head->elem)->rule != p) { From 62af9f622233af22c0ac58b94e58157bfd5fabd5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 029/103] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 10c8b0bd22bd1f172fb04aa2434e40035accd626 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:00:58 -0600 Subject: [PATCH 030/103] fixed the segfault! hand-initialized HParser needed PB_MIN set. --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- src/parsers/choice.c | 4 +++- src/parsers/sequence.c | 4 +++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass diff --git a/src/parsers/choice.c b/src/parsers/choice.c index 6db1378..bfc3f90 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &choice_vt; ret->env = (void*)s; + ret->vtable = &choice_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index eff4610..281b78f 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -159,6 +159,8 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &sequence_vt; ret->env = (void*)s; + ret->vtable = &sequence_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } From c32c5cf5eae3a51891c4bae2a55f66c1f520db26 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:05:48 -0600 Subject: [PATCH 031/103] Fixed a potential segfault; hand-initialized HParsers in h_choice and h_sequence need PB_MIN set. Conflicts: src/bindings/python/SConscript src/bindings/python/hammer_tests.py --- src/parsers/choice.c | 4 +++- src/parsers/sequence.c | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/parsers/choice.c b/src/parsers/choice.c index 6db1378..bb55fa8 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &choice_vt; ret->env = (void*)s; + ret->vtable = &choice_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index eff4610..bdea8c0 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -145,20 +145,22 @@ HParser* h_sequence__a(void *args[]) { HParser* h_sequence__ma(HAllocator* mm__, void *args[]) { size_t len = -1; // because do...while const HParser *arg; - + do { arg=((HParser **)args)[++len]; } while(arg); - + HSequence *s = h_new(HSequence, 1); s->p_array = h_new(HParser *, len); for (size_t i = 0; i < len; i++) { s->p_array[i] = ((HParser **)args)[i]; } - + s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &sequence_vt; ret->env = (void*)s; + ret->vtable = &sequence_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } From 87581e09cbdfde5261bb33c6bffacf5c3b078145 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 032/103] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 9091df8..a4e828d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -132,6 +132,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -592,7 +593,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From 42270b613dcbbf34bcea48b488cf759683c3b359 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 033/103] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From a4dbfc61f224510d7e75ea66ab5dc90e58e3b204 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 034/103] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..7a05636 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From a99d7a18d15362e97f9210231b40be7b28d41b85 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 035/103] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From c58555d6a960ef9552c0ea18d53b4aa97aeace3e Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 036/103] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From 4330a426e927fee43375d27fce998fcb96bdd5c8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 037/103] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 34062d50a20e289218d21c238c112ef0548bb61a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 038/103] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index a4e828d..6928cc9 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -34,7 +34,9 @@ extern "C" { #endif #ifndef __cplusplus +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H #endif typedef struct HParseState_ HParseState; @@ -183,7 +185,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -201,17 +203,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From eb04069817f445f66e1f46dd077e1a81d131c3b9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 039/103] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 6928cc9..5dac8bf 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -76,8 +76,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -87,6 +100,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -150,12 +166,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From 21f16ebefdbe8a5ff149188ad51f4fcdd2d61250 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 040/103] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From 902ca46b4d9c741dbe2c64f151d021fb043e3a78 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 041/103] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 5dac8bf..b4fc823 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -666,7 +666,7 @@ HParsedToken *h_act_ignore(const HParseResult *p); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} #ifdef __cplusplus From 551645e98069a906fd7f5ba336615b3bc38c563d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 042/103] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From c58a722aef9d6ad67de4683fb22c0b39c77d8ab4 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 043/103] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 9b5c868..cb5d7f6 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From 55af8ac28751469851a93b98254d1a43beb0920d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 044/103] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1458ecd..dd66860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 150e91537b3b5a0ce31302ba42609f1ace7a197f Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 045/103] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From 8fcad1e72ac22ce32843c434b2a58298e3a97fc2 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 046/103] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From 6746df20dbb2cc0a842e6c0bb4825bc26318380d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 047/103] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From b979f66c21ad8dbdbb1e60d78248b659e6557cfc Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 048/103] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From 2d459282d45bec0927ca81b68ff3cac398d06685 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 049/103] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From 5ced9fe9d5cb1d93fe9fb34dc156f6d6bc4fae04 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 050/103] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From 2e0c56f403bf4298d128584b714ea5a97c4ca75e Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 051/103] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From eb1cee7dee214abb0b6c431f3d52370aa08d619d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 052/103] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dd66860..2276989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 6e456e3daac1499ae017e44bad04f0d412f4f949 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 053/103] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 31f283e72023a5c5136810b91bf38103ec625140 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:10:56 -0600 Subject: [PATCH 054/103] Fixed TestRightrec --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass From 4811f583741ca8b24bb08c0fc173807a34a320f8 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 19 Nov 2013 21:14:39 -0600 Subject: [PATCH 055/103] Added token type registry. Closes #45 --- src/SConscript | 1 + src/hammer.h | 11 +++++++ src/registry.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/t_misc.c | 19 +++++++++++ 4 files changed, 118 insertions(+) create mode 100644 src/registry.c diff --git a/src/SConscript b/src/SConscript index 9b5c868..03308dd 100644 --- a/src/SConscript +++ b/src/SConscript @@ -49,6 +49,7 @@ misc_hammer_parts = [ 'glue.c', 'hammer.c', 'pprint.c', + 'registry.c', 'system_allocator.c'] tests = ['t_benchmark.c', diff --git a/src/hammer.h b/src/hammer.h index 508653d..0175142 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -633,4 +633,15 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results); void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} +// {{{ Token type registry +/// Allocate a new, unused (as far as this function knows) token type. +int h_allocate_token_type(const char* name); + +/// Get the token type associated with name. Returns -1 if name is unkown +int h_get_token_type_number(const char* name); + +/// Get the name associated with token_type. Returns NULL if the token type is unkown +const char* h_get_token_type_name(int token_type); +// }}} + #endif // #ifndef HAMMER_HAMMER__H diff --git a/src/registry.c b/src/registry.c new file mode 100644 index 0000000..c59b6ea --- /dev/null +++ b/src/registry.c @@ -0,0 +1,87 @@ +/* Parser combinators for binary formats. + * Copyright (C) 2012 Meredith L. Patterson, Dan "TQ" Hirsch + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include "hammer.h" +#include "internal.h" + +typedef struct Entry_ { + const char* name; + int value; +} Entry; + +static void *tt_registry = NULL; +static Entry** tt_by_id = NULL; +static int tt_by_id_sz = 0; +#define TT_START TT_USER +static int tt_next = TT_START; + +/* + // TODO: These are for the extension registry, which does not yet have a good name. +static void *ext_registry = NULL; +static Entry** ext_by_id = NULL; +static int ext_by_id_sz = 0; +static int ext_next = 0; +*/ + + +static int compare_entries(const void* v1, const void* v2) { + const Entry *e1 = (Entry*)v1, *e2 = (Entry*)v2; + return strcmp(e1->name, e2->name); +} + +int h_allocate_token_type(const char* name) { + Entry* new_entry = malloc(sizeof(*new_entry)); + new_entry->name = name; + new_entry->value = -1; + Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries); + if (probe->value != -1) { + // Token type already exists... + // TODO: treat this as a bug? + free(new_entry); + return probe->value; + } else { + // new value + probe->name = strdup(probe->name); // drop ownership of name + probe->value = tt_next++; + if ((probe->value - TT_START) >= tt_by_id_sz) { + if (tt_by_id_sz == 0) + tt_by_id = malloc(sizeof(*tt_by_id) * ((tt_by_id_sz = (tt_next - TT_START) * 16))); + else + tt_by_id = realloc(tt_by_id, sizeof(*tt_by_id) * ((tt_by_id_sz *= 2))); + } + assert(probe->value - TT_START < tt_by_id_sz); + tt_by_id[probe->value - TT_START] = probe; + return probe->value; + } +} +int h_get_token_type_number(const char* name) { + Entry e; + e.name = name; + Entry **ret = (Entry**)tfind(&e, &tt_registry, compare_entries); + if (ret == NULL) + return -1; + else + return (*ret)->value; +} +const char* h_get_token_type_name(int token_type) { + if (token_type >= tt_next || token_type < TT_START) + return NULL; + else + return tt_by_id[token_type - TT_START]->name; +} diff --git a/src/t_misc.c b/src/t_misc.c index c762c07..74a57ca 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -1,4 +1,5 @@ #include +#include #include "test_suite.h" #include "hammer.h" @@ -11,6 +12,24 @@ static void test_tt_user(void) { g_check_cmp_int32(TT_USER, >, TT_ERR); } +static void test_tt_registry(void) { + int id = h_allocate_token_type("com.upstandinghackers.test.token_type"); + g_check_cmp_int32(id, >=, TT_USER); + int id2 = h_allocate_token_type("com.upstandinghackers.test.token_type_2"); + g_check_cmp_int32(id2, !=, id); + g_check_cmp_int32(id2, >=, TT_USER); + g_check_cmp_int32(id, ==, h_get_token_type_number("com.upstandinghackers.test.token_type")); + g_check_cmp_int32(id2, ==, h_get_token_type_number("com.upstandinghackers.test.token_type_2")); + g_check_string("com.upstandinghackers.test.token_type", ==, h_get_token_type_name(id)); + g_check_string("com.upstandinghackers.test.token_type_2", ==, h_get_token_type_name(id2)); + if (h_get_token_type_name(0) != NULL) { + g_test_message("Unknown token type should not return a name"); + g_test_fail(); + } + g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, -1); +} + void register_misc_tests(void) { g_test_add_func("/core/misc/tt_user", test_tt_user); + g_test_add_func("/core/misc/tt_registry", test_tt_registry); } From 91221ac2563b301ccc3bd3dcec8bdf4dd1356e8d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 20:46:34 -0600 Subject: [PATCH 056/103] tweak SConstruct so that 'scan-build scons' works --- SConstruct | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SConstruct b/SConstruct index 623bc5d..78090c3 100644 --- a/SConstruct +++ b/SConstruct @@ -68,10 +68,15 @@ if GetOption("coverage"): LDFLAGS=["-fprofile-arcs", "-ftest-coverage"], LIBS=['gcov']) +env["CC"] = os.getenv("CC") or env["CC"] +env["CXX"] = os.getenv("CXX") or env["CXX"] + if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") +env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) + #rootpath = env['ROOTPATH'] = os.path.abspath('.') #env.Append(CPPPATH=os.path.join('#', "hammer")) From f220524b5a383a7945a1d834a3600bd07f7b2edc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:04:07 -0800 Subject: [PATCH 057/103] Generate pkg-config for Hammer Create a pkg-config file and install it to $prefix/lib/pkgconfig --- SConstruct | 6 ++++-- libhammer.pc.in | 10 ++++++++++ src/SConscript | 1 + tools/scanreplace.py | 15 +++++++++++++++ tools/scanreplace.pyc | Bin 0 -> 1263 bytes 5 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 libhammer.pc.in create mode 100644 tools/scanreplace.py create mode 100644 tools/scanreplace.pyc diff --git a/SConstruct b/SConstruct index 78090c3..422efe6 100644 --- a/SConstruct +++ b/SConstruct @@ -8,7 +8,7 @@ vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept)) -env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars) +env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars, tools=['default', 'scanreplace'], toolpath=['tools']) def calcInstallPath(*elements): path = os.path.abspath(os.path.join(*map(env.subst, elements))) @@ -28,7 +28,8 @@ if 'DESTDIR' in env: env['libpath'] = calcInstallPath("$prefix", "lib") env['incpath'] = calcInstallPath("$prefix", "include", "hammer") -# TODO: Add pkgconfig +env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig") +env.ScanReplace('libhammer.pc.in') env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes") @@ -89,3 +90,4 @@ env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/ env.Alias("install", "$libpath") env.Alias("install", "$incpath") +env.Alias("install", "$pkgconfigpath") diff --git a/libhammer.pc.in b/libhammer.pc.in new file mode 100644 index 0000000..97ded13 --- /dev/null +++ b/libhammer.pc.in @@ -0,0 +1,10 @@ +prefix=/usr +exec_prefix=${prefix} +includedir=${prefix}/include +libdir=${exec_prefix}/lib + +Name: libhammer +Description: The Hammer parsing library +Version: 0.9.0 +Cflags: -I${includedir}/hammer +Libs: -L${libdir} -lhammer diff --git a/src/SConscript b/src/SConscript index 03308dd..a06244b 100644 --- a/src/SConscript +++ b/src/SConscript @@ -64,6 +64,7 @@ libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_ env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) +env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') diff --git a/tools/scanreplace.py b/tools/scanreplace.py new file mode 100644 index 0000000..5321e48 --- /dev/null +++ b/tools/scanreplace.py @@ -0,0 +1,15 @@ +from string import Template + +def replace_action(target, source, env): + open(str(target[0]), 'w').write(Template(open(str(source[0]), 'r').read()).safe_substitute(env)) + return 0 + +def replace_string(target, source, env): + return "building '%s' from '%s'" % (str(target[0]), str(source[0])) + +def generate(env, **kw): + action = env.Action(replace_action, replace_string) + env['BUILDERS']['ScanReplace'] = env.Builder(action=action, src_suffix='.in', single_source=True) + +def exists(env): + return 1 diff --git a/tools/scanreplace.pyc b/tools/scanreplace.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c37386cee5c446284c300e734bcd48d69842fc26 GIT binary patch literal 1263 zcmZSn%*(Y>tT!l`0ScHH7#JKF7#NBz7#J8*7#LC*8FCmHav2$-7#SE-m>60Z7^0XM zQkcOEW`-0NFoT65g%!+TWk_LT2-aX{U|`7n|NsAg4MqkAh7t}228NK-+=86MlGJ2o z1_lNYW@BJra0c1r15(1kP{PPi1GZ1HnUNudiJ_T^A&ZHjgqfiPBF6}F3s{aBEXTso z%)rph$dJX#P{RabG=p5o$Plc-!oa}bmkIJv3CQ|#Fj=GlvakdsmtT;YR|0ZEaY<1L zD+2>Vc~NFbY7kfhNWLgFF{Okb8T}PW^sOL zQF1ESxYWEd4Ui#VqS%Omfk8jCIJKx)KPx{~zqlw_KO-?WH?>GNt0=!JFSSU&BtJi= zSid+qF|R1KASW?7Rj;73gpYxN0m_X}OfJdH&jZ<7400C(qX#2A_(1Lk2cHrsq8LG; z1PVST22c={Ffi1B0};eYV`2~s2D{y_Se$`@A*nPoCnYm4T|r&7SY08lC_fj>0tXm4 zzJfrOfi(nyVlIf2fq@~2i-Caw>@hGw4c`@)6lLb6gOW-yDAh4ACWB%N)S|?aRItB5@y5U?1PMq`V1h6>k$_}UKzRe?0KZI-IM`}Pe1kMYxS-@qu5&;c zBef#4xTF}AhM{rl2g!CIIY@kivl$}5g2X_H3zRGo4gj;rbpc<^ik# literal 0 HcmV?d00001 From dbe0785e2cbe41de8c77a8c584d40de2b7606b10 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:25:09 -0800 Subject: [PATCH 058/103] Remove pyc file --- tools/scanreplace.pyc | Bin 1263 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tools/scanreplace.pyc diff --git a/tools/scanreplace.pyc b/tools/scanreplace.pyc deleted file mode 100644 index c37386cee5c446284c300e734bcd48d69842fc26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1263 zcmZSn%*(Y>tT!l`0ScHH7#JKF7#NBz7#J8*7#LC*8FCmHav2$-7#SE-m>60Z7^0XM zQkcOEW`-0NFoT65g%!+TWk_LT2-aX{U|`7n|NsAg4MqkAh7t}228NK-+=86MlGJ2o z1_lNYW@BJra0c1r15(1kP{PPi1GZ1HnUNudiJ_T^A&ZHjgqfiPBF6}F3s{aBEXTso z%)rph$dJX#P{RabG=p5o$Plc-!oa}bmkIJv3CQ|#Fj=GlvakdsmtT;YR|0ZEaY<1L zD+2>Vc~NFbY7kfhNWLgFF{Okb8T}PW^sOL zQF1ESxYWEd4Ui#VqS%Omfk8jCIJKx)KPx{~zqlw_KO-?WH?>GNt0=!JFSSU&BtJi= zSid+qF|R1KASW?7Rj;73gpYxN0m_X}OfJdH&jZ<7400C(qX#2A_(1Lk2cHrsq8LG; z1PVST22c={Ffi1B0};eYV`2~s2D{y_Se$`@A*nPoCnYm4T|r&7SY08lC_fj>0tXm4 zzJfrOfi(nyVlIf2fq@~2i-Caw>@hGw4c`@)6lLb6gOW-yDAh4ACWB%N)S|?aRItB5@y5U?1PMq`V1h6>k$_}UKzRe?0KZI-IM`}Pe1kMYxS-@qu5&;c zBef#4xTF}AhM{rl2g!CIIY@kivl$}5g2X_H3zRGo4gj;rbpc<^ik# From f1a5844523b2424a784995dffcad54c4832764bc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:26:00 -0800 Subject: [PATCH 059/103] Ignore pyc files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e05a551..16fdda1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ Session.vim cscope.out build/ .sconsign.dblite +*.pyc From a98272b8c0979c191c0bda17cd1fc116f580f4df Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Wed, 20 Nov 2013 15:37:24 -0600 Subject: [PATCH 060/103] Reduce IRC spam --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1458ecd..246a3d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,9 @@ compiler: script: - scons notifications: - irc: "irc.upstandinghackers.com#hammer" + irc: + channels: + - "irc.upstandinghackers.com#hammer" + use_notice: true + skip_join: true + From 76034ae4d7db051d0dd351e9880b8322c6db5883 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:02:18 -0800 Subject: [PATCH 061/103] Install internal headers for Go bindings Go, and likely other language bindings will, need access to create new combinators. This functionality is currently located src/parsers/parser_internal.h. Install this header to the system with dependent headers until a better solution can be found. --- SConstruct | 4 ++++ src/SConscript | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 422efe6..ebed8b4 100644 --- a/SConstruct +++ b/SConstruct @@ -28,6 +28,8 @@ if 'DESTDIR' in env: env['libpath'] = calcInstallPath("$prefix", "lib") env['incpath'] = calcInstallPath("$prefix", "include", "hammer") +env['parsersincpath'] = calcInstallPath("$prefix", "include", "hammer", "parsers") +env['backendsincpath'] = calcInstallPath("$prefix", "include", "hammer", "backends") env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig") env.ScanReplace('libhammer.pc.in') @@ -90,4 +92,6 @@ env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/ env.Alias("install", "$libpath") env.Alias("install", "$incpath") +env.Alias("install", "$parsersincpath") +env.Alias("install", "$backendsincpath") env.Alias("install", "$pkgconfigpath") diff --git a/src/SConscript b/src/SConscript index a06244b..1d9ca76 100644 --- a/src/SConscript +++ b/src/SConscript @@ -6,7 +6,17 @@ bindings = [] dist_headers = [ "hammer.h", "allocator.h", - "glue.h" + "glue.h", + "internal.h" +] + +parsers_headers = [ + "parsers/parser_internal.h" +] + +backends_headers = [ + "backends/regex.h", + "backends/contextfree.h" ] parsers = ['parsers/%s.c'%s for s in @@ -64,6 +74,8 @@ libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_ env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) +env.Install("$parsersincpath", parsers_headers) +env.Install("$backendsincpath", backends_headers) env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() From 1d80058c851fb31dcc46ddcc7ceac12297fa9ca6 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:37:36 -0800 Subject: [PATCH 062/103] Properly name the lib Set the proper path for the shared library so that anything linking against it will be able to find the library. --- SConstruct | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SConstruct b/SConstruct index ebed8b4..0512d71 100644 --- a/SConstruct +++ b/SConstruct @@ -78,6 +78,8 @@ if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") +env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) + env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') From d0906e69e6a3cd17e68b43eb83b8e0b522f02fe8 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:50:03 -0800 Subject: [PATCH 063/103] Fix for linux This actually is only for OS X. On linux ldconfig must be run after installing in order to update the library cache. Need to look into how to get scons to do this for the user. --- SConstruct | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index 0512d71..c46c090 100644 --- a/SConstruct +++ b/SConstruct @@ -37,6 +37,7 @@ env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attr if not env['PLATFORM'] == 'darwin': env.MergeFlags("-lrt") + env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) AddOption("--variant", dest="variant", @@ -78,8 +79,6 @@ if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") -env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) - env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') From bb02aa18e76ed4236526572b3093ce38b64f2fbc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 22:09:25 -0800 Subject: [PATCH 064/103] Add warning message Specify warning for internal only headers that are now installed for use by extensions. --- src/backends/contextfree.h | 5 +++++ src/backends/regex.h | 5 +++++ src/internal.h | 5 +++++ src/parsers/parser_internal.h | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/src/backends/contextfree.h b/src/backends/contextfree.h index b387e55..ab04ab5 100644 --- a/src/backends/contextfree.h +++ b/src/backends/contextfree.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + // This is an internal header; it provides macros to make desugaring cleaner. #include #include "../internal.h" diff --git a/src/backends/regex.h b/src/backends/regex.h index a84904d..4ea85a8 100644 --- a/src/backends/regex.h +++ b/src/backends/regex.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + // Internal defs #ifndef HAMMER_BACKEND_REGEX__H #define HAMMER_BACKEND_REGEX__H diff --git a/src/internal.h b/src/internal.h index c402da5..89cb380 100644 --- a/src/internal.h +++ b/src/internal.h @@ -15,6 +15,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + #ifndef HAMMER_INTERNAL__H #define HAMMER_INTERNAL__H #include diff --git a/src/parsers/parser_internal.h b/src/parsers/parser_internal.h index aeb202b..ec97dd1 100644 --- a/src/parsers/parser_internal.h +++ b/src/parsers/parser_internal.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + #ifndef HAMMER_PARSE_INTERNAL__H #define HAMMER_PARSE_INTERNAL__H #include "../hammer.h" From 2cead893fc6dd31416cfd18f8755b71acaf477c4 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 22 Nov 2013 19:44:30 -0600 Subject: [PATCH 065/103] Allow in-place build via scons --in-place --- .gitignore | 1 + SConstruct | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 16fdda1..721dcf9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ Session.vim cscope.out build/ .sconsign.dblite +*.os *.pyc diff --git a/SConstruct b/SConstruct index c46c090..551f89e 100644 --- a/SConstruct +++ b/SConstruct @@ -53,7 +53,12 @@ AddOption("--coverage", action="store_true", help="Build with coverage instrumentation") -env['BUILDDIR'] = 'build/$VARIANT' +AddOption("--in-place", + dest="in_place", + default=False, + action="store_true", + help="Build in-place, rather than in the build/ tree") + dbg = env.Clone(VARIANT='debug') dbg.Append(CCFLAGS=['-g']) @@ -86,10 +91,16 @@ env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) Export('env') -env.SConscript(["src/SConscript"], variant_dir='build/$VARIANT/src') -env.SConscript(["examples/SConscript"], variant_dir='build/$VARIANT/examples') +if not GetOption("in_place"): + env['BUILD_BASE'] = 'build/$VARIANT' + env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') + env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples') +else: + env['BUILD_BASE'] = '.' + env.SConscript(["src/SConscript"]) + env.SConscript(["examples/SConscript"]) -env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/$VARIANT/src $SOURCE') +env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') env.Alias("install", "$libpath") env.Alias("install", "$incpath") From c6b4beefcb31c10926ea94a7c8ee5fdb9f2fa8ab Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 12:28:34 -0600 Subject: [PATCH 066/103] Make install_name darwin-only --- SConstruct | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index c46c090..b53947f 100644 --- a/SConstruct +++ b/SConstruct @@ -35,9 +35,10 @@ env.ScanReplace('libhammer.pc.in') env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes") -if not env['PLATFORM'] == 'darwin': +if env['PLATFORM'] == 'darwin': + env.Append(SHLINKFLAGS = ['-install_name', '$TARGET']) +else: env.MergeFlags("-lrt") - env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) AddOption("--variant", dest="variant", From e487d5435c89f93a1de0c65a604cfe7430002ba6 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 12:53:11 -0600 Subject: [PATCH 067/103] Added a void* user_data pointer to HAction --- examples/base64_sem1.c | 6 +++--- examples/base64_sem2.c | 2 +- examples/dns.c | 10 +++++----- examples/dns_common.c | 2 +- examples/dns_common.h | 2 +- examples/rr.c | 18 +++++++++--------- src/backends/llk.c | 4 ++-- src/backends/lr.c | 4 ++-- src/glue.c | 12 ++++++------ src/glue.h | 30 ++++++++++++++++++------------ src/hammer.h | 14 +++++++------- src/internal.h | 1 + src/parsers/action.c | 19 +++++++++++-------- src/parsers/bits.c | 16 ++++++---------- src/parsers/optional.c | 3 ++- src/parsers/sequence.c | 3 ++- src/parsers/token.c | 3 ++- src/t_parser.c | 7 ++++--- 18 files changed, 83 insertions(+), 73 deletions(-) diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c index 46e58da..4da171f 100644 --- a/examples/base64_sem1.c +++ b/examples/base64_sem1.c @@ -23,7 +23,7 @@ // They must be named act_. /// -HParsedToken *act_bsfdig(const HParseResult *p) +HParsedToken *act_bsfdig(const HParseResult *p, void* user_data) { HParsedToken *res = H_MAKE_UINT(0); @@ -54,7 +54,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0); #define act_document act_index0 // General-form action to turn a block of base64 digits into bytes. -HParsedToken *act_base64_n(int n, const HParseResult *p) +HParsedToken *act_base64_n(int n, const HParseResult *p, void* user_data) { HParsedToken *res = H_MAKE_SEQN(n); @@ -83,7 +83,7 @@ H_ACT_APPLY(act_base64_3, act_base64_n, 3); H_ACT_APPLY(act_base64_2, act_base64_n, 2); H_ACT_APPLY(act_base64_1, act_base64_n, 1); -HParsedToken *act_base64(const HParseResult *p) +HParsedToken *act_base64(const HParseResult *p, void* user_data) { assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used == 2); diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c index 84155da..b7a2263 100644 --- a/examples/base64_sem2.c +++ b/examples/base64_sem2.c @@ -49,7 +49,7 @@ uint8_t bsfdig_value(const HParsedToken *p) // helper: append a byte value to a sequence #define seq_append_byte(res, b) h_seq_snoc(res, H_MAKE_UINT(b)) -HParsedToken *act_base64(const HParseResult *p) +HParsedToken *act_base64(const HParseResult *p, void* user_data) { assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used == 2); diff --git a/examples/dns.c b/examples/dns.c index 3f730b9..5effb89 100644 --- a/examples/dns.c +++ b/examples/dns.c @@ -86,7 +86,7 @@ void set_rdata(struct dns_rr *rr, HCountedArray *rdata) { } } -HParsedToken* act_header(const HParseResult *p) { +HParsedToken* act_header(const HParseResult *p, void* user_data) { HParsedToken **fields = h_seq_elements(p->ast); dns_header_t header_ = { .id = H_CAST_UINT(fields[0]), @@ -109,7 +109,7 @@ HParsedToken* act_header(const HParseResult *p) { return H_MAKE(dns_header_t, header); } -HParsedToken* act_label(const HParseResult *p) { +HParsedToken* act_label(const HParseResult *p, void* user_data) { dns_label_t *r = H_ALLOC(dns_label_t); r->len = h_seq_len(p->ast); @@ -121,7 +121,7 @@ HParsedToken* act_label(const HParseResult *p) { return H_MAKE(dns_label_t, r); } -HParsedToken* act_rr(const HParseResult *p) { +HParsedToken* act_rr(const HParseResult *p, void* user_data) { dns_rr_t *rr = H_ALLOC(dns_rr_t); rr->name = *H_FIELD(dns_domain_t, 0); @@ -136,7 +136,7 @@ HParsedToken* act_rr(const HParseResult *p) { return H_MAKE(dns_rr_t, rr); } -HParsedToken* act_question(const HParseResult *p) { +HParsedToken* act_question(const HParseResult *p, void* user_data) { dns_question_t *q = H_ALLOC(dns_question_t); HParsedToken **fields = h_seq_elements(p->ast); @@ -153,7 +153,7 @@ HParsedToken* act_question(const HParseResult *p) { return H_MAKE(dns_question_t, q); } -HParsedToken* act_message(const HParseResult *p) { +HParsedToken* act_message(const HParseResult *p, void* user_data) { h_pprint(stdout, p->ast, 0, 2); dns_message_t *msg = H_ALLOC(dns_message_t); diff --git a/examples/dns_common.c b/examples/dns_common.c index 01dd8f0..3cdd04e 100644 --- a/examples/dns_common.c +++ b/examples/dns_common.c @@ -18,7 +18,7 @@ bool validate_label(HParseResult *p) { #define act_label h_act_flatten -HParsedToken* act_domain(const HParseResult *p) { +HParsedToken* act_domain(const HParseResult *p, void* user_data) { HParsedToken *ret = NULL; char *arr = NULL; diff --git a/examples/dns_common.h b/examples/dns_common.h index 8af014b..3e30770 100644 --- a/examples/dns_common.h +++ b/examples/dns_common.h @@ -7,6 +7,6 @@ HParser* init_domain(); HParser* init_character_string(); -HParsedToken* act_index0(const HParseResult *p); +HParsedToken* act_index0(const HParseResult *p, void* user_data); #endif diff --git a/examples/rr.c b/examples/rr.c index dd25063..4a7c4be 100644 --- a/examples/rr.c +++ b/examples/rr.c @@ -17,7 +17,7 @@ bool validate_null(HParseResult *p) { return (65536 > p->ast->seq->used); } -HParsedToken *act_null(const HParseResult *p) { +HParsedToken *act_null(const HParseResult *p, void* user_data) { dns_rr_null_t *null = H_ALLOC(dns_rr_null_t); size_t len = h_seq_len(p->ast); @@ -28,7 +28,7 @@ HParsedToken *act_null(const HParseResult *p) { return H_MAKE(dns_rr_null_t, null); } -HParsedToken *act_txt(const HParseResult *p) { +HParsedToken *act_txt(const HParseResult *p, void* user_data) { dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t); const HCountedArray *arr = H_CAST_SEQ(p->ast); @@ -47,7 +47,7 @@ HParsedToken *act_txt(const HParseResult *p) { return H_MAKE(dns_rr_txt_t, txt); } -HParsedToken* act_cstr(const HParseResult *p) { +HParsedToken* act_cstr(const HParseResult *p, void* user_data) { dns_cstr_t *cs = H_ALLOC(dns_cstr_t); const HCountedArray *arr = H_CAST_SEQ(p->ast); @@ -60,7 +60,7 @@ HParsedToken* act_cstr(const HParseResult *p) { return H_MAKE(dns_cstr_t, cs); } -HParsedToken* act_soa(const HParseResult *p) { +HParsedToken* act_soa(const HParseResult *p, void* user_data) { dns_rr_soa_t *soa = H_ALLOC(dns_rr_soa_t); soa->mname = *H_FIELD(dns_domain_t, 0); @@ -74,7 +74,7 @@ HParsedToken* act_soa(const HParseResult *p) { return H_MAKE(dns_rr_soa_t, soa); } -HParsedToken* act_wks(const HParseResult *p) { +HParsedToken* act_wks(const HParseResult *p, void* user_data) { dns_rr_wks_t *wks = H_ALLOC(dns_rr_wks_t); wks->address = H_FIELD_UINT(0); @@ -87,7 +87,7 @@ HParsedToken* act_wks(const HParseResult *p) { return H_MAKE(dns_rr_wks_t, wks); } -HParsedToken* act_hinfo(const HParseResult *p) { +HParsedToken* act_hinfo(const HParseResult *p, void* user_data) { dns_rr_hinfo_t *hinfo = H_ALLOC(dns_rr_hinfo_t); hinfo->cpu = *H_FIELD(dns_cstr_t, 0); @@ -96,7 +96,7 @@ HParsedToken* act_hinfo(const HParseResult *p) { return H_MAKE(dns_rr_hinfo_t, hinfo); } -HParsedToken* act_minfo(const HParseResult *p) { +HParsedToken* act_minfo(const HParseResult *p, void* user_data) { dns_rr_minfo_t *minfo = H_ALLOC(dns_rr_minfo_t); minfo->rmailbx = *H_FIELD(dns_domain_t, 0); @@ -105,7 +105,7 @@ HParsedToken* act_minfo(const HParseResult *p) { return H_MAKE(dns_rr_minfo_t, minfo); } -HParsedToken* act_mx(const HParseResult *p) { +HParsedToken* act_mx(const HParseResult *p, void* user_data) { dns_rr_mx_t *mx = H_ALLOC(dns_rr_mx_t); mx->preference = H_FIELD_UINT(0); @@ -184,7 +184,7 @@ HParser* init_rdata(uint16_t type) { for(uint16_t i = 0; ireshape) - tok = (HParsedToken *)x->reshape(make_result(arena, tok)); + tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data); // call validation and semantic action, if present if(x->pred && !x->pred(make_result(tarena, tok))) goto no_parse; // validation failed -> no parse if(x->action) - tok = (HParsedToken *)x->action(make_result(arena, tok)); + tok = (HParsedToken *)x->action(make_result(arena, tok), x->user_data); // append to result sequence h_carray_append(seq, tok); diff --git a/src/backends/lr.c b/src/backends/lr.c index d258e8a..3739ec9 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -307,13 +307,13 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // perform token reshape if indicated if(symbol->reshape) - value = (HParsedToken *)symbol->reshape(make_result(arena, value)); + value = (HParsedToken *)symbol->reshape(make_result(arena, value), symbol->user_data); // call validation and semantic action, if present if(symbol->pred && !symbol->pred(make_result(tarena, value))) return false; // validation failed -> no parse; terminate if(symbol->action) - value = (HParsedToken *)symbol->action(make_result(arena, value)); + value = (HParsedToken *)symbol->action(make_result(arena, value), symbol->user_data); // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we diff --git a/src/glue.c b/src/glue.c index 5e3804d..c2d915a 100644 --- a/src/glue.c +++ b/src/glue.c @@ -5,7 +5,7 @@ #include "parsers/parser_internal.h" // Helper to build HAction's that pick one index out of a sequence. -HParsedToken *h_act_index(int i, const HParseResult *p) +HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data) { if(!p) return NULL; @@ -23,7 +23,7 @@ HParsedToken *h_act_index(int i, const HParseResult *p) return tok->seq->elements[i]; } -HParsedToken *h_act_first(const HParseResult *p) { +HParsedToken *h_act_first(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -31,7 +31,7 @@ HParsedToken *h_act_first(const HParseResult *p) { return p->ast->seq->elements[0]; } -HParsedToken *h_act_second(const HParseResult *p) { +HParsedToken *h_act_second(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -39,7 +39,7 @@ HParsedToken *h_act_second(const HParseResult *p) { return p->ast->seq->elements[1]; } -HParsedToken *h_act_last(const HParseResult *p) { +HParsedToken *h_act_last(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -59,7 +59,7 @@ static void act_flatten_(HCountedArray *seq, const HParsedToken *tok) { } } -HParsedToken *h_act_flatten(const HParseResult *p) { +HParsedToken *h_act_flatten(const HParseResult *p, void* user_data) { HCountedArray *seq = h_carray_new(p->arena); act_flatten_(seq, p->ast); @@ -72,7 +72,7 @@ HParsedToken *h_act_flatten(const HParseResult *p) { return res; } -HParsedToken *h_act_ignore(const HParseResult *p) { +HParsedToken *h_act_ignore(const HParseResult *p, void* user_data) { return NULL; } diff --git a/src/glue.h b/src/glue.h index 54b5985..7486e46 100644 --- a/src/glue.h +++ b/src/glue.h @@ -56,13 +56,19 @@ #define H_RULE(rule, def) HParser *rule = def -#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule) +#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) #define H_VRULE(rule, def) HParser *rule = \ - h_attr_bool(def, validate_ ## rule) + h_attr_bool(def, validate_ ## rule) #define H_VARULE(rule, def) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule) #define H_AVRULE(rule, def) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule) + h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, NULL) +#define H_ADRULE(rule, def, data) HParser *rule = \ + h_action(def, act_ ## rule, data) +#define H_VADRULE(rule, def, data) HParser *rule = \ + h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule) +#define H_AVDRULE(rule, def, data) HParser *rule = \ + h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, data) // @@ -88,18 +94,18 @@ // action such as h_act_index. // -HParsedToken *h_act_index(int i, const HParseResult *p); -HParsedToken *h_act_first(const HParseResult *p); -HParsedToken *h_act_second(const HParseResult *p); -HParsedToken *h_act_last(const HParseResult *p); -HParsedToken *h_act_flatten(const HParseResult *p); -HParsedToken *h_act_ignore(const HParseResult *p); +HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data); +HParsedToken *h_act_first(const HParseResult *p, void* user_data); +HParsedToken *h_act_second(const HParseResult *p, void* user_data); +HParsedToken *h_act_last(const HParseResult *p, void* user_data); +HParsedToken *h_act_flatten(const HParseResult *p, void* user_data); +HParsedToken *h_act_ignore(const HParseResult *p, void* user_data); // Define 'myaction' as a specialization of 'paction' by supplying the leading // parameters. #define H_ACT_APPLY(myaction, paction, ...) \ - HParsedToken *myaction(const HParseResult *p) { \ - return paction(__VA_ARGS__, p); \ + HParsedToken *myaction(const HParseResult *p, void* user_data) { \ + return paction(__VA_ARGS__, p, user_data); \ } diff --git a/src/hammer.h b/src/hammer.h index 0175142..bd3ae02 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -113,7 +113,7 @@ typedef struct HBitWriter_ HBitWriter; * say, structs) and stuff values for them into the void* in the * tagged union in HParsedToken. */ -typedef HParsedToken* (*HAction)(const HParseResult *p); +typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data); /** * Type of a boolean attribute-checking function, used in the @@ -349,7 +349,7 @@ HAMMER_FN_DECL(HParser*, h_middle, const HParser* p, const HParser* x, const HPa * * Result token type: any */ -HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a); +HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a, void* user_data); /** * Parse a single character in the given charset. @@ -621,11 +621,11 @@ void h_bit_writer_free(HBitWriter* w); // General-purpose actions for use with h_action // XXX to be consolidated with glue.h when merged upstream -HParsedToken *h_act_first(const HParseResult *p); -HParsedToken *h_act_second(const HParseResult *p); -HParsedToken *h_act_last(const HParseResult *p); -HParsedToken *h_act_flatten(const HParseResult *p); -HParsedToken *h_act_ignore(const HParseResult *p); +HParsedToken *h_act_first(const HParseResult *p, void* userdata); +HParsedToken *h_act_second(const HParseResult *p, void* userdata); +HParsedToken *h_act_last(const HParseResult *p, void* userdata); +HParsedToken *h_act_flatten(const HParseResult *p, void* userdata); +HParsedToken *h_act_ignore(const HParseResult *p, void* userdata); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); diff --git a/src/internal.h b/src/internal.h index 89cb380..3c639d0 100644 --- a/src/internal.h +++ b/src/internal.h @@ -365,6 +365,7 @@ struct HCFChoice_ { // to execute before action and pred are applied. HAction action; HPredicate pred; + void* user_data; }; struct HCFSequence_ { diff --git a/src/parsers/action.c b/src/parsers/action.c index 52c9bc1..e254a89 100644 --- a/src/parsers/action.c +++ b/src/parsers/action.c @@ -4,6 +4,7 @@ typedef struct { const HParser *p; HAction action; + void* user_data; } HParseAction; static HParseResult* parse_action(void *env, HParseState *state) { @@ -12,8 +13,8 @@ static HParseResult* parse_action(void *env, HParseState *state) { HParseResult *tmp = h_do_parse(a->p, state); //HParsedToken *tok = a->action(h_do_parse(a->p, state)); if(tmp) { - const HParsedToken *tok = a->action(tmp); - return make_result(state->arena, (HParsedToken*)tok); + const HParsedToken *tok = a->action(tmp, a->user_data); + return make_result(state->arena, (HParsedToken*)tok); } else return NULL; } else // either the parser's missing or the action's missing @@ -27,6 +28,7 @@ static void desugar_action(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_BEGIN_SEQ() { HCFS_DESUGAR(a->p); } HCFS_END_SEQ(); + HCFS_THIS_CHOICE->user_data = a->user_data; HCFS_THIS_CHOICE->action = a->action; HCFS_THIS_CHOICE->reshape = h_act_first; } HCFS_END_CHOICE(); @@ -44,7 +46,7 @@ static bool action_isValidCF(void *env) { static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) { HParseResult res; - HAction action = arg; + HParseAction *a = arg; assert(ctx->stack_count >= 1); if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) { assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK); @@ -56,7 +58,7 @@ static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) { } res.arena = arena; - HParsedToken *tok = action(&res); + HParsedToken *tok = a->action(&res, a->user_data); if (tok != NULL) ctx->stack[ctx->stack_count-1] = tok; else @@ -69,7 +71,7 @@ static bool action_ctrvm(HRVMProg *prog, void* env) { h_rvm_insert_insn(prog, RVM_PUSH, 0); if (!h_compile_regex(prog, a->p)) return false; - h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a->action)); + h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a)); return true; } @@ -81,13 +83,14 @@ static const HParserVtable action_vt = { .compile_to_rvm = action_ctrvm, }; -HParser* h_action(const HParser* p, const HAction a) { - return h_action__m(&system_allocator, p, a); +HParser* h_action(const HParser* p, const HAction a, void* user_data) { + return h_action__m(&system_allocator, p, a, user_data); } -HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a) { +HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a, void* user_data) { HParseAction *env = h_new(HParseAction, 1); env->p = p; env->action = a; + env->user_data = user_data; return h_new_parser(mm__, &action_vt, env); } diff --git a/src/parsers/bits.c b/src/parsers/bits.c index 93b4aef..716524c 100644 --- a/src/parsers/bits.c +++ b/src/parsers/bits.c @@ -17,7 +17,9 @@ static HParseResult* parse_bits(void* env, HParseState *state) { return make_result(state->arena, result); } -static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) { +static HParsedToken *reshape_bits(const HParseResult *p, void* signedp_p) { + // signedp == NULL iff unsigned + bool signedp = (signedp_p != NULL); // XXX works only for whole bytes // XXX assumes big-endian assert(p->ast); @@ -45,12 +47,6 @@ static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) { return ret; } -static HParsedToken *reshape_bits_unsigned(const HParseResult *p) { - return reshape_bits(p, false); -} -static HParsedToken *reshape_bits_signed(const HParseResult *p) { - return reshape_bits(p, true); -} static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) { struct bits_env *bits = (struct bits_env*)env; @@ -67,9 +63,9 @@ static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_ADD_CHARSET(match_all); } } HCFS_END_SEQ(); - HCFS_THIS_CHOICE->reshape = bits->signedp - ? reshape_bits_signed - : reshape_bits_unsigned; + HCFS_THIS_CHOICE->reshape = reshape_bits; + HCFS_THIS_CHOICE->user_data = bits->signedp ? HCFS_THIS_CHOICE : NULL; // HCFS_THIS_CHOICE is an arbitrary non-null pointer + } HCFS_END_CHOICE(); } diff --git a/src/parsers/optional.c b/src/parsers/optional.c index c4282a9..ff9fc15 100644 --- a/src/parsers/optional.c +++ b/src/parsers/optional.c @@ -22,7 +22,7 @@ static bool opt_isValidCF(void *env) { return p->vtable->isValidCF(p->env); } -static HParsedToken* reshape_optional(const HParseResult *p) { +static HParsedToken* reshape_optional(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -52,6 +52,7 @@ static void desugar_optional(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_BEGIN_SEQ() { } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_optional; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index bdea8c0..42c0913 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -43,7 +43,7 @@ static bool sequence_isValidCF(void *env) { return true; } -static HParsedToken *reshape_sequence(const HParseResult *p) { +static HParsedToken *reshape_sequence(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -72,6 +72,7 @@ static void desugar_sequence(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_DESUGAR(s->p_array[i]); } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_sequence; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..0a43f8d 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -20,7 +20,7 @@ static HParseResult* parse_token(void *env, HParseState *state) { } -static HParsedToken *reshape_token(const HParseResult *p) { +static HParsedToken *reshape_token(const HParseResult *p, void* user_data) { // fetch sequence of uints from p assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -52,6 +52,7 @@ static void desugar_token(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_ADD_CHAR(tok->str[i]); } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_token; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/t_parser.c b/src/t_parser.c index bba0148..292d1c4 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -162,7 +162,7 @@ static void test_middle(gconstpointer backend) { #include -HParsedToken* upcase(const HParseResult *p) { +HParsedToken* upcase(const HParseResult *p, void* user_data) { switch(p->ast->token_type) { case TT_SEQUENCE: { @@ -202,7 +202,8 @@ static void test_action(gconstpointer backend) { h_ch('B'), NULL), NULL), - upcase); + upcase, + NULL); g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x41 u0x42)"); g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "AB", 2, "(u0x41 u0x42)"); @@ -433,7 +434,7 @@ static void test_ambiguous(gconstpointer backend) { HParser *p_ = h_ch('+'); HParser *E_ = h_indirect(); h_bind_indirect(E_, h_choice(h_sequence(E_, p_, E_, NULL), d_, NULL)); - HParser *expr_ = h_action(E_, h_act_flatten); + HParser *expr_ = h_action(E_, h_act_flatten, NULL); g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "(u0x64)"); g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)"); From 69e84bcbb047c0de92d2595742d3dfa337a940d3 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 13:01:55 -0600 Subject: [PATCH 068/103] Added a void* user_data pointer to HPredicate --- examples/dns.c | 4 ++-- examples/dns_common.c | 2 +- examples/rr.c | 2 +- src/backends/llk.c | 2 +- src/backends/lr.c | 2 +- src/glue.h | 12 +++++++----- src/hammer.h | 4 ++-- src/parsers/attr_bool.c | 17 ++++++++++------- src/t_parser.c | 5 +++-- 9 files changed, 28 insertions(+), 22 deletions(-) diff --git a/examples/dns.c b/examples/dns.c index 5effb89..63df3a4 100644 --- a/examples/dns.c +++ b/examples/dns.c @@ -15,7 +15,7 @@ // Validations /// -bool validate_hdzero(HParseResult *p) { +bool validate_hdzero(HParseResult *p, void* user_data) { if (TT_UINT != p->ast->token_type) return false; return (0 == p->ast->uint); @@ -25,7 +25,7 @@ bool validate_hdzero(HParseResult *p) { * Every DNS message should have QDCOUNT entries in the question * section, and ANCOUNT+NSCOUNT+ARCOUNT resource records. */ -bool validate_message(HParseResult *p) { +bool validate_message(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; diff --git a/examples/dns_common.c b/examples/dns_common.c index 3cdd04e..bf934d6 100644 --- a/examples/dns_common.c +++ b/examples/dns_common.c @@ -10,7 +10,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0) /** * A label can't be more than 63 characters. */ -bool validate_label(HParseResult *p) { +bool validate_label(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; return (64 > p->ast->seq->used); diff --git a/examples/rr.c b/examples/rr.c index 4a7c4be..c179922 100644 --- a/examples/rr.c +++ b/examples/rr.c @@ -11,7 +11,7 @@ // Validations and Semantic Actions /// -bool validate_null(HParseResult *p) { +bool validate_null(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; return (65536 > p->ast->seq->used); diff --git a/src/backends/llk.c b/src/backends/llk.c index 39ac07a..2bc39da 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -375,7 +375,7 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data); // call validation and semantic action, if present - if(x->pred && !x->pred(make_result(tarena, tok))) + if(x->pred && !x->pred(make_result(tarena, tok), x->user_data)) goto no_parse; // validation failed -> no parse if(x->action) tok = (HParsedToken *)x->action(make_result(arena, tok), x->user_data); diff --git a/src/backends/lr.c b/src/backends/lr.c index 3739ec9..e7f2377 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -310,7 +310,7 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) value = (HParsedToken *)symbol->reshape(make_result(arena, value), symbol->user_data); // call validation and semantic action, if present - if(symbol->pred && !symbol->pred(make_result(tarena, value))) + if(symbol->pred && !symbol->pred(make_result(tarena, value), symbol->user_data)) return false; // validation failed -> no parse; terminate if(symbol->action) value = (HParsedToken *)symbol->action(make_result(arena, value), symbol->user_data); diff --git a/src/glue.h b/src/glue.h index 7486e46..74963b0 100644 --- a/src/glue.h +++ b/src/glue.h @@ -58,17 +58,19 @@ #define H_RULE(rule, def) HParser *rule = def #define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) #define H_VRULE(rule, def) HParser *rule = \ - h_attr_bool(def, validate_ ## rule) + h_attr_bool(def, validate_ ## rule, NULL) #define H_VARULE(rule, def) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule, NULL) #define H_AVRULE(rule, def) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, NULL) + h_action(h_attr_bool(def, validate_ ## rule, NULL), act_ ## rule, NULL) #define H_ADRULE(rule, def, data) HParser *rule = \ h_action(def, act_ ## rule, data) +#define H_VDRULE(rule, def, data) HParser *rule = \ + h_attr_bool(def, validate_ ## rule, data) #define H_VADRULE(rule, def, data) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule, data) #define H_AVDRULE(rule, def, data) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, data) + h_action(h_attr_bool(def, validate_ ## rule, data), act_ ## rule, data) // diff --git a/src/hammer.h b/src/hammer.h index bd3ae02..541e38d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -120,7 +120,7 @@ typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data); * attr_bool() parser. It can be any (user-defined) function that takes * a HParseResult* and returns true or false. */ -typedef bool (*HPredicate)(HParseResult *p); +typedef bool (*HPredicate)(HParseResult *p, void* user_data); typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; @@ -515,7 +515,7 @@ HAMMER_FN_DECL(HParser*, h_length_value, const HParser* length, const HParser* v * * Result token type: p's result type if pred succeeded, NULL otherwise. */ -HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred); +HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred, void* user_data); /** * The 'and' parser asserts that a conditional syntax is satisfied, diff --git a/src/parsers/attr_bool.c b/src/parsers/attr_bool.c index fc980b2..e8359ab 100644 --- a/src/parsers/attr_bool.c +++ b/src/parsers/attr_bool.c @@ -4,13 +4,14 @@ typedef struct { const HParser *p; HPredicate pred; + void* user_data; } HAttrBool; static HParseResult* parse_attr_bool(void *env, HParseState *state) { HAttrBool *a = (HAttrBool*)env; HParseResult *res = h_do_parse(a->p, state); if (res && res->ast) { - if (a->pred(res)) + if (a->pred(res, a->user_data)) return res; else return NULL; @@ -42,12 +43,13 @@ static void desugar_ab(HAllocator *mm__, HCFStack *stk__, void *env) { } HCFS_END_SEQ(); HCFS_THIS_CHOICE->pred = a->pred; HCFS_THIS_CHOICE->reshape = h_act_first; + HCFS_THIS_CHOICE->user_data = a->user_data; } HCFS_END_CHOICE(); } static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) { HParseResult res; - HPredicate pred = arg; + HAttrBool *ab = arg; assert(ctx->stack_count >= 1); if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) { assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK); @@ -59,7 +61,7 @@ static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) { res.ast = NULL; } res.arena = arena; - return pred(&res); + return ab->pred(&res, ab->user_data); } static bool ab_ctrvm(HRVMProg *prog, void *env) { @@ -67,7 +69,7 @@ static bool ab_ctrvm(HRVMProg *prog, void *env) { h_rvm_insert_insn(prog, RVM_PUSH, 0); if (!h_compile_regex(prog, ab->p)) return false; - h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab->pred)); + h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab)); return true; } @@ -80,12 +82,13 @@ static const HParserVtable attr_bool_vt = { }; -HParser* h_attr_bool(const HParser* p, HPredicate pred) { - return h_attr_bool__m(&system_allocator, p, pred); +HParser* h_attr_bool(const HParser* p, HPredicate pred, void* user_data) { + return h_attr_bool__m(&system_allocator, p, pred, user_data); } -HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred) { +HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred, void* user_data) { HAttrBool *env = h_new(HAttrBool, 1); env->p = p; env->pred = pred; + env->user_data = user_data; return h_new_parser(mm__, &attr_bool_vt, env); } diff --git a/src/t_parser.c b/src/t_parser.c index 292d1c4..12edba9 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -365,7 +365,7 @@ static void test_epsilon_p(gconstpointer backend) { g_check_parse_match(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)"); } -bool validate_test_ab(HParseResult *p) { +bool validate_test_ab(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; if (TT_UINT != p->ast->seq->elements[0]->token_type) @@ -377,7 +377,8 @@ bool validate_test_ab(HParseResult *p) { static void test_attr_bool(gconstpointer backend) { const HParser *ab_ = h_attr_bool(h_many1(h_choice(h_ch('a'), h_ch('b'), NULL)), - validate_test_ab); + validate_test_ab, + NULL); g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)"); g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "bb", 2, "(u0x62 u0x62)"); From 890c6383a78f469429aeddf5c3f02af318cf103f Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 13:09:39 -0600 Subject: [PATCH 069/103] Added documentation for H_{A,V,VA,AV}DRULE --- src/glue.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/glue.h b/src/glue.h index 74963b0..1fe6ce4 100644 --- a/src/glue.h +++ b/src/glue.h @@ -53,7 +53,10 @@ // H_AVRULE is like H_VARULE but the action is attached outside the validation, // i.e. the validation receives the uninterpreted AST as input. // - +// H_ADRULE, H_VDRULE, H_AVDRULE, and H_VADRULE are the same as the +// equivalent non-D variants, except that they also allow you to uset +// the user_data pointer. In cases where both an attr_bool and an +// action are used, the same userdata pointer is given to both. #define H_RULE(rule, def) HParser *rule = def #define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) From de6f6115a80db68d5d5b887f12d7b8cec1e19e23 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 070/103] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 541e38d..08819f5 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -126,6 +126,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -586,7 +587,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From 28bee303f9d1885c69b0428548ca89e528d6e201 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 071/103] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From 9d43244c2e50dd3c92df2ad8b46a885cf340c272 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 072/103] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 0a43f8d..d36ec54 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -80,6 +80,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From a31f2d31a4f3b5ebae5df16aa254c5a12689d9b9 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 073/103] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From 1841c9d77ecc006046c164293f3ba67c17cad161 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 074/103] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From 75453d8b2f88336131e31130a89fc25c327cb421 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 075/103] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 49f22e2afffe70c93f5cc8ecc86b5a8039f5dac5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 076/103] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index 08819f5..02e4085 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -29,7 +29,9 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H typedef struct HParseState_ HParseState; @@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From 8f427b5a6d4d9b8b36552c51f20ff1cebd3a88d2 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 077/103] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 02e4085..c8a2bf3 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -70,8 +70,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -81,6 +94,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -144,12 +160,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From 0e4253d0efc2f13b43e35f28e51e68def7a4b763 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 078/103] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From 73eabc7bf9e5eaccaa3175a16c331d7c697c01c8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 079/103] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index c8a2bf3..2aded7a 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p, void* userdata); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} // {{{ Token type registry From ada7bf89ee61d12b2f624cb6e82168f78d647379 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 080/103] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From 77d48e2b7c939e9066e8a899cce47aaeec0c7ed1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 081/103] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 1d9ca76..0cb41af 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From dc9124724af7ffe4b837e6777c96080fe2d0b4d7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 082/103] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 246a3d6..f23f01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 27f94dbe610e7c30573c741a3a1be09b360e95a7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 083/103] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From 4bd2fc9e72f4149b9398d827c15ed3d4e7eb2a33 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 084/103] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From eba8ecc6c39f38efaebd76aba206cc98f5936164 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 085/103] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From 073d6d8a275ad4826b0f6a301f63b45920ca6efb Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 086/103] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From f685f9ea4ed8ad14178ad7f48b5237171c94162b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 087/103] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From 3228a86b90682115decf12be64338be3582283b3 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 088/103] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From 96760f1dafe792105cf18cc3dae1ef03b21fae39 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 089/103] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From 0aac5f4622d52b9df8f01769a6a6d81a63baceb1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 090/103] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f23f01c..0e406ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 12bb5cac36c5225d09bdd029147ea46eb1337133 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 091/103] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 8b6d6084c5ed15630d19a059e037525193835ef8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:00:58 -0600 Subject: [PATCH 092/103] fixed the segfault! hand-initialized HParser needed PB_MIN set. --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass From 954b6e32d3757fd8d1cf7bffc76ea4018ebcbb4c Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 22 Nov 2013 19:42:02 -0600 Subject: [PATCH 093/103] Changed typemap for strings to also parse lengths --- src/bindings/python/hammer_tests.py | 341 ++++++++++++++-------------- src/bindings/swig/hammer.i | 25 +- 2 files changed, 195 insertions(+), 171 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 41cd13f..a5967e6 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,11 +4,11 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2", 3) + cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95", 2), None) + self.assertEqual(h.h_parse(self.parser, "95"), None) class TestChParser(unittest.TestCase): @classmethod @@ -16,201 +16,202 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\xfe"), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a", 2), None) + self.assertEqual(h.h_parse(self.parser, "_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x", 3), None) + self.assertEqual(h.h_parse(self.parser, " x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ba", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, " a", 2), None) - self.assertEqual(h.h_parse(self.parser, "a ", 2), None) - self.assertEqual(h.h_parse(self.parser, " b ", 3), None) - self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) - self.assertEqual(h.h_parse(self.parser, " ab", 3), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, " a"), None) + self.assertEqual(h.h_parse(self.parser, "a "), None) + self.assertEqual(h.h_parse(self.parser, " b "), None) + self.assertEqual(h.h_parse(self.parser, "ba "), None) + self.assertEqual(h.h_parse(self.parser, " ab"), None) -# class TestAction(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) -# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +@unittest.skip("Action not implemented yet") +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc", 3) + cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc", 3) + cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2), None) + self.assertEqual(h.h_parse(self.parser, "aa"), None) class TestNothingP(unittest.TestCase): @classmethod @@ -219,243 +220,243 @@ class TestNothingP(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, "b", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, "b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c", 4), None) + self.assertEqual(h.h_parse(self.parser, "a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c", 1), None) + self.assertEqual(h.h_parse(self.parser, "c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6", 1), None) + self.assertEqual(h.h_parse(self.parser, "6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) + cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5", 1), None) - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "5"), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) + self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef", 4), None) - self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) + self.assertEqual(h.h_parse(self.parser, "adef"), None) + self.assertEqual(h.h_parse(self.parser, "dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed", 3), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "aed"), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass # class TestAttrBool(unittest.TestCase): # @classmethod # def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) # def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + self.assertEqual(h.h_parse(self.parser, "0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) + self.assertEqual(h.h_parse(self.parser, "a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "a+b").ast.token_data.seq tree[1] = tree[1].token_data.seq[0] self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree = h.h_parse(self.parser, "a++b").ast.token_data.seq tree[0] = chr(tree[0].token_data.uint) tree[1] = tree[1].token_data.bytes tree[2] = chr(tree[2].token_data.uint) @@ -471,9 +472,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -482,14 +483,14 @@ class TestRightrec(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_indirect() a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + tree = h.h_parse(self.parser, "a").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) @@ -506,9 +507,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81a9dd1..4fb30c5 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -14,13 +14,35 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } + +%typemap(newfree) HParseResult* { + h_parse_result_free($1); + } + +%newobject h_parse +%delobject h_parse_result_free + + /* +%typemap(in) (uint8_t* str, size_t len) { + if (PyString_Check($input) || + PyUnicode_Check($input)) { + PyString_AsStringAndSize($input, (char**)&$1, &$2); + } else { + PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); + } + } +*/ +%apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} +%apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); int size = PyList_Size($input); int i = 0; int res = 0; - $1 = (void**)malloc(size*sizeof(HParser*)); + $1 = (void**)malloc((size+1)*sizeof(HParser*)); for (i=0; i Date: Sat, 23 Nov 2013 16:40:57 -0600 Subject: [PATCH 094/103] Got rid of accessors in Python; made h_action work --- src/bindings/python/hammer.py | 488 ---------------------------- src/bindings/python/hammer_tests.py | 181 +++++------ src/bindings/swig/hammer.i | 134 ++++++-- 3 files changed, 194 insertions(+), 609 deletions(-) delete mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py deleted file mode 100644 index 36b78c8..0000000 --- a/src/bindings/python/hammer.py +++ /dev/null @@ -1,488 +0,0 @@ -from cffi import FFI -import threading -import sys - -_ffi = FFI() - -# {{{ Types - -_ffi.cdef("typedef struct HAllocator_ HAllocator;") -_ffi.cdef("typedef struct HArena_ HArena;") -_ffi.cdef("typedef int bool;") -_ffi.cdef("typedef struct HParseState_ HParseState;") -_ffi.cdef(""" -typedef enum HParserBackend_ { - PB_MIN = 0, - PB_PACKRAT = 0, // PB_MIN is always the default. - PB_REGULAR, - PB_LLk, - PB_LALR, - PB_GLR -// TODO: support PB_MAX -} HParserBackend; -""") -_ffi.cdef(""" -typedef enum HTokenType_ { - // Before you change the explicit values of these, think of the poor bindings ;_; - TT_NONE = 1, - TT_BYTES = 2, - TT_SINT = 4, - TT_UINT = 8, - TT_SEQUENCE = 16, - TT_RESERVED_1, // reserved for backend-specific internal use - TT_ERR = 32, - TT_USER = 64, - TT_MAX -} HTokenType; -""") -_ffi.cdef(""" -typedef struct HCountedArray_ { - size_t capacity; - size_t used; - HArena * arena; - struct HParsedToken_ **elements; -} HCountedArray; -""") -_ffi.cdef(""" -typedef struct HBytes_ { - const uint8_t *token; - size_t len; -} HBytes; -""") -_ffi.cdef(""" -typedef struct HParsedToken_ { - HTokenType token_type; - union { - HBytes bytes; - int64_t sint; - uint64_t uint; - double dbl; - float flt; - HCountedArray *seq; // a sequence of HParsedToken's - void *user; - }; - size_t index; - char bit_offset; -} HParsedToken; -""") -_ffi.cdef(""" -typedef struct HParseResult_ { - const HParsedToken *ast; - long long bit_length; - HArena * arena; -} HParseResult; -""") - -_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -_ffi.cdef(""" -typedef struct HCFChoice_ HCFChoice; -typedef struct HRVMProg_ HRVMProg; -typedef struct HParserVtable_ HParserVtable; -""") - -_ffi.cdef("typedef struct HParser_ HParser;") -_ffi.cdef(""" -typedef struct HParserTestcase_ { - unsigned char* input; - size_t length; - char* output_unambiguous; -} HParserTestcase; - -typedef struct HCaseResult_ { - bool success; - union { - const char* actual_results; // on failure, filled in with the results of h_write_result_unamb - size_t parse_time; // on success, filled in with time for a single parse, in nsec - }; -} HCaseResult; - -typedef struct HBackendResults_ { - HParserBackend backend; - bool compile_success; - size_t n_testcases; - size_t failed_testcases; // actually a count... - HCaseResult *cases; -} HBackendResults; - -typedef struct HBenchmarkResults_ { - size_t len; - HBackendResults *results; -} HBenchmarkResults; -""") - -# }}} -# {{{ Arena functions -_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") -_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -# }}} -# {{{ cdefs -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' -_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_ch(const uint8_t c);") -_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_bits(size_t len, bool sign);") -_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -_ffi.cdef("HParser* h_int64(void);") -_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int32(void);") -_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int16(void);") -_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int8(void);") -_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint64(void);") -_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint32(void);") -_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint16(void);") -_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint8(void);") -_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_whitespace(const HParser* p);") -_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_end_p(void);") -_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_nothing_p(void);") -_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_sequence(HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__a(void* args);") -_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_choice(HParser* p, ...);") -_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_choice__a(void* args);") -_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_many(const HParser* p);") -_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_many1(const HParser* p);") -_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_optional(const HParser* p);") -_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_ignore(const HParser* p);") -_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_epsilon_p(void);") -_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_and(const HParser* p);") -_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_not(const HParser* p);") -_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_indirect(void);") -_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_parse_result_free(HParseResult *result);") -_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") - -_lib = _ffi.verify("#include ", - libraries=['hammer']) - -_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# }}} -class _DynamicScopeHolder(threading.local): - """A dynamically-scoped holder of python objects, which may or may not - otherwise appear in the object graph. Intended for use with CFFI """ - def __init__(self): - self._ctxstack = [] - def __enter__(self): - self._ctxstack.append([]) - def __exit__(self, exc_type, exc_value, traceback): - self._ctxstack.pop() - return False - def stash(self, *objs): - if len(self._ctxstack) < 1: - raise Exception("Not in any dynamic scope") - for obj in objs: - self._ctxstack[-1].append(obj) -def _fromHParsedToken(cobj): - # TODO: Free the toplevel parser - tt = cobj.token_type - - if cobj.token_type == _lib.TT_BYTES: - return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == _lib.TT_ERR: - # I have no idea what this is for - pass - elif cobj.token_type == _lib.TT_NONE: - return None - elif cobj.token_type == _lib.TT_SEQUENCE: - return [_fromHParsedToken(cobj.seq.elements[i]) - for i in range(cobj.seq.used)] - elif cobj.token_type == _lib.TT_SINT: - return cobj.sint - elif cobj.token_type == _lib.TT_UINT: - return cobj.uint - elif cobj.token_type == _lib.TT_PYTHON: - return _ffi.from_handle(cobj.user) - -_parser_result_holder = _DynamicScopeHolder() -def _toHParsedToken(arena, pyobj): - if pyobj is None: - return _ffi.NULL - cobj = _ffi.new_handle(pyobj) - _parser_result_holder.stash(cobj) - - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) - hpt.token_type = _lib.TT_PYTHON - hpt.user = cobj - hpt.bit_offset = chr(127) - hpt.index = 0 - return hpt - -def _fromParseResult(cobj): - ret = _fromHParsedToken(cobj.ast) - _lib.h_parse_result_free(cobj) - return ret - -def _to_haction(fn): - """Turn a function that transforms a parsed value into an HAction""" - def action(parse_result): - res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) - if res != _ffi.NULL and parse_result.ast != _ffi.NULL: - res.index = parse_result.ast.index - res.bit_offset = parse_result.ast.bit_offset - return res - return _ffi.callback("HParsedToken*(HParseResult*)", action) - -def _to_hpredicate(fn): - """Turn a function that transforms a parsed value into an HAction""" - def predicate(parse_result): - res = fn(_fromParseResult(parse_result)) - # TODO: Handle exceptions; parse should fail. - if type(res) != bool: - raise TypeError("Predicates should return a bool") - return res - return _ffi.callback("bool(HParseResult*)", predicate) - -class Parser(object): - # TODO: Map these to individually garbage-collected blocks of - # memory. Perhaps with an arena allocator with block size of 1? - # There has to be something more efficient than that, though. - - # TODO: How do we handle encodings? By default, we're using UTF-8 - def __init__(self, internal, deps): - """Create a new parser from an FFI object. Not for user code""" - self._parser = internal - self._deps = deps - - def parse(self, string): - with _parser_result_holder: - pres = _lib.h_parse(self._parser, string, len(string)) - if pres: - return _fromParseResult(pres) - else: - return None - - def __mul__(self, count): - return repeat_n(self, count) - - - -class IndirectParser(Parser): - def bind(self, inner): - _lib.h_bind_indirect(self._parser, inner._parser) - self._deps = (inner,) - -class BitsParser(Parser): - pass - -def token(token): - # TODO: Does not clone argument. - if isinstance(token, unicode): - token = token.encode("utf-8") - return Parser(_lib.h_token(token, len(token)), ()) - -def ch(char): - """Returns either a token or an int, depending on the type of the - argument""" - if isinstance(char, int): - return Parser(_lib.h_ch(char), ()) - else: - return token(char) - -def ch_range(chr1, chr2): - if not isinstance(chr1, str) or not isinstance(chr2, str): - raise TypeError("ch_range can't handle unicode") - def my_action(pr): - # print "In action: ", pr - return pr - return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) - -def int_range(parser, i1, i2): - if type(parser) != BitsParser: - raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) - -def bits(length, signedp): - return BitsParser(_lib.h_bits(length, signedp), ()) - -def int64(): return bits(64, True) -def int32(): return bits(32, True) -def int16(): return bits(16, True) -def int8 (): return bits(8, True) -def uint64(): return bits(64, False) -def uint32(): return bits(32, False) -def uint16(): return bits(16, False) -def uint8 (): return bits(8, False) - -def whitespace(p): - return Parser(_lib.h_whitespace(p._parser), (p,)) -def left(p1, p2): - return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) -def right(p1, p2): - return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) -def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) -def action(parser, action): - caction = _to_haction(action) - return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) - -def in_(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_in(charset, len(charset)), ()) -def not_in(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_not_in(charset, len(charset)), ()) -def end_p(): - return Parser(_lib.h_end_p(), ()) -def nothing_p(): - return Parser(_lib.h_nothing_p(), ()) -def sequence(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_sequence(*plist), (plist,)) -def choice(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_choice(*plist), (plist,)) -def butnot(p1, p2): - return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) -def difference(p1, p2): - return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) -def xor(p1, p2): - return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) -def many(p1): - return Parser(_lib.h_many(p1._parser), (p1,)) -def many1(p1): - return Parser(_lib.h_many1(p1._parser), (p1,)) -def repeat_n(p1, n): - return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) -def optional(p1): - return Parser(_lib.h_optional(p1._parser), (p1,)) -def ignore(p1): - return Parser(_lib.h_ignore(p1._parser), (p1,)) -def sepBy(p, sep): - return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) -def sepBy1(p, sep): - return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) -def epsilon_p(): - return Parser(_lib.h_epsilon_p(), ()) -def length_value(p_len, p_value): - return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) -def attr_bool(parser, predicate): - cpredicate = _to_hpredicate(predicate) - return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) -def and_(parser): - return Parser(_lib.h_and(parser._parser), (parser,)) -def not_(parser): - return Parser(_lib.h_not(parser._parser), (parser,)) -def indirect(): - return IndirectParser(_lib.h_indirect(), ()) -def bind_indirect(indirect, inner): - indirect.bind(inner) - -def parse(parser): - return parser.parse() - -# Unfortunately, "in", "and", and "not" are keywords. This makes them -# show up in the module namespace for the use of automated tools. Do -# not attempt to use them by hand; only use the mangled forms (with -# the '_') -sys.modules[__name__].__dict__["in"] = in_ -sys.modules[__name__].__dict__["and"] = and_ -sys.modules[__name__].__dict__["not"] = not_ - -def run_test(): - p_test = sepBy1(choice(ch('1'), - ch('2'), - ch('3')), - ch(',')) - return p_test.parse("1,2,3") - -# {{{ Automatic parser construction... python specific - -# TODO: Implement Parsable metaclass, which requires the existence of -# a "parse" method. - -# This is expected to be extended by user code. As a general rule, -# only provide auto-parsers for your own types. -AUTO_PARSERS = { - str: token, - unicode: token, -} - -def _auto_seq(lst): - return sequence(*(auto_1(p, default_method=_auto_choice) - for p in lst)) - -def _auto_choice(lst): - return choice(*(auto_1(p, default_method=_auto_seq) - for p in lst)) - -def auto_1(arg, default_method=_auto_choice): - if isinstance(arg, Parser): - return arg - elif type(arg) in AUTO_PARSERS: - return AUTO_PARSERS[type(arg)](arg) - else: - return default_method(arg) - -def auto(*args): - return auto_1(args, default_method=_auto_choice) - -# }}} diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a5967e6..82f4084 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95"), None) @@ -16,8 +16,8 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -36,7 +36,7 @@ class TestInt64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) @@ -45,8 +45,8 @@ class TestInt32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -56,8 +56,8 @@ class TestInt16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xfe"), None) self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -67,7 +67,7 @@ class TestInt8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -76,7 +76,7 @@ class TestUint64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) @@ -85,7 +85,7 @@ class TestUint32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -94,7 +94,7 @@ class TestUint16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -103,7 +103,7 @@ class TestUint8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -112,7 +112,7 @@ class TestIntRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05"), 5) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x0b"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -133,8 +133,8 @@ class TestWhitespaceEnd(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, ""), None) # empty string + self.assertEqual(h.h_parse(self.parser, " "), None) # empty string def test_failure(self): self.assertEqual(h.h_parse(self.parser, " x"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,14 +175,16 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -@unittest.skip("Action not implemented yet") +#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), + h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + lambda x: [chr(y).upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "XX"), None) @@ -191,7 +193,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -200,7 +202,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -209,7 +211,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -227,7 +229,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -237,9 +239,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -248,8 +250,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -258,8 +260,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -268,7 +270,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -277,7 +279,7 @@ class TestDifference(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab"), "ab") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -286,8 +288,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) + self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -297,10 +299,10 @@ class TestMany(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) def test_failure(self): pass @@ -309,9 +311,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -321,7 +323,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -331,10 +333,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) + self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -345,7 +346,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -354,11 +355,11 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -367,10 +368,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -380,7 +381,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) def test_failure(self): pass @@ -389,7 +390,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -398,7 +399,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -407,8 +408,8 @@ class TestEpsilonP3(unittest.TestCase): # def setUpClass(cls): # cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) # def test_failure(self): # self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -417,7 +418,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) def test_failure(self): pass @@ -435,7 +436,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) def test_failure(self): pass @@ -444,7 +445,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -453,14 +454,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b").ast.token_data.seq - tree[1] = tree[1].token_data.seq[0] - self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b").ast.token_data.seq - tree[0] = chr(tree[0].token_data.uint) - tree[1] = tree[1].token_data.bytes - tree[2] = chr(tree[2].token_data.uint) - self.assertEqual(tree, ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) def test_failure(self): pass @@ -472,9 +467,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a"), "a") +# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -485,15 +480,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) + self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) + self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) def test_failure(self): pass @@ -507,9 +496,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): # # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 4fb30c5..23bcc15 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,6 @@ %module hammer %nodefaultctor; +//%nodefaultdtor; %include "stdint.i" //%include "typemaps.i" @@ -7,31 +8,6 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; -%typemap(in) uint8_t* { - Py_INCREF($input); - $1 = (uint8_t*)PyString_AsString($input); - } -%typemap(out) uint8_t* { - $result = PyString_FromString((char*)$1); - } - -%typemap(newfree) HParseResult* { - h_parse_result_free($1); - } - -%newobject h_parse -%delobject h_parse_result_free - - /* -%typemap(in) (uint8_t* str, size_t len) { - if (PyString_Check($input) || - PyUnicode_Check($input)) { - PyString_AsStringAndSize($input, (char**)&$1, &$2); - } else { - PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); - } - } -*/ %apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} @@ -79,6 +55,43 @@ PyList_SetItem($result, i, o); } } +%typemap(out) struct HParseResult_* { + if ($1 == NULL) { + // TODO: raise parse failure + Py_INCREF(Py_None); + $result = Py_None; + } else { + $result = hpt_to_python($1->ast); + } + } + +%inline %{ + static int h_tt_python; + %} +%init %{ + h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); + %} + +/* +%typemap(in) (HPredicate* pred, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_predicate; + } +*/ +%typemap(in) (const HAction a, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_action; + } + +%inline { + struct HParsedToken_; + struct HParseResult_; + static PyObject* hpt_to_python(const struct HParsedToken_ *token); + + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); + } #else #warning no uint8_t* typemaps defined #endif @@ -92,3 +105,74 @@ %include "allocator.h" %include "hammer.h" + +%extend HArena_ { + ~HArena_() { + h_delete_arena($self); + } + }; +%extend HParseResult_ { + ~HParseResult_() { + h_parse_result_free($self); + } +}; + +%newobject h_parse; +%delobject h_parse_result_free; +%newobject h_new_arena; +%delobject h_delete_arena; + +#ifdef SWIGPYTHON +%inline { + static PyObject* hpt_to_python(const HParsedToken *token) { + // Caller holds a reference to returned object + PyObject *ret; + if (token == NULL) { + Py_RETURN_NONE; + } + switch (token->token_type) { + case TT_NONE: + Py_RETURN_NONE; + break; + case TT_BYTES: + return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); + case TT_SINT: + // TODO: return PyINT if appropriate + return PyLong_FromLong(token->token_data.sint); + case TT_UINT: + // TODO: return PyINT if appropriate + return PyLong_FromUnsignedLong(token->token_data.uint); + case TT_SEQUENCE: + ret = PyTuple_New(token->token_data.seq->used); + for (int i = 0; i < token->token_data.seq->used; i++) { + PyTuple_SET_ITEM(ret, i, hpt_to_python(token->token_data.seq->elements[i])); + } + return ret; + default: + if (token->token_type == h_tt_python) { + ret = (PyObject*)token->token_data.user; + Py_INCREF(ret); + return ret; + } else { + return SWIG_NewPointerObj((void*)token, SWIGTYPE_p_HParsedToken_, 0 | 0); + // TODO: support registry + } + + } + } + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + if (ret == NULL) { + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + HParsedToken *tok = h_make(p->arena, h_tt_python, ret); + return tok; + } + + } +#endif From 294bca3fce98ead2a3383ab578e39c9dfde0d124 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 19:53:58 -0600 Subject: [PATCH 095/103] Cleaned up python bindings interface a bit; now returns strings rather than character codes if appropriate --- src/bindings/python/SConscript | 2 +- src/bindings/python/hammer_tests.py | 109 ++++++++++++++-------------- src/bindings/swig/hammer.i | 77 ++++++++++++++++++-- 3 files changed, 126 insertions(+), 62 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 0bde353..fd03a3f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -18,6 +18,6 @@ libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") +pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 82f4084..7a7d5f1 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, "\ta"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,13 +175,12 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), - lambda x: [chr(y).upper() for y in x]) + lambda x: [y.upper() for y in x]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) @@ -193,7 +192,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -202,7 +201,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), "d") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -211,7 +210,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -229,7 +228,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -239,9 +238,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -250,8 +249,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -260,8 +259,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "aa"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -270,7 +269,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), "4") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -288,8 +287,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) - self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), "0") + self.assertEqual(h.h_parse(self.parser, "9"), "9") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -300,9 +299,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass @@ -311,9 +310,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -323,7 +322,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -333,9 +332,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) + self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) + self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) + self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -346,7 +345,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) + self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -355,10 +354,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -368,10 +367,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -381,7 +380,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) + self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) def test_failure(self): pass @@ -390,7 +389,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -399,7 +398,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -418,7 +417,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) + self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) def test_failure(self): pass @@ -436,7 +435,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) + self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) def test_failure(self): pass @@ -445,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) + self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -454,8 +453,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) - self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) + self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -480,9 +479,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) - self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) - self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) + self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 23bcc15..a2a8940 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,8 @@ %module hammer + %nodefaultctor; -//%nodefaultdtor; %include "stdint.i" - //%include "typemaps.i" - //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) %ignore HCountedArray_; @@ -12,6 +10,67 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} + + +%rename(_h_ch) h_ch; +%pythoncode %{ + def h_ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return h_token(ch) + else: + return _h_ch(ch) +%} + +%rename(_h_ch_range) h_ch_range; +%pythoncode %{ + def h_ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return h_action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +%} + +%rename(_h_in) h_in; +%rename(_h_not_in) h_not_in; +%pythoncode %{ + def h_in(charset): + return h_action(_h_in(charset), chr) + def h_not_in(charset): + return h_action(_h_not_in(charset), chr) + %} + +%inline { + static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; + + static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + _helper_ParseError = parse_error; + _helper_Placeholder = placeholder; + } + } + +%pythoncode %{ + class Placeholder(object): + """The python equivalent of TT_NONE""" + def __str__(self): + return "Placeholder" + def __repr__(self): + return "Placeholder" + def __eq__(self, other): + return type(self) == type(other) + class ParseError(Exception): + """The parse failed; the message may have more information""" + pass + + _hammer._register_helpers(ParseError, + Placeholder) + %} + %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); @@ -64,7 +123,9 @@ $result = hpt_to_python($1->ast); } } - +%typemap(newfree) struct HParseResult_* { + h_parse_result_free($input); + } %inline %{ static int h_tt_python; %} @@ -72,6 +133,8 @@ h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); %} + + /* %typemap(in) (HPredicate* pred, void* user_data) { Py_INCREF($input); @@ -132,7 +195,7 @@ } switch (token->token_type) { case TT_NONE: - Py_RETURN_NONE; + return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL); break; case TT_BYTES: return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); @@ -173,6 +236,8 @@ HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } - + } + + #endif From 43357bbcda19ff22b509ecbd134b0b14bbe1b0ef Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sun, 24 Nov 2013 18:43:14 -0600 Subject: [PATCH 096/103] AttrBool working --- src/bindings/python/hammer_tests.py | 18 ++++++++-------- src/bindings/swig/hammer.i | 32 ++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 7a7d5f1..587fbd5 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -402,15 +402,15 @@ class TestEpsilonP3(unittest.TestCase): def test_failure(self): pass -# class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) + self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index a2a8940..20661c8 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -135,26 +135,28 @@ -/* -%typemap(in) (HPredicate* pred, void* user_data) { + +%typemap(in) (HPredicate pred, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_predicate; } -*/ + %typemap(in) (const HAction a, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_action; } -%inline { +%inline %{ + struct HParsedToken_; struct HParseResult_; static PyObject* hpt_to_python(const struct HParsedToken_ *token); static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); - } + static int call_predicate(const struct HParseResult_ *p, void* user_data); + %} #else #warning no uint8_t* typemaps defined #endif @@ -164,6 +166,7 @@ #include "allocator.h" #include "hammer.h" #include "internal.h" +#include "glue.h" %} %include "allocator.h" %include "hammer.h" @@ -233,11 +236,30 @@ assert(ret != NULL); } // TODO: add reference to ret to parse-local data + // For now, just hold onto reference HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } + static int call_predicate(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + int rret = 0; + if (ret == NULL) { + // TODO: throw exception + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + rret = PyObject_IsTrue(ret); + Py_DECREF(ret); + return rret; + } + } + #endif From cdc7ea83816289231ef14d655c0c5f4138be8fb0 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Mon, 25 Nov 2013 02:49:02 -0600 Subject: [PATCH 097/103] Finished Python bindings --- src/bindings/python/hammer_tests.py | 398 ++++++++++++++-------------- src/bindings/swig/hammer.i | 145 +++++++--- 2 files changed, 315 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 587fbd5..9014d5c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,457 +4,462 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2") + cls.parser = h.token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95"), None) + self.assertEqual(self.parser.parse("95"), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.h_ch(0xa2) - cls.parser_chr = h.h_ch("\xa2") + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_ch_range("a", "c") + cls.parser = h.ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int64() + cls.parser = h.int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int32() + cls.parser = h.int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int16() + cls.parser = h.int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe"), None) - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int8() + cls.parser = h.int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) + self.assertEqual(self.parser.parse("\x88"), -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint64() + cls.parser = h.uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint32() + cls.parser = h.uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint16() + cls.parser = h.uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint8() + cls.parser = h.uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) + self.assertEqual(self.parser.parse("\x78"), 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int_range(h.h_uint8(), 3, 10) + cls.parser = h.int_range(h.uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05"), 5) + self.assertEqual(self.parser.parse("\x05"), 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b"), None) + self.assertEqual(self.parser.parse("\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_ch("a")) + cls.parser = h.whitespace(h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, "\ta"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a"), None) + self.assertEqual(self.parser.parse("_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_end_p()) + cls.parser = h.whitespace(h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), None) # empty string - self.assertEqual(h.h_parse(self.parser, " "), None) # empty string + self.assertEqual(self.parser.parse(""), None) # empty string + self.assertEqual(self.parser.parse(" "), None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x"), None) + self.assertEqual(self.parser.parse(" x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.left(h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), "a") + self.assertEqual(self.parser.parse("a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) + cls.parser = h.right(h.ch(" "), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ba"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), "a") + self.assertEqual(self.parser.parse(" a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, " a"), None) - self.assertEqual(h.h_parse(self.parser, "a "), None) - self.assertEqual(h.h_parse(self.parser, " b "), None) - self.assertEqual(h.h_parse(self.parser, "ba "), None) - self.assertEqual(h.h_parse(self.parser, " ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), - h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), + h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) + self.assertEqual(self.parser.parse("ab"), ["A", "B"]) + self.assertEqual(self.parser.parse("AB"), ["A", "B"]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX"), None) + self.assertEqual(self.parser.parse("XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc") + cls.parser = h.in_("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc") + cls.parser = h.not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), "d") + self.assertEqual(self.parser.parse("d"), "d") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) + cls.parser = h.sequence(h.ch("a"), h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa"), None) + self.assertEqual(self.parser.parse("aa"), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_nothing_p() + cls.parser = h.nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, "b"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c"), None) + self.assertEqual(self.parser.parse("a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.choice(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c"), None) + self.assertEqual(self.parser.parse("c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.butnot(h.ch("a"), h.token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "aa"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), "4") + self.assertEqual(self.parser.parse("4"), "4") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6"), None) + self.assertEqual(self.parser.parse("6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) + cls.parser = h.difference(h.token("ab"), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), "ab") + self.assertEqual(self.parser.parse("ab"), "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), "0") - self.assertEqual(h.h_parse(self.parser, "9"), "9") + self.assertEqual(self.parser.parse("0"), "0") + self.assertEqual(self.parser.parse("9"), "9") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5"), None) - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("5"), None) + self.assertEqual(self.parser.parse("a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) + self.assertEqual(self.parser.parse(""), ()) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("b"), ('b',)) + self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) - self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) + self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse("b"), ("b",)) + self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) + self.assertEqual(self.parser.parse("abdef"), ('a', 'b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef"), None) - self.assertEqual(h.h_parse(self.parser, "dabdef"), None) + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) - self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) - self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) + self.assertEqual(self.parser.parse("abd"), ('a','b','d')) + self.assertEqual(self.parser.parse("acd"), ('a','c','d')) + self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed"), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) + self.assertEqual(self.parser.parse("abc"), ("a","c")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) - self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) + self.assertEqual(self.parser.parse(""), ()) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) + self.assertEqual(self.parser.parse("ab"), ("a", "b")) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestAttrBool(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), + lambda x: x[0] == x[1]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) - self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + self.assertEqual(self.parser.parse("aa"), ("a", "a")) + self.assertEqual(self.parser.parse("bb"), ("b", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) + self.assertEqual(self.parser.parse("0"), ("0",)) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0"), None) + self.assertEqual(self.parser.parse("0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) + self.assertEqual(self.parser.parse("12"), ('1',)) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), + h.choice(h.ch("+"), h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) + self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b"), None) + self.assertEqual(self.parser.parse("a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), + h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) - self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) + self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b')) + self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -462,26 +467,29 @@ class TestNot2(unittest.TestCase): # #class TestLeftrec(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # a = h.h_ch("a") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # cls.parser = h.indirect() +# # a = h.ch("a") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a"), "a") -# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) +# # self.assertEqual(self.parser.parse("a"), "a") +# # self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass + class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) + #raise unittest.SkipTest("Bind doesn't work right now") + cls.parser = h.indirect() + a = h.ch("a") + cls.parser.bind(h.choice(h.sequence(a, cls.parser), + h.epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) - self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("aa"), ('a', ('a',))) + self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass @@ -489,15 +497,15 @@ class TestRightrec(unittest.TestCase): # #class TestAmbiguous(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # d = h.h_ch("d") -# # p = h.h_ch("+") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # cls.parser = h.indirect() +# # d = h.ch("d") +# # p = h.ch("+") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d"), ["d"]) +# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+"), None) +# # self.assertEqual(self.parser.parse("d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 20661c8..e251904 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -12,43 +12,13 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} -%rename(_h_ch) h_ch; -%pythoncode %{ - def h_ch(ch): - if isinstance(ch, str) or isinstance(ch, unicode): - return h_token(ch) - else: - return _h_ch(ch) -%} - -%rename(_h_ch_range) h_ch_range; -%pythoncode %{ - def h_ch_range(c1, c2): - dostr = isinstance(c1, str) - dostr2 = isinstance(c2, str) - if isinstance(c1, unicode) or isinstance(c2, unicode): - raise TypeError("ch_range only works on bytes") - if dostr != dostr2: - raise TypeError("Both arguments to ch_range must be the same type") - if dostr: - return h_action(_h_ch_range(c1, c2), chr) - else: - return _h_ch_range(c1, c2) -%} - -%rename(_h_in) h_in; -%rename(_h_not_in) h_not_in; -%pythoncode %{ - def h_in(charset): - return h_action(_h_in(charset), chr) - def h_not_in(charset): - return h_action(_h_not_in(charset), chr) - %} +%rename("_%s") ""; +// %rename(_h_ch) h_ch; %inline { static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; - static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + static void register_helpers(PyObject* parse_error, PyObject *placeholder) { _helper_ParseError = parse_error; _helper_Placeholder = placeholder; } @@ -260,6 +230,115 @@ } +%rename("%s") ""; +%extend HParser_ { + HParseResult* parse(const uint8_t* input, size_t length) { + return h_parse($self, input, length); + } + bool compile(HParserBackend backend) { + return h_compile($self, backend, NULL) == 0; + } + PyObject* __dir__() { + PyObject* ret = PyList_New(2); + PyList_SET_ITEM(ret, 0, PyString_FromString("parse")); + PyList_SET_ITEM(ret, 1, PyString_FromString("compile")); + return ret; + } +} + +%pythoncode %{ + +def action(p, act): + return _h_action(p, act) +def attr_bool(p, pred): + return _h_attr_bool(p, pred) + +def ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return token(ch) + else: + return _h_ch(ch) + +def ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +def epsilon_p(): return _h_epsilon_p() +def end_p(): + return _h_end_p() +def in_(charset): + return action(_h_in(charset), chr) +def not_in(charset): + return action(_h_not_in(charset), chr) +def not_(p): return _h_not(p) +def int_range(p, i1, i2): + return _h_int_range(p, i1, i2) +def token(string): + return _h_token(string) +def whitespace(p): + return _h_whitespace(p) +def xor(p1, p2): + return _h_xor(p1, p2) +def butnot(p1, p2): + return _h_butnot(p1, p2) +def and_(p1): + return _h_and(p1) +def difference(p1, p2): + return _h_difference(p1, p2) + +def sepBy(p, sep): return _h_sepBy(p, sep) +def sepBy1(p, sep): return _h_sepBy1(p, sep) +def many(p): return _h_many(p) +def many1(p): return _h_many1(p) +def repeat_n(p, n): return _h_repeat_n(p, n) +def choice(*args): return _h_choice__a(list(args)) +def sequence(*args): return _h_sequence__a(list(args)) + +def optional(p): return _h_optional(p) +def nothing_p(): return _h_nothing_p() +def ignore(p): return _h_ignore(p) + +def left(p1, p2): return _h_left(p1, p2) +def middle(p1, p2, p3): return _h_middle(p1, p2, p3) +def right(p1, p2): return _h_right(p1, p2) + + +class HIndirectParser(_HParser_): + def __init__(self): + # Shoves the guts of an _HParser_ into a HIndirectParser. + tret = _h_indirect() + self.__dict__.clear() + self.__dict__.update(tret.__dict__) + + def __dir__(self): + return super(HIndirectParser, self).__dir__() + ['bind'] + def bind(self, parser): + _h_bind_indirect(self, parser) + +def indirect(): + return HIndirectParser() + +def bind_indirect(indirect, new_parser): + indirect.bind(new_parser) + +def uint8(): return _h_uint8() +def uint16(): return _h_uint16() +def uint32(): return _h_uint32() +def uint64(): return _h_uint64() +def int8(): return _h_int8() +def int16(): return _h_int16() +def int32(): return _h_int32() +def int64(): return _h_int64() + + +%} #endif From f21954eecd430ee3760366551ee2977d496335b0 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:29:28 -0800 Subject: [PATCH 098/103] everything's working for new-build except Clean() --- SConstruct | 19 ++++++++++++++----- src/SConscript | 25 ++++++++++++++----------- src/bindings/python/SConscript | 13 +++++++++---- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/SConstruct b/SConstruct index 50c185b..e30f6df 100644 --- a/SConstruct +++ b/SConstruct @@ -7,9 +7,13 @@ import sys vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept)) +vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['python'])) env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars, tools=['default', 'scanreplace'], toolpath=['tools']) +if not 'bindings' in env: + env['bindings'] = [] + def calcInstallPath(*elements): path = os.path.abspath(os.path.join(*map(env.subst, elements))) if 'DESTDIR' in env: @@ -90,18 +94,23 @@ env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') #env.Append(CPPPATH=os.path.join('#', "hammer")) +testruns = [] + Export('env') +Export('testruns') if not GetOption("in_place"): env['BUILD_BASE'] = 'build/$VARIANT' - env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') - env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples') + lib = env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') + env.Alias("examples", env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples')) else: env['BUILD_BASE'] = '.' - env.SConscript(["src/SConscript"]) - env.SConscript(["examples/SConscript"]) + lib = env.SConscript(["src/SConscript"]) + env.Alias(env.SConscript(["examples/SConscript"])) -env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') +#env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') + +env.Alias("test", testruns) env.Alias("install", "$libpath") env.Alias("install", "$incpath") diff --git a/src/SConscript b/src/SConscript index 0cb41af..0388393 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,6 @@ # -*- python -*- -Import('env') - -bindings = ['python'] +import os.path +Import('env testruns') dist_headers = [ "hammer.h", @@ -62,15 +61,16 @@ misc_hammer_parts = [ 'registry.c', 'system_allocator.c'] -tests = ['t_benchmark.c', - 't_bitreader.c', - 't_bitwriter.c', - 't_parser.c', - 't_grammar.c', - 't_misc.c'] +ctests = ['t_benchmark.c', + 't_bitreader.c', + 't_bitwriter.c', + 't_parser.c', + 't_grammar.c', + 't_misc.c'] libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts) libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts) +Default(libhammer_shared, libhammer_static) env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) @@ -81,9 +81,12 @@ env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') testenv.Append(LIBS=['hammer'], LIBPATH=['.']) -testenv.Program('test_suite', tests + ['test_suite.c']) +ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c']) +ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path])) +AlwaysBuild(ctest) +testruns.append(ctest) Export("libhammer_static libhammer_shared") -for b in bindings: +for b in env['bindings']: env.SConscript(["bindings/%s/SConscript" % b]) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index fd03a3f..5d78875 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,11 +1,12 @@ # -*- python -*- -import os.path -Import('env libhammer_shared') +import os, os.path +Import('env libhammer_shared testruns') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) -pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CPPPATH = ['../../']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.ParseConfig("pkg-config --cflags python") pythonenv.Append(LIBS = ['hammer']) pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) @@ -15,9 +16,13 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytestexec = pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytest = Alias("testpython", [pytestexec], pytestexec) +AlwaysBuild(pytest) +testruns.append(pytest) Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) From 1e7553d3dfd0e7f2634411f715aee590754a4a0e Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:40:15 -0800 Subject: [PATCH 099/103] build//src/bindings/python/hammer_tests.py still hangs around but everything else is cleaned up. --- src/bindings/python/SConscript | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 5d78875..0f1e9a9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -20,9 +20,10 @@ Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestexec = pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytests = ['hammer_tests.py'] +pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "nosetests -vv $SOURCE") pytest = Alias("testpython", [pytestexec], pytestexec) AlwaysBuild(pytest) testruns.append(pytest) -Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) + From 5dea57c4d482929e43d6e9cfa9a496e518dc3dc5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:45:34 -0800 Subject: [PATCH 100/103] build matrix for python --- .travis.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e406ca..a594c74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,31 @@ language: c compiler: - gcc - clang +env: + - BINDINGS=none +matrix: + include: + - compiler: gcc + language: python + python: 2.7 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.7 + env: BINDINGS=python + - compiler: gcc + language: python + python: 2.6 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.6 + env: BINDINGS=python before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev python-nose + - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq swig python-dev; fi script: - - scons + - scons bindings=$BINDINGS test notifications: irc: channels: From bd1603812952b01852a910c7eae09a7ec1900306 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:53:30 -0800 Subject: [PATCH 101/103] Python 2.6 is unsupported because the PyCapsule API was backported from Python 3 to 2.7. Trying Python 3.3 and 3.2 just for grins. --- .travis.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a594c74..8717876 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,19 +8,27 @@ matrix: include: - compiler: gcc language: python - python: 2.7 + python: 3.3 env: BINDINGS=python - compiler: clang language: python - python: 2.7 + python: 3.3 env: BINDINGS=python - compiler: gcc language: python - python: 2.6 + python: 3.2 env: BINDINGS=python - compiler: clang language: python - python: 2.6 + python: 3.2 + env: BINDINGS=python + - compiler: gcc + language: python + python: 2.7 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.7 env: BINDINGS=python before_install: - sudo apt-get update -qq From 75255d2e473943982c387dd63660d443020970d0 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:55:23 -0800 Subject: [PATCH 102/103] SCons doesn't support Python 3! So 2.7 only it is. Oh well. --- .travis.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8717876..8c5af52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,6 @@ env: - BINDINGS=none matrix: include: - - compiler: gcc - language: python - python: 3.3 - env: BINDINGS=python - - compiler: clang - language: python - python: 3.3 - env: BINDINGS=python - - compiler: gcc - language: python - python: 3.2 - env: BINDINGS=python - - compiler: clang - language: python - python: 3.2 - env: BINDINGS=python - compiler: gcc language: python python: 2.7 From f611a49ab2c2bcc9a1f0ce5101cd418822a6145c Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 16:04:13 -0800 Subject: [PATCH 103/103] cpp stuff doesn't belong on python-bindings branch --- src/hammer.cxx | 164 ----------------- src/hammer.hxx | 491 ------------------------------------------------- 2 files changed, 655 deletions(-) delete mode 100644 src/hammer.cxx delete mode 100644 src/hammer.hxx diff --git a/src/hammer.cxx b/src/hammer.cxx deleted file mode 100644 index e7f3cb7..0000000 --- a/src/hammer.cxx +++ /dev/null @@ -1,164 +0,0 @@ -#include "hammer.hxx" - -namespace hammer { - - typedef variant AnyResult; - - const BytesResult::result_type BytesResult::result() { return _bytes; } - const UintResult::result_type UintResult::result() { return _uint; } - const IntResult::result_type IntResult::result() { return _sint; } - const SequenceResult::result_type SequenceResult::result() { return _seq; } - - template<> - BytesResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); - } - - template<> - BytesResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); - } - - template<> - UintResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return UintResult(res->ast->uint); - } - - template<> - UintResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return UintResult(res->ast->uint); - } - - template<> - IntResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return IntResult(res->ast->sint); - } - - template<> - IntResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return IntResult(res->ast->sint); - } - - template<> - NullResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return NullResult(); - } - - template<> - NullResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return NullResult(); - } - - vector make_seq(HCountedArray *seq) { - vector ret; - for (size_t i=0; iused; ++i) { - switch(seq->elements[i]->token_type) { - case TT_NONE: - ret.push_back(NullResult()); - break; - case TT_BYTES: - ret.push_back(BytesResult(vector(seq->elements[i]->bytes.token, seq->elements[i]->bytes.token+seq->elements[i]->bytes.len))); - break; - case TT_SINT: - ret.push_back(IntResult(seq->elements[i]->sint)); - break; - case TT_UINT: - ret.push_back(UintResult(seq->elements[i]->uint)); - break; - case TT_SEQUENCE: - ret.push_back(make_seq(seq->elements[i]->seq)); - break; - default: - //TODO some kind of error - break; - } - } - return ret; - } - - template<> - SequenceResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return SequenceResult(make_seq(res->ast->seq)); - } - - template<> - SequenceResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return SequenceResult(make_seq(res->ast->seq)); - } - - template - Many Parser::many() { - return Many(*this); - } - - template<> - Many Parser::many() { - return Many(*this); - } - - template - RepeatN Parser::many(size_t n) { - return RepeatN(this, n); - } - - template - Optional Parser::optional() { - return Optional(this); - } - - template - RepeatN Parser::operator[](size_t n) { - return RepeatN(this, n); - } - - IntRange Int64::in_range(const int64_t lower, const int64_t upper) { - Int64 p = Int64(); - return IntRange(p, lower, upper); - } - - IntRange Int32::in_range(const int32_t lower, const int32_t upper) { - Int32 p = Int32(); - return IntRange(p, lower, upper); - } - - IntRange Int16::in_range(const int16_t lower, const int16_t upper) { - Int16 p = Int16(); - return IntRange(p, lower, upper); - } - - IntRange Int8::in_range(const int8_t lower, const int8_t upper) { - Int8 p = Int8(); - return IntRange(p, lower, upper); - } - - IntRange Uint64::in_range(const uint64_t lower, const uint64_t upper) { - Uint64 p = Uint64(); - return IntRange(p, lower, upper); - } - - IntRange Uint32::in_range(const uint32_t lower, const uint32_t upper) { - Uint32 p = Uint32(); - return IntRange(p, lower, upper); - } - - IntRange Uint16::in_range(const uint16_t lower, const uint16_t upper) { - Uint16 p = Uint16(); - return IntRange(p, lower, upper); - } - - IntRange Uint8::in_range(const uint8_t lower, const uint8_t upper) { - Uint8 p = Uint8(); - return IntRange(p, lower, upper); - } - -} diff --git a/src/hammer.hxx b/src/hammer.hxx deleted file mode 100644 index 8f3200c..0000000 --- a/src/hammer.hxx +++ /dev/null @@ -1,491 +0,0 @@ -#ifndef HAMMER_HAMMER__HXX -#define HAMMER_HAMMER__HXX - -#include "hammer.h" -#include -#include -#include -#include - -using std::list; using std::string; using std::vector; -using boost::variant; - -namespace hammer { - - template - class ParseResult { - public: - typedef T result_type; - protected: - ParseResult() { } - }; - - class BytesResult : public ParseResult > { - public: - typedef vector result_type; - BytesResult(const vector res) : _bytes(res) { } - const result_type result(); - private: - BytesResult() { } - result_type _bytes; - }; - - class UintResult : public ParseResult { - public: - typedef uint64_t result_type; - UintResult(const uint64_t res) : _uint(res) { } - const result_type result(); - private: - UintResult() { } - result_type _uint; - }; - - class IntResult : public ParseResult { - public: - typedef int64_t result_type; - IntResult(const int64_t res) : _sint(res) { } - const result_type result(); - private: - IntResult() { } - result_type _sint; - }; - - class NullResult: public ParseResult { - public: - NullResult() { } - typedef void* result_type; - const result_type result() { return NULL; } - }; - - class SequenceResult : public ParseResult > > { - public: - typedef vector > result_type; - SequenceResult(result_type res) : _seq(res) { } - const result_type result(); - private: - SequenceResult() { } - result_type _seq; - }; - - /* forward declarations */ - template class Many; - class Many1; - template class Optional; - class RepeatN; - template class IntRange; - - template - class Parser { - public: - typedef T result_type; - result_type parse(const string &input); - result_type parse(const uint8_t *input, size_t length); - Many many(); - RepeatN many(size_t n); - Optional optional(); - RepeatN operator[](size_t n); - HParser* parser() { return _parser; } - protected: - HParser* _parser; - Parser() { } - // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... - }; - - class Token : public Parser { - public: - Token(string &str) : _tok(str) { - _parser = h_token(reinterpret_cast(str.c_str()), str.size()); - } - Token(const uint8_t *str, size_t length) : _tok(reinterpret_cast(str), length) { - _parser = h_token(str, length); - } - private: - string _tok; - }; - - class Ch : public Parser { - public: - friend class Parser; - Ch(const uint8_t c) : _c(c) { - _parser = h_ch(c); - } - private: - uint8_t _c; - }; - - class ChRange : public Parser { - public: - ChRange(const uint8_t lower, const uint8_t upper) : _lower(lower), _upper(upper) { - _parser = h_ch_range(lower, upper); - } - private: - uint8_t _lower, _upper; - }; - - class SignedBits : public Parser { - public: - SignedBits(size_t len) : _len(len) { - _parser = h_bits(len, true); - } - private: - size_t _len; - }; - - class UnsignedBits : public Parser { - public: - UnsignedBits(size_t len) : _len(len) { - _parser = h_bits(len, false); - } - private: - size_t _len; - }; - - class Int64 : public Parser { - public: - Int64() { - _parser = h_int64(); - } - IntRange in_range(const int64_t lower, const int64_t upper); - }; - - class Int32 : public Parser { - public: - Int32() { - _parser = h_int32(); - } - IntRange in_range(const int32_t lower, const int32_t upper); - }; - - class Int16 : public Parser { - public: - Int16() { - _parser = h_int16(); - } - IntRange in_range(const int16_t lower, const int16_t upper); - }; - - class Int8 : public Parser { - public: - Int8() { - _parser = h_int8(); - } - IntRange in_range(const int8_t lower, const int8_t upper); - }; - - class Uint64 : public Parser { - public: - Uint64() { - _parser = h_uint64(); - } - IntRange in_range(const uint64_t lower, const uint64_t upper); - }; - - class Uint32 : public Parser { - public: - Uint32() { - _parser = h_uint32(); - } - IntRange in_range(const uint32_t lower, const uint32_t upper); - }; - - class Uint16 : public Parser { - public: - Uint16() { - _parser = h_uint16(); - } - IntRange in_range(const uint16_t lower, const uint16_t upper); - }; - - class Uint8 : public Parser { - public: - Uint8() { - _parser = h_uint8(); - } - IntRange in_range(const uint8_t lower, const uint8_t upper); - }; - - template - class IntRange : public Parser { - public: - IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { - this->_parser = h_int_range(p.parser(), lower, upper); - } - private: - Parser _p; - int64_t _lower, _upper; - }; - - template - class Whitespace : public Parser { - public: - typedef typename T::result_type result_type; - Whitespace(Parser &p) : _p(p) { - this->_parser = h_whitespace(p.parser()); - } - private: - Parser _p; - }; - - template - class Left : public Parser { - public: - typedef typename T::result_type result_type; - Left(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_left(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Right : public Parser { - public: - typedef typename U::result_type result_type; - Right(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_right(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Middle : public Parser { - public: - typedef typename U::result_type result_type; - Middle(Parser &p, Parser &x, Parser &q) : _p(p), _x(x), _q(q) { - this->_parser = h_middle(p.parser(), x.parser(), q.parser()); - } - private: - Parser _p; - Parser _x; - Parser _q; - }; - - /* what are we doing about h_action? */ - - class In : public Parser { - public: - In(string &charset) : _charset(charset) { - _parser = h_in(reinterpret_cast(charset.c_str()), charset.size()); - } - In(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { - _parser = h_in(charset, length); - } - private: - string _charset; - }; - - class NotIn : public Parser { - public: - NotIn(string &charset) : _charset(charset) { - _parser = h_not_in(reinterpret_cast(charset.c_str()), charset.size()); - } - NotIn(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { - _parser = h_not_in(charset, length); - } - private: - string _charset; - }; - - class End : public Parser { - public: - End() { - _parser = h_end_p(); - } - }; - - class Nothing : public Parser { - public: - Nothing() { - _parser = h_nothing_p(); - } - }; - - class Sequence : public Parser { - friend class Parser; - public: - Sequence(list &ps) : _ps(ps) { - void *parsers[ps.size()]; - size_t i = 0; - for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = const_cast(it->parser()); - } - _parser = h_sequence__a(parsers); - } - // maybe also a begin and end iterator version - private: - list _ps; - }; - - class Choice : public Parser { - public: - Choice(list > &ps) : _ps(ps) { - void *parsers[ps.size()]; - size_t i = 0; - for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = const_cast(it->parser()); - } - _parser = h_choice__a(parsers); - } - private: - list > _ps; - }; - - template - class ButNot : public Parser { - public: - typedef typename T::result_type result_type; - ButNot(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_butnot(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Difference : public Parser { - public: - typedef typename T::result_type result_type; - Difference(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_difference(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Xor : public Parser > { - public: - typedef variant result_type; - Xor(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_xor(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Many : public Parser { - public: - Many(Parser &p) : _p(p) { - _parser = h_many(p.parser()); - } - private: - Parser _p; - }; - - class Many1 : public Parser { - public: - Many1(Parser &p) : _p(p) { - _parser = h_many1(p.parser()); - } - private: - Parser _p; - }; - - class RepeatN: public Parser { - public: - RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { - _parser = h_repeat_n(p.parser(), n); - } - private: - Parser _p; - size_t _n; - }; - - template - class Optional : public Parser { - public: - typedef typename T::result_type result_type; - Optional(Parser &p) : _p(p) { - this->_parser = h_optional(p.parser()); - } - private: - Parser _p; - }; - - class Ignore : public Parser { - public: - Ignore(Parser &p) : _p(p) { - _parser = h_ignore(p.parser()); - } - private: - Parser _p; - }; - - class SepBy : public Parser { - public: - SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { - _parser = h_sepBy(p.parser(), sep.parser()); - } - private: - Parser _p, _sep; - }; - - class SepBy1 : public Parser { - public: - SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { - _parser = h_sepBy1(p.parser(), sep.parser()); - } - private: - Parser _p, _sep; - }; - - class Epsilon : public Parser { - public: - Epsilon() { - _parser = h_epsilon_p(); - } - }; - - template - class LengthValue : public Parser { - public: - LengthValue(Parser &length, Parser &value) : _length(length), _value(value) { - _parser = h_length_value(length.parser(), value.parser()); - } - private: - Parser _length; - Parser _value; - }; - - /* FIXME attr_bool */ - - class And : public Parser { - public: - And(Parser &p) : _p(p) { - _parser = h_and(p.parser()); - } - private: - Parser _p; - }; - - class Not : public Parser { - public: - Not(Parser &p) : _p(p) { - _parser = h_not(p.parser()); - } - private: - Parser _p; - }; - - template - class Indirect : public Parser { - public: - typedef typename T::result_type result_type; - Indirect(Parser &p) : _p(p) { - this->_parser = h_indirect(); - h_bind_indirect(this->_parser, p.parser()); - } - private: - Parser _p; - }; - -} - -#endif