From 2408106191956f20360c3663e4ba0d11ee6cb0d8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Fri, 9 Aug 2013 18:45:26 +0200 Subject: [PATCH 01/88] C++ bindings getting underway. --- src/hammer.cxx | 152 +++++++++++++++ src/hammer.h | 10 + src/hammer.hxx | 493 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 655 insertions(+) create mode 100644 src/hammer.cxx create mode 100644 src/hammer.hxx diff --git a/src/hammer.cxx b/src/hammer.cxx new file mode 100644 index 0000000..dc2b104 --- /dev/null +++ b/src/hammer.cxx @@ -0,0 +1,152 @@ +#include "hammer.hxx" + +namespace hammer { + + typedef variant AnyResult; + + const BytesResult::result_type BytesResult::result() { return _bytes; } + const UintResult::result_type UintResult::result() { return _uint; } + const IntResult::result_type IntResult::result() { return _sint; } + const SequenceResult::result_type SequenceResult::result() { return _seq; } + + template<> + BytesResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); + } + + template<> + BytesResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); + } + + template<> + UintResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return UintResult(res->ast->uint); + } + + template<> + UintResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return UintResult(res->ast->uint); + } + + template<> + IntResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return IntResult(res->ast->sint); + } + + template<> + IntResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return IntResult(res->ast->sint); + } + + template<> + NullResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return NullResult(); + } + + template<> + NullResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return NullResult(); + } + + vector make_seq(HCountedArray *seq) { + vector ret; + for (size_t i=0; iused; ++i) { + switch(seq->elements[i]->token_type) { + case TT_NONE: + ret.push_back(NullResult()); + break; + case TT_BYTES: + ret.push_back(BytesResult(vector(seq->elements[i]->bytes.token, seq->elements[i]->bytes.token+seq->elements[i]->bytes.len))); + break; + case TT_SINT: + ret.push_back(IntResult(seq->elements[i]->sint)); + break; + case TT_UINT: + ret.push_back(UintResult(seq->elements[i]->uint)); + break; + case TT_SEQUENCE: + ret.push_back(make_seq(seq->elements[i]->seq)); + break; + default: + //TODO some kind of error + break; + } + } + return ret; + } + + template<> + SequenceResult Parser::parse(const string &input) { + HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); + return SequenceResult(make_seq(res->ast->seq)); + } + + template<> + SequenceResult Parser::parse(const uint8_t *input, size_t length) { + HParseResult *res = h_parse(_parser, input, length); + return SequenceResult(make_seq(res->ast->seq)); + } + + template + Many Parser::many() { + return Many(this); + } + + template + RepeatN Parser::many(size_t n) { + return RepeatN(this, n); + } + + template + Optional Parser::optional() { + return Optional(this); + } + + template + RepeatN Parser::operator[](size_t n) { + return RepeatN(this, n); + } + + /* + Int64 Int64::in_range(const int64_t lower, const int64_t upper) { + return IntRange(Int64(), lower, upper); + } + + Int32 Int32::in_range(const int32_t lower, const int32_t upper) { + return IntRange(Int32(), lower, upper); + } + + Int16 Int16::in_range(const int16_t lower, const int16_t upper) { + return IntRange(Int16(), lower, upper); + } + + Int8 Int8::in_range(const int8_t lower, const int8_t upper) { + return IntRange(Int8(), lower, upper); + } + + Uint64 Uint64::in_range(const uint64_t lower, const uint64_t upper) { + return IntRange(Uint64(), lower, upper); + } + + Uint32 Uint32::in_range(const uint32_t lower, const uint32_t upper) { + return IntRange(Uint32(), lower, upper); + } + + Uint16 Uint16::in_range(const uint16_t lower, const uint16_t upper) { + return IntRange(Uint16(), lower, upper); + } + + Uint8 Uint8::in_range(const uint8_t lower, const uint8_t upper) { + return IntRange(Uint8(), lower, upper); + } + */ +} diff --git a/src/hammer.h b/src/hammer.h index 67fb8e4..89a9570 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -27,7 +27,13 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __cplusplus typedef int bool; +#endif typedef struct HParseState_ HParseState; @@ -617,4 +623,8 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results); void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} +#ifdef __cplusplus +} +#endif + #endif // #ifndef HAMMER_HAMMER__H diff --git a/src/hammer.hxx b/src/hammer.hxx new file mode 100644 index 0000000..0baee86 --- /dev/null +++ b/src/hammer.hxx @@ -0,0 +1,493 @@ +#ifndef HAMMER_HAMMER__HXX +#define HAMMER_HAMMER__HXX + +#include "hammer.h" +#include +#include +#include +#include + +using std::list; using std::string; using std::vector; +using boost::variant; + +namespace hammer { + + template + class ParseResult { + public: + typedef T result_type; + protected: + ParseResult() { } + }; + + class BytesResult : public ParseResult > { + public: + typedef vector result_type; + BytesResult(const vector res) : _bytes(res) { } + const result_type result(); + private: + BytesResult() { } + result_type _bytes; + }; + + class UintResult : public ParseResult { + public: + typedef uint64_t result_type; + UintResult(const uint64_t res) : _uint(res) { } + const result_type result(); + private: + UintResult() { } + result_type _uint; + }; + + class IntResult : public ParseResult { + public: + typedef int64_t result_type; + IntResult(const int64_t res) : _sint(res) { } + const result_type result(); + private: + IntResult() { } + result_type _sint; + }; + + class NullResult: public ParseResult { + public: + NullResult() { } + typedef void* result_type; + const result_type result() { return NULL; } + }; + + class SequenceResult : public ParseResult > > { + public: + typedef vector > result_type; + SequenceResult(result_type res) : _seq(res) { } + const result_type result(); + private: + SequenceResult() { } + result_type _seq; + }; + + /* forward declarations */ + template class Many; + template class Many1; + template class Optional; + template class RepeatN; + + template + class Parser { + public: + typedef T result_type; + result_type parse(const string &input); + result_type parse(const uint8_t *input, size_t length); + Many many(); + RepeatN many(size_t n); + Optional optional(); + RepeatN operator[](size_t n); + HParser* parser() { return _parser; } + HParser* _parser; + Parser() { } + }; + + class Token : public Parser { + public: + Token(string &str) : _tok(str) { + _parser = h_token(reinterpret_cast(str.c_str()), str.size()); + } + Token(const uint8_t *str, size_t length) : _tok(reinterpret_cast(str), length) { + _parser = h_token(str, length); + } + private: + string _tok; + }; + + class Ch : public Parser { + public: + Ch(const uint8_t c) : _c(c) { + _parser = h_ch(c); + } + private: + uint8_t _c; + }; + + class ChRange : public Parser { + public: + ChRange(const uint8_t lower, const uint8_t upper) : _lower(lower), _upper(upper) { + _parser = h_ch_range(lower, upper); + } + private: + uint8_t _lower, _upper; + }; + + class IntRange : public Parser { + public: + IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { + _parser = h_int_range(p.parser(), lower, upper); + } + private: + Parser _p; + int64_t _lower, _upper; + }; + + class SignedBits : public Parser { + public: + SignedBits(size_t len) : _len(len) { + _parser = h_bits(len, true); + } + private: + size_t _len; + }; + + class UnsignedBits : public Parser { + public: + UnsignedBits(size_t len) : _len(len) { + _parser = h_bits(len, false); + } + private: + size_t _len; + }; + + class Int64 : public Parser { + public: + Int64() { + _parser = h_int64(); + } + Int64 in_range(const int64_t lower, const int64_t upper); + }; + + class Int32 : public Parser { + public: + Int32() { + _parser = h_int32(); + } + Int32 in_range(const int32_t lower, const int32_t upper); + }; + + class Int16 : public Parser { + public: + Int16() { + _parser = h_int16(); + } + Int16 in_range(const int16_t lower, const int16_t upper); + }; + + class Int8 : public Parser { + public: + Int8() { + _parser = h_int8(); + } + Int8 in_range(const int8_t lower, const int8_t upper); + }; + + class Uint64 : public Parser { + public: + Uint64() { + _parser = h_uint64(); + } + Uint64 in_range(const uint64_t lower, const uint64_t upper); + }; + + class Uint32 : public Parser { + public: + Uint32() { + _parser = h_uint32(); + } + Uint32 in_range(const uint32_t lower, const uint32_t upper); + }; + + class Uint16 : public Parser { + public: + Uint16() { + _parser = h_uint16(); + } + Uint16 in_range(const uint16_t lower, const uint16_t upper); + }; + + class Uint8 : public Parser { + public: + Uint8() { + _parser = h_uint8(); + } + Uint8 in_range(const uint8_t lower, const uint8_t upper); + }; + + template + class Whitespace : public Parser { + public: + typedef typename T::result_type result_type; + Whitespace(Parser &p) : _p(p) { + this->_parser = h_whitespace(p.parser()); + } + private: + Parser _p; + }; + + template + class Left : public Parser { + public: + typedef typename T::result_type result_type; + Left(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_left(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Right : public Parser { + public: + typedef typename U::result_type result_type; + Right(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_right(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Middle : public Parser { + public: + typedef typename U::result_type result_type; + Middle(Parser &p, Parser &x, Parser &q) : _p(p), _x(x), _q(q) { + this->_parser = h_middle(p.parser(), x.parser(), q.parser()); + } + private: + Parser _p; + Parser _x; + Parser _q; + }; + + /* what are we doing about h_action? */ + + class In : public Parser { + public: + In(string &charset) : _charset(charset) { + _parser = h_in(reinterpret_cast(charset.c_str()), charset.size()); + } + In(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { + _parser = h_in(charset, length); + } + private: + string _charset; + }; + + class NotIn : public Parser { + public: + NotIn(string &charset) : _charset(charset) { + _parser = h_not_in(reinterpret_cast(charset.c_str()), charset.size()); + } + NotIn(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { + _parser = h_not_in(charset, length); + } + private: + string _charset; + }; + + class End : public Parser { + public: + End() { + _parser = h_end_p(); + } + }; + + class Nothing : public Parser { + public: + Nothing() { + _parser = h_nothing_p(); + } + }; + + class Sequence : public Parser { + public: + Sequence(list &ps) : _ps(ps) { + void *parsers[ps.size()]; + size_t i = 0; + for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { + parsers[i] = it->parser(); + } + _parser = h_sequence__a(parsers); + } + // maybe also a begin and end iterator version + private: + list _ps; + }; + + class Choice : public Parser { + public: + Choice(list > &ps) : _ps(ps) { + void *parsers[ps.size()]; + size_t i = 0; + for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { + parsers[i] = it->parser(); + } + _parser = h_choice__a(parsers); + } + private: + list > _ps; + }; + + template + class ButNot : public Parser { + public: + typedef typename T::result_type result_type; + ButNot(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_butnot(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Difference : public Parser { + public: + typedef typename T::result_type result_type; + Difference(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_difference(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Xor : public Parser > { + public: + typedef variant result_type; + Xor(Parser &p, Parser &q) : _p(p), _q(q) { + this->_parser = h_xor(p.parser(), q.parser()); + } + private: + Parser _p; + Parser _q; + }; + + template + class Many : public Parser { + public: + Many(Parser &p) : _p(p) { + _parser = h_many(p.parser()); + } + private: + Parser _p; + }; + + template + class Many1 : public Parser { + public: + Many1(Parser &p) : _p(p) { + _parser = h_many1(p.parser()); + } + private: + Parser _p; + }; + + template + class RepeatN: public Parser { + public: + RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { + _parser = h_repeat_n(p.parser(), n); + } + private: + Parser _p; + size_t _n; + }; + + template + class Optional : public Parser { + public: + typedef typename T::result_type result_type; + Optional(Parser &p) : _p(p) { + this->_parser = h_optional(p.parser()); + } + private: + Parser _p; + }; + + template + class Ignore : public Parser { + public: + Ignore(Parser &p) : _p(p) { + _parser = h_ignore(p.parser()); + } + private: + Parser _p; + }; + + template + class SepBy : public Parser { + public: + SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { + _parser = h_sepBy(p.parser(), sep.parser()); + } + private: + Parser _p; + Parser _sep; + }; + + template + class SepBy1 : public Parser { + public: + SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { + _parser = h_sepBy1(p.parser(), sep.parser()); + } + private: + Parser _p; + Parser _sep; + }; + + class Epsilon : public Parser { + public: + Epsilon() { + _parser = h_epsilon_p(); + } + }; + + template + class LengthValue : public Parser { + public: + LengthValue(Parser &length, Parser &value) : _length(length), _value(value) { + _parser = h_length_value(length.parser(), value.parser()); + } + private: + Parser _length; + Parser _value; + }; + + /* FIXME attr_bool */ + + template + class And : public Parser { + public: + And(Parser &p) : _p(p) { + _parser = h_and(p.parser()); + } + private: + Parser _p; + }; + + template + class Not : public Parser { + public: + Not(Parser &p) : _p(p) { + _parser = h_not(p.parser()); + } + private: + Parser _p; + }; + + template + class Indirect : public Parser { + public: + Indirect(Parser &p) : _p(p) { + this->_parser = h_indirect(); + h_bind_indirect(this->_parser, p.parser()); + } + private: + Parser _p; + }; + +} + +#endif From 2e76cf2acfd8df306f1ef67f7c5b850aaa0d3f96 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Fri, 9 Aug 2013 22:49:16 +0200 Subject: [PATCH 02/88] and now in_range works too. YEAH. --- src/hammer.cxx | 43 +++++++++++++++++++++++++------------------ src/hammer.hxx | 49 +++++++++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/hammer.cxx b/src/hammer.cxx index dc2b104..1cf38c2 100644 --- a/src/hammer.cxx +++ b/src/hammer.cxx @@ -116,37 +116,44 @@ namespace hammer { return RepeatN(this, n); } - /* - Int64 Int64::in_range(const int64_t lower, const int64_t upper) { - return IntRange(Int64(), lower, upper); + IntRange Int64::in_range(const int64_t lower, const int64_t upper) { + Int64 p = Int64(); + return IntRange(p, lower, upper); } - Int32 Int32::in_range(const int32_t lower, const int32_t upper) { - return IntRange(Int32(), lower, upper); + IntRange Int32::in_range(const int32_t lower, const int32_t upper) { + Int32 p = Int32(); + return IntRange(p, lower, upper); } - Int16 Int16::in_range(const int16_t lower, const int16_t upper) { - return IntRange(Int16(), lower, upper); + IntRange Int16::in_range(const int16_t lower, const int16_t upper) { + Int16 p = Int16(); + return IntRange(p, lower, upper); } - Int8 Int8::in_range(const int8_t lower, const int8_t upper) { - return IntRange(Int8(), lower, upper); + IntRange Int8::in_range(const int8_t lower, const int8_t upper) { + Int8 p = Int8(); + return IntRange(p, lower, upper); } - Uint64 Uint64::in_range(const uint64_t lower, const uint64_t upper) { - return IntRange(Uint64(), lower, upper); + IntRange Uint64::in_range(const uint64_t lower, const uint64_t upper) { + Uint64 p = Uint64(); + return IntRange(p, lower, upper); } - Uint32 Uint32::in_range(const uint32_t lower, const uint32_t upper) { - return IntRange(Uint32(), lower, upper); + IntRange Uint32::in_range(const uint32_t lower, const uint32_t upper) { + Uint32 p = Uint32(); + return IntRange(p, lower, upper); } - Uint16 Uint16::in_range(const uint16_t lower, const uint16_t upper) { - return IntRange(Uint16(), lower, upper); + IntRange Uint16::in_range(const uint16_t lower, const uint16_t upper) { + Uint16 p = Uint16(); + return IntRange(p, lower, upper); } - Uint8 Uint8::in_range(const uint8_t lower, const uint8_t upper) { - return IntRange(Uint8(), lower, upper); + IntRange Uint8::in_range(const uint8_t lower, const uint8_t upper) { + Uint8 p = Uint8(); + return IntRange(p, lower, upper); } - */ + } diff --git a/src/hammer.hxx b/src/hammer.hxx index 0baee86..287ca21 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,6 +72,7 @@ namespace hammer { template class Many1; template class Optional; template class RepeatN; + template class IntRange; template class Parser { @@ -83,9 +84,11 @@ namespace hammer { RepeatN many(size_t n); Optional optional(); RepeatN operator[](size_t n); - HParser* parser() { return _parser; } - HParser* _parser; + const HParser* parser() { return _parser; } + protected: + const HParser* _parser; Parser() { } + // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... }; class Token : public Parser { @@ -118,16 +121,6 @@ namespace hammer { uint8_t _lower, _upper; }; - class IntRange : public Parser { - public: - IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { - _parser = h_int_range(p.parser(), lower, upper); - } - private: - Parser _p; - int64_t _lower, _upper; - }; - class SignedBits : public Parser { public: SignedBits(size_t len) : _len(len) { @@ -151,7 +144,7 @@ namespace hammer { Int64() { _parser = h_int64(); } - Int64 in_range(const int64_t lower, const int64_t upper); + IntRange in_range(const int64_t lower, const int64_t upper); }; class Int32 : public Parser { @@ -159,7 +152,7 @@ namespace hammer { Int32() { _parser = h_int32(); } - Int32 in_range(const int32_t lower, const int32_t upper); + IntRange in_range(const int32_t lower, const int32_t upper); }; class Int16 : public Parser { @@ -167,7 +160,7 @@ namespace hammer { Int16() { _parser = h_int16(); } - Int16 in_range(const int16_t lower, const int16_t upper); + IntRange in_range(const int16_t lower, const int16_t upper); }; class Int8 : public Parser { @@ -175,7 +168,7 @@ namespace hammer { Int8() { _parser = h_int8(); } - Int8 in_range(const int8_t lower, const int8_t upper); + IntRange in_range(const int8_t lower, const int8_t upper); }; class Uint64 : public Parser { @@ -183,7 +176,7 @@ namespace hammer { Uint64() { _parser = h_uint64(); } - Uint64 in_range(const uint64_t lower, const uint64_t upper); + IntRange in_range(const uint64_t lower, const uint64_t upper); }; class Uint32 : public Parser { @@ -191,7 +184,7 @@ namespace hammer { Uint32() { _parser = h_uint32(); } - Uint32 in_range(const uint32_t lower, const uint32_t upper); + IntRange in_range(const uint32_t lower, const uint32_t upper); }; class Uint16 : public Parser { @@ -199,7 +192,7 @@ namespace hammer { Uint16() { _parser = h_uint16(); } - Uint16 in_range(const uint16_t lower, const uint16_t upper); + IntRange in_range(const uint16_t lower, const uint16_t upper); }; class Uint8 : public Parser { @@ -207,7 +200,18 @@ namespace hammer { Uint8() { _parser = h_uint8(); } - Uint8 in_range(const uint8_t lower, const uint8_t upper); + IntRange in_range(const uint8_t lower, const uint8_t upper); + }; + + template + class IntRange : public Parser { + public: + IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { + this->_parser = h_int_range(p.parser(), lower, upper); + } + private: + Parser _p; + int64_t _lower, _upper; }; template @@ -299,12 +303,13 @@ namespace hammer { }; class Sequence : public Parser { + friend class Parser; public: Sequence(list &ps) : _ps(ps) { void *parsers[ps.size()]; size_t i = 0; for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = it->parser(); + parsers[i] = const_cast(it->parser()); } _parser = h_sequence__a(parsers); } @@ -319,7 +324,7 @@ namespace hammer { void *parsers[ps.size()]; size_t i = 0; for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = it->parser(); + parsers[i] = const_cast(it->parser()); } _parser = h_choice__a(parsers); } From 691f3fb5472a2e58974d079d1a004d3900025d01 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 11 Aug 2013 01:13:07 +0200 Subject: [PATCH 03/88] Many apparently requires a template specialization. This is absurd, but whatever. I'll ask stackoverflow. --- src/hammer.cxx | 15 ++++++++++----- src/hammer.hxx | 49 +++++++++++++++++++++---------------------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/hammer.cxx b/src/hammer.cxx index 1cf38c2..e7f3cb7 100644 --- a/src/hammer.cxx +++ b/src/hammer.cxx @@ -98,12 +98,17 @@ namespace hammer { template Many Parser::many() { - return Many(this); + return Many(*this); + } + + template<> + Many Parser::many() { + return Many(*this); } template - RepeatN Parser::many(size_t n) { - return RepeatN(this, n); + RepeatN Parser::many(size_t n) { + return RepeatN(this, n); } template @@ -112,8 +117,8 @@ namespace hammer { } template - RepeatN Parser::operator[](size_t n) { - return RepeatN(this, n); + RepeatN Parser::operator[](size_t n) { + return RepeatN(this, n); } IntRange Int64::in_range(const int64_t lower, const int64_t upper) { diff --git a/src/hammer.hxx b/src/hammer.hxx index 287ca21..8f3200c 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -69,9 +69,9 @@ namespace hammer { /* forward declarations */ template class Many; - template class Many1; + class Many1; template class Optional; - template class RepeatN; + class RepeatN; template class IntRange; template @@ -81,12 +81,12 @@ namespace hammer { result_type parse(const string &input); result_type parse(const uint8_t *input, size_t length); Many many(); - RepeatN many(size_t n); + RepeatN many(size_t n); Optional optional(); - RepeatN operator[](size_t n); - const HParser* parser() { return _parser; } + RepeatN operator[](size_t n); + HParser* parser() { return _parser; } protected: - const HParser* _parser; + HParser* _parser; Parser() { } // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... }; @@ -105,6 +105,7 @@ namespace hammer { class Ch : public Parser { public: + friend class Parser; Ch(const uint8_t c) : _c(c) { _parser = h_ch(c); } @@ -378,24 +379,22 @@ namespace hammer { Parser _p; }; - template class Many1 : public Parser { public: - Many1(Parser &p) : _p(p) { + Many1(Parser &p) : _p(p) { _parser = h_many1(p.parser()); } private: - Parser _p; + Parser _p; }; - template class RepeatN: public Parser { public: - RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { + RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { _parser = h_repeat_n(p.parser(), n); } private: - Parser _p; + Parser _p; size_t _n; }; @@ -410,36 +409,31 @@ namespace hammer { Parser _p; }; - template class Ignore : public Parser { public: - Ignore(Parser &p) : _p(p) { + Ignore(Parser &p) : _p(p) { _parser = h_ignore(p.parser()); } private: - Parser _p; + Parser _p; }; - template class SepBy : public Parser { public: - SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { + SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { _parser = h_sepBy(p.parser(), sep.parser()); } private: - Parser _p; - Parser _sep; + Parser _p, _sep; }; - template class SepBy1 : public Parser { public: - SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { + SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { _parser = h_sepBy1(p.parser(), sep.parser()); } private: - Parser _p; - Parser _sep; + Parser _p, _sep; }; class Epsilon : public Parser { @@ -462,29 +456,28 @@ namespace hammer { /* FIXME attr_bool */ - template class And : public Parser { public: - And(Parser &p) : _p(p) { + And(Parser &p) : _p(p) { _parser = h_and(p.parser()); } private: - Parser _p; + Parser _p; }; - template class Not : public Parser { public: Not(Parser &p) : _p(p) { _parser = h_not(p.parser()); } private: - Parser _p; + Parser _p; }; template class Indirect : public Parser { public: + typedef typename T::result_type result_type; Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); From 6a29277db492eccfd3c2e9c75507352002c82d51 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 3 Sep 2013 05:32:21 +0200 Subject: [PATCH 04/88] C++ tests set up. They probably don't compile; need to write iostream ops for prettyprinting and set up C++ build in scons. --- src/hammer.hxx | 13 + src/pprint.c | 20 +- src/t_cplusplus.cxx | 565 ++++++++++++++++++++++++++++++++++++++++++++ src/t_parser.c | 362 ++++++++++++++-------------- src/test_suite.c | 6 + src/test_suite.h | 70 +++++- 6 files changed, 851 insertions(+), 185 deletions(-) create mode 100644 src/t_cplusplus.cxx diff --git a/src/hammer.hxx b/src/hammer.hxx index 8f3200c..c0c39c5 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,6 +72,8 @@ namespace hammer { class Many1; template class Optional; class RepeatN; + class Ignore; + class Indirect; template class IntRange; template @@ -83,8 +85,12 @@ namespace hammer { Many many(); RepeatN many(size_t n); Optional optional(); + Ignore ignore(); RepeatN operator[](size_t n); HParser* parser() { return _parser; } + int compile(HParserBackend backend, const void* params) { + return h_compile(_parser, backend, params); + } protected: HParser* _parser; Parser() { } @@ -478,10 +484,17 @@ namespace hammer { class Indirect : public Parser { public: typedef typename T::result_type result_type; + /* Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); } + */ + Indirect() : _p(0) {} + bind(Parser &p) { + this->_parser = h_indirect(); + h_bind_indirect(this->_parser, p.parser()); + } private: Parser _p; }; diff --git a/src/pprint.c b/src/pprint.c index d8b22e2..969476c 100644 --- a/src/pprint.c +++ b/src/pprint.c @@ -22,6 +22,14 @@ #include "internal.h" #include +#ifdef __cplusplus +#include "hammer.hxx" +#include +#include + +using std::string; using std::stringstream; +#endif + typedef struct pp_state { int delta; int indent_amt; @@ -172,7 +180,13 @@ char* h_write_result_unamb__m(HAllocator* mm__, const HParsedToken* tok) { unamb_sub(tok, &buf); append_buf_c(&buf, 0); return buf.output; + +#ifdef __cplusplus +template +string& h_write_result_unamb(ParseResult tok) { + stringstream s; + s << boost::get(tok.result()[0]); + return s.str(); +} +#endif } - - - diff --git a/src/t_cplusplus.cxx b/src/t_cplusplus.cxx new file mode 100644 index 0000000..88e0c29 --- /dev/null +++ b/src/t_cplusplus.cxx @@ -0,0 +1,565 @@ +#include "hammer.hxx" +#include "test_suite.h" +#include +#include +#include +//#include +//#include + +static void test_token_cxx(gconstpointer backend) { + hammer::Token token_ = hammer::Token("95\xa2"); + + g_check_parse_ok_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95\xa2", 3, "<39.35.a2>"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95", 2); +} + +static void test_ch_cxx(gconstpointer backend) { + hammer::Ch ch_ = hammer::Ch(0xa2); + + g_check_parse_ok_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa2", "u0xa2"); // FIXME this will fail, c++ pretty-printing doesn't do this yet + g_check_parse_failed_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa3"); +} + +static void test_ch_range_cxx(gconstpointer backend) { + hammer::ChRange range_ = hammer::ChRange('a', 'c'); + + g_check_parse_ok_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "b"); + g_check_parse_failed_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); +} + +static void test_int64_cxx(gconstpointer backend) { + hammer::Int64 int64_ = hammer::Int64(); + + g_check_parse_ok_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00\x00", "s-0x200000000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00"); +} + +static void test_int32_cxx(gconstpointer backend) { + hammer::Int32 int32_ = hammer::Int32(); + + g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00\x00", "s-0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00"); + + g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "s0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00"); +} + +static void test_int16_cxx(gconstpointer backend) { + hammer::Int16 int16_ = hammer::Int16(); + + g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe\x00", "s-0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe"); + + g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "s0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); +} + +static void test_int8_cxx(gconstpointer backend) { + hammer::Int8 int8_ = hammer::Int8(); + + g_check_parse_ok_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x88", "s-0x78"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_uint64_cxx(gconstpointer backend) { + hammer::Uint64 uint64_ = hammer::Uint64(); + + g_check_parse_ok_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00\x00", "u0x200000000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00"); +} + +static void test_uint32_cxx(gconstpointer backend) { + hammer::Uint32 uint32_ = hammer::Uint32(); + + g_check_parse_ok_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "u0x20000"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00", 3); +} + +static void test_uint16_cxx(gconstpointer backend) { + hammer::Uint16 uint16_ = hammer::Uint16(); + + g_check_parse_ok_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "u0x200"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); +} + +static void test_uint8_cxx(gconstpointer backend) { + hammer::Uint8 uint8_ = hammer::Uint8(); + + g_check_parse_ok_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x78", "u0x78"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_int_range_cxx(gconstpointer backend) { + hammer::IntRange int_range_ = hammer::Uint8().in_range(3, 10); + + g_check_parse_ok_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\x05", "u0x5"); // FIXME this will fail because pretty-printing + g_check_parse_failed_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\xb"); +} + +static void test_whitespace_cxx(gconstpointer backend) { + hammer::Whitespace whitespace_ = hammer::Whitespace(hammer::Ch('a')); + hammer::Whitespace whitespace_end = hammer::Whitespace(hammer::End()); + + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "\ta", "u0x61"); + g_check_parse_failed_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "_a"); + + g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend), "", "NULL"); + g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," ", "NULL"); + g_check_parse_failed_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," x"); +} + +static void test_left_cxx(gconstpointer backend) { + hammer::Left left_ = hammer::Left(hammer::Ch('a'), hammer::Ch(' ')); + + g_check_parse_ok_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a ", "u0x61"); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); +} + +static void test_right_cxx(gconstpointer backend) { + hammer::Right right_ = hammer::Right(hammer::Ch(' '), hammer::Ch('a')); + + g_check_parse_ok_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "ba"); +} + +static void test_middle_cxx(gconstpointer backend) { + hammer::Middle middle_ = hammer::Middle(hammer::Ch(' '), hammer::Ch('a'), hammer::Ch(' ')); + + g_check_parse_ok_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a ", "u0x61"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a"); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " b "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "ba "); + g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " ab"); +} + +static void test_in_cxx(gconstpointer backend) { + std::string options = "abc"; + hammer::In in_ = hammer::In(options); + g_check_parse_ok_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); + g_check_parse_failed_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); + +} + +static void test_not_in_cxx(gconstpointer backend) { + std::string options = "abc"; + hammer::NotIn not_in_ = hammer::NotIn(options); + g_check_parse_ok_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "u0x64"); + g_check_parse_failed_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + +} + +static void test_end_cxx(gconstpointer backend) { + hammer::End end_ = hammer::Sequence(hammer::Ch('a'), hammer::End()); + g_check_parse_ok_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_failed_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "aa"); +} + +static void test_nothing_cxx(gconstpointer backend) { + hammer::Nothing nothing_ = hammer::Nothing(); + g_check_parse_failed_cxx(nothing_, (HParserBackend)GPOINTER_TO_INT(backend),"a"); +} + +static void test_sequence_cxx(gconstpointer backend) { + hammer::Sequence sequence_1 = hammer::Sequence(hammer::Ch('a'), hammer::Ch('b')); + hammer::Sequence sequence_2 = hammer::Sequence(hammer::Ch('a'), hammer::Whitespace(hammer::Ch('b'))); + + g_check_parse_ok_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a"); + g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "b"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); +} + +static void test_choice_cxx(gconstpointer backend) { + hammer::Choice choice_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')); + + g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); + g_check_parse_failed_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "c"); +} + +static void test_butnot_cxx(gconstpointer backend) { + hammer::ButNot butnot_1 = hammer::ButNot(hammer::Ch('a'), hammer::Token("ab")); + hammer::ButNot butnot_2 = hammer::ButNot(hammer::ChRange('0', '9'), hammer::Ch('6')); + + g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_failed_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); + g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "u0x61"); + g_check_parse_failed_cxx(butnot_2, (HParserBackend)GPOINTER_TO_INT(backend), "6"); +} + +static void test_difference_cxx(gconstpointer backend) { + hammer::Difference difference_ = hammer::Difference(hammer::Token("ab"), hammer::Ch('a')); + + g_check_parse_ok_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "<61.62>"); + g_check_parse_failed_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); +} + +static void test_xor_cxx(gconstpointer backend) { + hammer::Xor xor_ = hammer::Xor(hammer::ChRange('0', '6'), hammer::ChRange('5', '9')); + + g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "0", "u0x30"); + g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "9", "u0x39"); + g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "5"); + g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); +} + +static void test_many_cxx(gconstpointer backend) { + hammer::Many many_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many(); + + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); + g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); +} + +static void test_many1_cxx(gconstpointer backend) { + hammer::Many1 many1_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many1(); + + g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); + g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); + g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "daabbabadef"); +} + +static void test_repeat_n_cxx(gconstpointer backend) { + hammer::RepeatN repeat_n_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b'))[2]; + + g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "adef"); + g_check_parse_ok_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "abdef", "(u0x61 u0x62)"); + g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "dabdef"); +} + +static void test_optional_cxx(gconstpointer backend) { + hammer::Optional optional_ = hammer::Sequence(hammer::Ch('a'), + hammer::Choice(hammer::Ch('b'), hammer::Ch('c')).optional(), + hammer::ch('d')); + + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "abd", "(u0x61 u0x62 u0x64)"); + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "acd", "(u0x61 u0x63 u0x64)"); + g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ad", "(u0x61 null u0x64)"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "aed"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); + g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); +} + +static void test_ignore_cxx(gconstpointer backend) { + hammer::Ignore ignore_ = hammer::Sequence(hammer::Ch('a'), + hammer::ch('b').ignore(), + hammer::ch('c')); + + g_check_parse_ok_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "abc", "(u0x61 u0x63)"); + g_check_parse_failed_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); +} + +static void test_sepBy_cxx(gconstpointer backend) { + hammer::SepBy sepBy_ = hammer::SepBy(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); + + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); + g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); +} + +static void test_sepBy1_cxx(gconstpointer backend) { + hammer::SepBy1 sepBy1_ = hammer::SepBy1(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); + + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); + g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); + g_check_parse_failed_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); +} + +static void test_epsilon_cxx(gconstpointer backend) { + hammer::Epsilon epsilon_p_1 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon(), hammer::Ch('b')); + hammer::Epsilon epsilon_p_2 = hammer::Sequence(hammer::Epsilon(), hammer::Ch('a')); + hammer::Epsilon epsilon_p_3 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon()); + + g_check_parse_ok_cxx(epsilon_p_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); + g_check_parse_ok_cxx(epsilon_p_2, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); +} + +static void test_and_cxx(gconstpointer backend) { + hammer::And and_1 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('0')); + hammer::And and_2 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('1')); + hammer::And and_3 = hammer::Sequence(hammer::Ch('1'), hammer::And(hammer::Ch('2'))); + + g_check_parse_ok_cxx(and_1, (HParserBackend)GPOINTER_TO_INT(backend), "0", "(u0x30)"); + g_check_parse_failed_cxx(and_2, (HParserBackend)GPOINTER_TO_INT(backend), "0"); + g_check_parse_ok_cxx(and_3, (HParserBackend)GPOINTER_TO_INT(backend), "12", "(u0x31)"); +} + +static void test_not_cxx(gconstpointer backend) { + hammer::Not not_1 = hammer::Sequence(hammer::Ch('a'), hammer::Choice(hammer::Ch('+'), hammer::Token("++")), hammer::Ch('b')); + hammer::Not not_2 = hammer::Sequence(hammer::Ch('a'), + hammer::Choice(hammer::Sequence(hammer::Ch('+'), + hammer::Not(hammer::Ch('+'))), + hammer::Token("++")), + hammer::Ch('b')); + + g_check_parse_ok_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 u0x2b u0x62)"); + g_check_parse_failed_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a++b"); + g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 (u0x2b) u0x62)"); + g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", "(u0x61 <2b.2b> u0x62)"); +} + +static void test_leftrec(gconstpointer backend) { + hammer::Ch a_ = hammer::Ch('a'); + + hammer::Indirect lr_ = hammer::Indirect(); + + lr_.bind(hammer::Choice(hammer::Sequence(lr_, a_), a_)); + + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 u0x61)"); + g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "((u0x61 u0x61) u0x61)"); +} + +static void test_rightrec(gconstpointer backend) { + hammer::Ch a_ = hammer::Ch('a'); + + hammer::Indirect rr_ = hammer::Indirect(); + rr_.bind(hammer::Choice(hammer::Sequence(a_, rr_), hammer::Epsilon())); + + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 (u0x61))"); + g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "(u0x61 (u0x61 (u0x61)))"); +} + +// FIXME won't be ready until action is ready +/* +static void test_ambiguous(gconstpointer backend) { + hammer::Ch d_ = hammer::Ch('d'); + hammer::Ch p_ = hammer::Ch('+'); + hammer::Indirect E_ = hammer::Indirect(); + E_.bind(hammer::Choice(hammer::Sequence(E_, p_, E_), d_)); + hammer::Action expr_ = hammer::Action(E_, act_flatten); + + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "(u0x64)"); + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", "(u0x64 u0x2b u0x64)"); + g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", "(u0x64 u0x2b u0x64 u0x2b u0x64)"); + g_check_parse_failed_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+"); +} +*/ + +void register_cxx_tests(void) { + g_test_add_data_func("/core/c++/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range_cxx); + g_test_add_data_func("/core/c++/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left_cxx); + g_test_add_data_func("/core/c++/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right_cxx); + g_test_add_data_func("/core/c++/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle_cxx); + g_test_add_data_func("/core/c++/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in_cxx); + g_test_add_data_func("/core/c++/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/packrat/end", GINT_TO_POINTER(PB_PACKRAT), test_end_cxx); + g_test_add_data_func("/core/c++/parser/packrat/nothing", GINT_TO_POINTER(PB_PACKRAT), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot_cxx); + g_test_add_data_func("/core/c++/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference_cxx); + g_test_add_data_func("/core/c++/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor_cxx); + g_test_add_data_func("/core/c++/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many_cxx); + g_test_add_data_func("/core/c++/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n_cxx); + g_test_add_data_func("/core/c++/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/packrat/epsilon", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_cxx); + g_test_add_data_func("/core/c++/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and_cxx); + g_test_add_data_func("/core/c++/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not_cxx); + // g_test_add_data_func("/core/c++/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec_cxx); + + g_test_add_data_Func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); + g_test_add_data_func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); + g_test_add_data_func("/core/c++/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left_cxx); + g_test_add_data_func("/core/c++/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right_cxx); + g_test_add_data_func("/core/c++/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle_cxx); + //g_test_add_data_func("/core/c++/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action_cxx); + g_test_add_data_func("/core/c++/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in_cxx); + g_test_add_data_func("/core/c++/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/llk/end", GINT_TO_POINTER(PB_LLk), test_end_cxx); + g_test_add_data_func("/core/c++/parser/llk/nothing", GINT_TO_POINTER(PB_LLk), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many_cxx); + g_test_add_data_func("/core/c++/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/llk/epsilon", GINT_TO_POINTER(PB_LLk), test_epsilon_cxx); + //g_test_add_data_func("/core/c++/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore_cxx); + //g_test_add_data_func("/core/c++/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec_cxx); + + g_test_add_data_func("/core/c++/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle_cxx); + // g_test_add_data_func("/core/c++/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/regex/end", GINT_TO_POINTER(PB_REGULAR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/regex/nothing", GINT_TO_POINTER(PB_REGULAR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n_cxx); + g_test_add_data_func("/core/c++/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/regex/epsilon", GINT_TO_POINTER(PB_REGULAR), test_epsilon_cxx); + // g_test_add_data_func("/core/c++/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore_cxx); + + g_test_add_data_func("/core/c++/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle_cxx); + // g_test_add_data_func("/core/c++/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/lalr/end", GINT_TO_POINTER(PB_LALR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/lalr/nothing", GINT_TO_POINTER(PB_LALR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/lalr/epsilon", GINT_TO_POINTER(PB_LALR), test_epsilon_cxx); + // g_test_add_data_func("/core/c++/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec_cxx); + + g_test_add_data_func("/core/c++/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token_cxx); + g_test_add_data_func("/core/c++/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch_cxx); + g_test_add_data_func("/core/c++/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range_cxx); + g_test_add_data_func("/core/c++/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64_cxx); + g_test_add_data_func("/core/c++/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32_cxx); + g_test_add_data_func("/core/c++/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16_cxx); + g_test_add_data_func("/core/c++/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16_cxx); + g_test_add_data_func("/core/c++/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8_cxx); + g_test_add_data_func("/core/c++/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range_cxx); +#if 0 + g_test_add_data_func("/core/c++/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64_cxx); + g_test_add_data_func("/core/c++/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32_cxx); +#endif + g_test_add_data_func("/core/c++/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace_cxx); + g_test_add_data_func("/core/c++/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left_cxx); + g_test_add_data_func("/core/c++/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right_cxx); + g_test_add_data_func("/core/c++/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle_cxx); + g_test_add_data_func("/core/c++/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action_cxx); + g_test_add_data_func("/core/c++/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in_cxx); + g_test_add_data_func("/core/c++/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in_cxx); + g_test_add_data_func("/core/c++/parser/glr/end", GINT_TO_POINTER(PB_GLR), test_end_cxx); + g_test_add_data_func("/core/c++/parser/glr/nothing", GINT_TO_POINTER(PB_GLR), test_nothing_cxx); + g_test_add_data_func("/core/c++/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence_cxx); + g_test_add_data_func("/core/c++/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice_cxx); + g_test_add_data_func("/core/c++/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many_cxx); + g_test_add_data_func("/core/c++/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1_cxx); + g_test_add_data_func("/core/c++/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional_cxx); + g_test_add_data_func("/core/c++/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy_cxx); + g_test_add_data_func("/core/c++/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1_cxx); + g_test_add_data_func("/core/c++/parser/glr/epsilon", GINT_TO_POINTER(PB_GLR), test_epsilon_cxx); + g_test_add_data_func("/core/c++/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool_cxx); + g_test_add_data_func("/core/c++/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore_cxx); + g_test_add_data_func("/core/c++/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec_cxx); + g_test_add_data_func("/core/c++/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec_cxx); + g_test_add_data_func("/core/c++/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous_cxx); + +} + +/* +int main(int argc, char **argv) { + hammer::Ch p = hammer::Ch('a'); + std::string test = "a"; + hammer::UintResult r = p.parse(test); + std::cerr << r.result() << std::endl; + hammer::SequenceResult s = p.many().parse(test); + hammer::UintResult u = boost::get(s.result()[0]); + std::cerr << u.result() << std::endl; + return 0; +} +*/ diff --git a/src/t_parser.c b/src/t_parser.c index 59adf36..c49e443 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -442,199 +442,199 @@ static void test_ambiguous(gconstpointer backend) { } void register_parser_tests(void) { - g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); - g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); - g_test_add_data_func("/core/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); - g_test_add_data_func("/core/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); - g_test_add_data_func("/core/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); - g_test_add_data_func("/core/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); - g_test_add_data_func("/core/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); - g_test_add_data_func("/core/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); - g_test_add_data_func("/core/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); - g_test_add_data_func("/core/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); - g_test_add_data_func("/core/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); - g_test_add_data_func("/core/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); + g_test_add_data_func("/core/c/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); + g_test_add_data_func("/core/c/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); + g_test_add_data_func("/core/c/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); + g_test_add_data_func("/core/c/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); + g_test_add_data_func("/core/c/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); + g_test_add_data_func("/core/c/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); + g_test_add_data_func("/core/c/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); + g_test_add_data_func("/core/c/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); + g_test_add_data_func("/core/c/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); + g_test_add_data_func("/core/c/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); + g_test_add_data_func("/core/c/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); + g_test_add_data_func("/core/c/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); #if 0 - g_test_add_data_func("/core/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); - g_test_add_data_func("/core/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); + g_test_add_data_func("/core/c/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); + g_test_add_data_func("/core/c/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); #endif - g_test_add_data_func("/core/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); - g_test_add_data_func("/core/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); - g_test_add_data_func("/core/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); - g_test_add_data_func("/core/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); - g_test_add_data_func("/core/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); - g_test_add_data_func("/core/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); - g_test_add_data_func("/core/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); - g_test_add_data_func("/core/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); - g_test_add_data_func("/core/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); - g_test_add_data_func("/core/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); - g_test_add_data_func("/core/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); - g_test_add_data_func("/core/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); - g_test_add_data_func("/core/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); - g_test_add_data_func("/core/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); - g_test_add_data_func("/core/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); - g_test_add_data_func("/core/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); - g_test_add_data_func("/core/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); - g_test_add_data_func("/core/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); - g_test_add_data_func("/core/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); - g_test_add_data_func("/core/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); - g_test_add_data_func("/core/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); - g_test_add_data_func("/core/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); - g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); - g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); - g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); - g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); + g_test_add_data_func("/core/c/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); + g_test_add_data_func("/core/c/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); + g_test_add_data_func("/core/c/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); + g_test_add_data_func("/core/c/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); + g_test_add_data_func("/core/c/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); + g_test_add_data_func("/core/c/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); + g_test_add_data_func("/core/c/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); + g_test_add_data_func("/core/c/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); + g_test_add_data_func("/core/c/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); + g_test_add_data_func("/core/c/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); + g_test_add_data_func("/core/c/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); + g_test_add_data_func("/core/c/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); + g_test_add_data_func("/core/c/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); + g_test_add_data_func("/core/c/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); + g_test_add_data_func("/core/c/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); + g_test_add_data_func("/core/c/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); + g_test_add_data_func("/core/c/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); + g_test_add_data_func("/core/c/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); + g_test_add_data_func("/core/c/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); + g_test_add_data_func("/core/c/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); + g_test_add_data_func("/core/c/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); + g_test_add_data_func("/core/c/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); + g_test_add_data_func("/core/c/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); + g_test_add_data_func("/core/c/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); + g_test_add_data_func("/core/c/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); + // g_test_add_data_func("/core/c/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + g_test_add_data_func("/core/c/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); - g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); - g_test_add_data_func("/core/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); - g_test_add_data_func("/core/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); - g_test_add_data_func("/core/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); - g_test_add_data_func("/core/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); - g_test_add_data_func("/core/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); - g_test_add_data_func("/core/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); - g_test_add_data_func("/core/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); - g_test_add_data_func("/core/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); - g_test_add_data_func("/core/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); - g_test_add_data_func("/core/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); - g_test_add_data_func("/core/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); + g_test_add_data_func("/core/c/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); + g_test_add_data_func("/core/c/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); + g_test_add_data_func("/core/c/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); + g_test_add_data_func("/core/c/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); + g_test_add_data_func("/core/c/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); + g_test_add_data_func("/core/c/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); + g_test_add_data_func("/core/c/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); + g_test_add_data_func("/core/c/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); + g_test_add_data_func("/core/c/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); + g_test_add_data_func("/core/c/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); + g_test_add_data_func("/core/c/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); + g_test_add_data_func("/core/c/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); #if 0 - g_test_add_data_func("/core/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); - g_test_add_data_func("/core/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); + g_test_add_data_func("/core/c/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); + g_test_add_data_func("/core/c/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); #endif - g_test_add_data_func("/core/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); - g_test_add_data_func("/core/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); - g_test_add_data_func("/core/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); - g_test_add_data_func("/core/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); - g_test_add_data_func("/core/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); - g_test_add_data_func("/core/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); - g_test_add_data_func("/core/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); - g_test_add_data_func("/core/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); - g_test_add_data_func("/core/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); - g_test_add_data_func("/core/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); - g_test_add_data_func("/core/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); - g_test_add_data_func("/core/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); - g_test_add_data_func("/core/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); - g_test_add_data_func("/core/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); - g_test_add_data_func("/core/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); - g_test_add_data_func("/core/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); - g_test_add_data_func("/core/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); - g_test_add_data_func("/core/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); - g_test_add_data_func("/core/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); - //g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); - g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); + g_test_add_data_func("/core/c/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); + g_test_add_data_func("/core/c/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); + g_test_add_data_func("/core/c/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); + g_test_add_data_func("/core/c/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); + g_test_add_data_func("/core/c/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); + g_test_add_data_func("/core/c/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); + g_test_add_data_func("/core/c/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); + g_test_add_data_func("/core/c/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); + g_test_add_data_func("/core/c/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); + g_test_add_data_func("/core/c/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); + g_test_add_data_func("/core/c/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); + g_test_add_data_func("/core/c/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); + g_test_add_data_func("/core/c/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); + g_test_add_data_func("/core/c/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); + g_test_add_data_func("/core/c/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); + g_test_add_data_func("/core/c/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); + g_test_add_data_func("/core/c/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); + g_test_add_data_func("/core/c/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); + g_test_add_data_func("/core/c/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); + //g_test_add_data_func("/core/c/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); + g_test_add_data_func("/core/c/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); - g_test_add_data_func("/core/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); - g_test_add_data_func("/core/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); - g_test_add_data_func("/core/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); - g_test_add_data_func("/core/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); - g_test_add_data_func("/core/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); - g_test_add_data_func("/core/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); - g_test_add_data_func("/core/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); - g_test_add_data_func("/core/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); - g_test_add_data_func("/core/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); - g_test_add_data_func("/core/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); - g_test_add_data_func("/core/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); - g_test_add_data_func("/core/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); + g_test_add_data_func("/core/c/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); + g_test_add_data_func("/core/c/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); + g_test_add_data_func("/core/c/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); + g_test_add_data_func("/core/c/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); + g_test_add_data_func("/core/c/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); + g_test_add_data_func("/core/c/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); + g_test_add_data_func("/core/c/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); + g_test_add_data_func("/core/c/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); + g_test_add_data_func("/core/c/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); + g_test_add_data_func("/core/c/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); + g_test_add_data_func("/core/c/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); + g_test_add_data_func("/core/c/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); - g_test_add_data_func("/core/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); + g_test_add_data_func("/core/c/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); + g_test_add_data_func("/core/c/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); #endif - g_test_add_data_func("/core/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); - g_test_add_data_func("/core/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); - g_test_add_data_func("/core/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); - g_test_add_data_func("/core/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); - g_test_add_data_func("/core/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); - g_test_add_data_func("/core/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); - g_test_add_data_func("/core/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); - g_test_add_data_func("/core/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); - g_test_add_data_func("/core/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); - g_test_add_data_func("/core/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); - g_test_add_data_func("/core/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); - g_test_add_data_func("/core/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); - g_test_add_data_func("/core/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); - g_test_add_data_func("/core/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); - g_test_add_data_func("/core/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); - g_test_add_data_func("/core/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); - g_test_add_data_func("/core/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); - g_test_add_data_func("/core/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); - g_test_add_data_func("/core/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); - g_test_add_data_func("/core/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); + g_test_add_data_func("/core/c/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); + g_test_add_data_func("/core/c/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); + g_test_add_data_func("/core/c/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); + g_test_add_data_func("/core/c/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); + g_test_add_data_func("/core/c/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); + g_test_add_data_func("/core/c/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); + g_test_add_data_func("/core/c/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); + g_test_add_data_func("/core/c/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); + g_test_add_data_func("/core/c/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); + g_test_add_data_func("/core/c/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); + g_test_add_data_func("/core/c/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); + g_test_add_data_func("/core/c/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); + g_test_add_data_func("/core/c/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); + g_test_add_data_func("/core/c/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); + g_test_add_data_func("/core/c/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); + g_test_add_data_func("/core/c/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); + g_test_add_data_func("/core/c/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); + g_test_add_data_func("/core/c/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); + g_test_add_data_func("/core/c/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); - g_test_add_data_func("/core/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); - g_test_add_data_func("/core/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); - g_test_add_data_func("/core/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); - g_test_add_data_func("/core/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); - g_test_add_data_func("/core/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); - g_test_add_data_func("/core/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); - g_test_add_data_func("/core/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); - g_test_add_data_func("/core/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); - g_test_add_data_func("/core/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); - g_test_add_data_func("/core/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); - g_test_add_data_func("/core/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); - g_test_add_data_func("/core/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); + g_test_add_data_func("/core/c/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); + g_test_add_data_func("/core/c/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); + g_test_add_data_func("/core/c/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); + g_test_add_data_func("/core/c/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); + g_test_add_data_func("/core/c/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); + g_test_add_data_func("/core/c/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); + g_test_add_data_func("/core/c/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); + g_test_add_data_func("/core/c/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); + g_test_add_data_func("/core/c/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); + g_test_add_data_func("/core/c/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); + g_test_add_data_func("/core/c/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); + g_test_add_data_func("/core/c/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); - g_test_add_data_func("/core/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); + g_test_add_data_func("/core/c/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); + g_test_add_data_func("/core/c/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); #endif - g_test_add_data_func("/core/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); - g_test_add_data_func("/core/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); - g_test_add_data_func("/core/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); - g_test_add_data_func("/core/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); - g_test_add_data_func("/core/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); - g_test_add_data_func("/core/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); - g_test_add_data_func("/core/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); - g_test_add_data_func("/core/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); - g_test_add_data_func("/core/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); - g_test_add_data_func("/core/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); - g_test_add_data_func("/core/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); - g_test_add_data_func("/core/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); - g_test_add_data_func("/core/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); - g_test_add_data_func("/core/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); - g_test_add_data_func("/core/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); - g_test_add_data_func("/core/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); - g_test_add_data_func("/core/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); - g_test_add_data_func("/core/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); - g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); - g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); - g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); + g_test_add_data_func("/core/c/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); + g_test_add_data_func("/core/c/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); + g_test_add_data_func("/core/c/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); + g_test_add_data_func("/core/c/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); + g_test_add_data_func("/core/c/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); + g_test_add_data_func("/core/c/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); + g_test_add_data_func("/core/c/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); + g_test_add_data_func("/core/c/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); + g_test_add_data_func("/core/c/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); + g_test_add_data_func("/core/c/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); + g_test_add_data_func("/core/c/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); + g_test_add_data_func("/core/c/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); + g_test_add_data_func("/core/c/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); + g_test_add_data_func("/core/c/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); + g_test_add_data_func("/core/c/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); + g_test_add_data_func("/core/c/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); + g_test_add_data_func("/core/c/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); + g_test_add_data_func("/core/c/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); + g_test_add_data_func("/core/c/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); + g_test_add_data_func("/core/c/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); - g_test_add_data_func("/core/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); - g_test_add_data_func("/core/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); - g_test_add_data_func("/core/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); - g_test_add_data_func("/core/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); - g_test_add_data_func("/core/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); - g_test_add_data_func("/core/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); - g_test_add_data_func("/core/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); - g_test_add_data_func("/core/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); - g_test_add_data_func("/core/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); - g_test_add_data_func("/core/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); - g_test_add_data_func("/core/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); - g_test_add_data_func("/core/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); + g_test_add_data_func("/core/c/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); + g_test_add_data_func("/core/c/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); + g_test_add_data_func("/core/c/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); + g_test_add_data_func("/core/c/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); + g_test_add_data_func("/core/c/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); + g_test_add_data_func("/core/c/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); + g_test_add_data_func("/core/c/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); + g_test_add_data_func("/core/c/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); + g_test_add_data_func("/core/c/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); + g_test_add_data_func("/core/c/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); + g_test_add_data_func("/core/c/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); + g_test_add_data_func("/core/c/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); #if 0 - g_test_add_data_func("/core/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); - g_test_add_data_func("/core/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); + g_test_add_data_func("/core/c/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); + g_test_add_data_func("/core/c/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); #endif - g_test_add_data_func("/core/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); - g_test_add_data_func("/core/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); - g_test_add_data_func("/core/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); - g_test_add_data_func("/core/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); - g_test_add_data_func("/core/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); - g_test_add_data_func("/core/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); - g_test_add_data_func("/core/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); - g_test_add_data_func("/core/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); - g_test_add_data_func("/core/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); - g_test_add_data_func("/core/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); - g_test_add_data_func("/core/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); - g_test_add_data_func("/core/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); - g_test_add_data_func("/core/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); - g_test_add_data_func("/core/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); - g_test_add_data_func("/core/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); - g_test_add_data_func("/core/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); - g_test_add_data_func("/core/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); - g_test_add_data_func("/core/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); - g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); - g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); - g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); - g_test_add_data_func("/core/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); + g_test_add_data_func("/core/c/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); + g_test_add_data_func("/core/c/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); + g_test_add_data_func("/core/c/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); + g_test_add_data_func("/core/c/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); + g_test_add_data_func("/core/c/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); + g_test_add_data_func("/core/c/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); + g_test_add_data_func("/core/c/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); + g_test_add_data_func("/core/c/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); + g_test_add_data_func("/core/c/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); + g_test_add_data_func("/core/c/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); + g_test_add_data_func("/core/c/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); + g_test_add_data_func("/core/c/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); + g_test_add_data_func("/core/c/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); + g_test_add_data_func("/core/c/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); + g_test_add_data_func("/core/c/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); + g_test_add_data_func("/core/c/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); + g_test_add_data_func("/core/c/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); + g_test_add_data_func("/core/c/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); + g_test_add_data_func("/core/c/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); + g_test_add_data_func("/core/c/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); + g_test_add_data_func("/core/c/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); + g_test_add_data_func("/core/c/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); } diff --git a/src/test_suite.c b/src/test_suite.c index e065f13..9191f93 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -25,6 +25,9 @@ extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); extern void register_benchmark_tests(); +#ifdef __cplusplus +extern void register_cxx_tests(); +#endif int main(int argc, char** argv) { g_test_init(&argc, &argv, NULL); @@ -36,6 +39,9 @@ int main(int argc, char** argv) { register_grammar_tests(); register_misc_tests(); register_benchmark_tests(); + #ifdef __cplusplus + register_cxx_tests(); + #endif g_test_run(); } diff --git a/src/test_suite.h b/src/test_suite.h index fc008e7..76c6d78 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -19,6 +19,16 @@ #define HAMMER_TEST_SUITE__H #include +#ifdef __cplusplus +extern "C" { +#include +#include +#endif + +#ifndef __cplusplus +typedef int bool; +#endif + // Equivalent to g_assert_*, but not using g_assert... #define g_check_inttype(fmt, typ, n1, op, n2) do { \ typ _n1 = (n1); \ @@ -52,6 +62,19 @@ } \ } while(0) +#ifdef __cplusplus +#define g_check_string_cxx(n1, op, n2) do { \ + std::string _n1 = (n1); \ + std::string _n2 = (n2); \ + if (!(_n1.compare(_n2) op 0)) { \ + g_test_message("Check failed: (%s) (%s %s %s)", \ + #n1 " " #op " " #n2, \ + _n1.c_str(), #op, _n2.c_str()); \ + g_test_fail(); \ + } \ + } while(0) +#endif + #define g_check_regular(lang) do { \ if (!lang->isValidRegular(lang->env)) { \ g_test_message("Language is not regular"); \ @@ -95,6 +118,21 @@ g_test_fail(); \ } \ } while(0) + +#ifdef __cplusplus +#define g_check_parse_failed_cxx(parser, backend, input) do { \ + int skip = parser.compile((HParserBackend)backend, NULL); \ + if(skip != 0) { \ + g_test_message("Backend not applicable, skipping test"); \ + break; \ + } \ + parser::result_type result = parser.parse(input); \ + if (NULL != result) { \ + g_test_message("Check failed: shouldn't have succeeded, but did"); \ + g_test_fail(); \ + } \ + } while(0) +#endif #define g_check_parse_ok(parser, backend, input, inp_len, result) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ @@ -120,6 +158,34 @@ } \ } while(0) +#ifdef __cplusplus +#define g_check_parse_ok_cxx(parser, backend, input, result) do { \ + int skip = parser.compile((HParserBackend)backend, NULL); \ + if(skip) { \ + g_test_message("Backend not applicable, skipping test"); \ + break; \ + } \ + parser::result_type res = parser.parse((std::string)input); \ + if (!res) { \ + g_test_message("Parse failed on line %d", __LINE__); \ + g_test_fail(); \ + } else { \ + std::string& cres = h_write_result_unamb(res); \ + g_check_string_cxx(cres, ==, (std::string)result); \ + /* + system_allocator.free(&system_allocator, cres); \ + HArenaStats stats; \ + h_allocator_stats(res->arena, &stats); \ + g_test_message("Parse used %zd bytes, wasted %zd bytes. " \ + "Inefficiency: %5f%%", \ + stats.used, stats.wasted, \ + stats.wasted * 100. / (stats.used+stats.wasted)); \ + h_delete_arena(res->arena); \ + */ // FIXME might've made a memory leak there + } \ + } while(0) +#endif + #define g_check_hashtable_present(table, key) do { \ if(!h_hashtable_present(table, key)) { \ g_test_message("Check failed: key should have been in table, but wasn't"); \ @@ -197,6 +263,8 @@ #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) - +#ifdef __cplusplus +} +#endif #endif // #ifndef HAMMER_TEST_SUITE__H From 0db4925469927323e0013199aec67069ff8b2923 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 3 Sep 2013 05:33:30 +0200 Subject: [PATCH 05/88] Revert "C++ tests set up. They probably don't compile; need to write iostream ops for prettyprinting and set up C++ build in scons." Should have branched it. This reverts commit 6a29277db492eccfd3c2e9c75507352002c82d51. --- src/hammer.hxx | 13 - src/pprint.c | 20 +- src/t_cplusplus.cxx | 565 -------------------------------------------- src/t_parser.c | 362 ++++++++++++++-------------- src/test_suite.c | 6 - src/test_suite.h | 70 +----- 6 files changed, 185 insertions(+), 851 deletions(-) delete mode 100644 src/t_cplusplus.cxx diff --git a/src/hammer.hxx b/src/hammer.hxx index c0c39c5..8f3200c 100644 --- a/src/hammer.hxx +++ b/src/hammer.hxx @@ -72,8 +72,6 @@ namespace hammer { class Many1; template class Optional; class RepeatN; - class Ignore; - class Indirect; template class IntRange; template @@ -85,12 +83,8 @@ namespace hammer { Many many(); RepeatN many(size_t n); Optional optional(); - Ignore ignore(); RepeatN operator[](size_t n); HParser* parser() { return _parser; } - int compile(HParserBackend backend, const void* params) { - return h_compile(_parser, backend, params); - } protected: HParser* _parser; Parser() { } @@ -484,17 +478,10 @@ namespace hammer { class Indirect : public Parser { public: typedef typename T::result_type result_type; - /* Indirect(Parser &p) : _p(p) { this->_parser = h_indirect(); h_bind_indirect(this->_parser, p.parser()); } - */ - Indirect() : _p(0) {} - bind(Parser &p) { - this->_parser = h_indirect(); - h_bind_indirect(this->_parser, p.parser()); - } private: Parser _p; }; diff --git a/src/pprint.c b/src/pprint.c index 969476c..d8b22e2 100644 --- a/src/pprint.c +++ b/src/pprint.c @@ -22,14 +22,6 @@ #include "internal.h" #include -#ifdef __cplusplus -#include "hammer.hxx" -#include -#include - -using std::string; using std::stringstream; -#endif - typedef struct pp_state { int delta; int indent_amt; @@ -180,13 +172,7 @@ char* h_write_result_unamb__m(HAllocator* mm__, const HParsedToken* tok) { unamb_sub(tok, &buf); append_buf_c(&buf, 0); return buf.output; +} + + -#ifdef __cplusplus -template -string& h_write_result_unamb(ParseResult tok) { - stringstream s; - s << boost::get(tok.result()[0]); - return s.str(); -} -#endif -} diff --git a/src/t_cplusplus.cxx b/src/t_cplusplus.cxx deleted file mode 100644 index 88e0c29..0000000 --- a/src/t_cplusplus.cxx +++ /dev/null @@ -1,565 +0,0 @@ -#include "hammer.hxx" -#include "test_suite.h" -#include -#include -#include -//#include -//#include - -static void test_token_cxx(gconstpointer backend) { - hammer::Token token_ = hammer::Token("95\xa2"); - - g_check_parse_ok_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95\xa2", 3, "<39.35.a2>"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(token_, (HParserBackend)GPOINTER_TO_INT(backend), "95", 2); -} - -static void test_ch_cxx(gconstpointer backend) { - hammer::Ch ch_ = hammer::Ch(0xa2); - - g_check_parse_ok_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa2", "u0xa2"); // FIXME this will fail, c++ pretty-printing doesn't do this yet - g_check_parse_failed_cxx(ch_, (HParserBackend)GPOINTER_TO_INT(backend), "\xa3"); -} - -static void test_ch_range_cxx(gconstpointer backend) { - hammer::ChRange range_ = hammer::ChRange('a', 'c'); - - g_check_parse_ok_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "b"); - g_check_parse_failed_cxx(range_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); -} - -static void test_int64_cxx(gconstpointer backend) { - hammer::Int64 int64_ = hammer::Int64(); - - g_check_parse_ok_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00\x00", "s-0x200000000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int64_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xff\xff\xfe\x00\x00\x00"); -} - -static void test_int32_cxx(gconstpointer backend) { - hammer::Int32 int32_ = hammer::Int32(); - - g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00\x00", "s-0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\xff\xfe\x00"); - - g_check_parse_ok_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "s0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00"); -} - -static void test_int16_cxx(gconstpointer backend) { - hammer::Int16 int16_ = hammer::Int16(); - - g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe\x00", "s-0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\xfe"); - - g_check_parse_ok_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "s0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); -} - -static void test_int8_cxx(gconstpointer backend) { - hammer::Int8 int8_ = hammer::Int8(); - - g_check_parse_ok_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x88", "s-0x78"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_uint64_cxx(gconstpointer backend) { - hammer::Uint64 uint64_ = hammer::Uint64(); - - g_check_parse_ok_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00\x00", "u0x200000000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint64_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x00\x00\x02\x00\x00\x00"); -} - -static void test_uint32_cxx(gconstpointer backend) { - hammer::Uint32 uint32_ = hammer::Uint32(); - - g_check_parse_ok_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00\x00", "u0x20000"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint32_, (HParserBackend)GPOINTER_TO_INT(backend), "\x00\x02\x00", 3); -} - -static void test_uint16_cxx(gconstpointer backend) { - hammer::Uint16 uint16_ = hammer::Uint16(); - - g_check_parse_ok_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02\x00", "u0x200"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint16_, (HParserBackend)GPOINTER_TO_INT(backend), "\x02"); -} - -static void test_uint8_cxx(gconstpointer backend) { - hammer::Uint8 uint8_ = hammer::Uint8(); - - g_check_parse_ok_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), "\x78", "u0x78"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(uint8_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_int_range_cxx(gconstpointer backend) { - hammer::IntRange int_range_ = hammer::Uint8().in_range(3, 10); - - g_check_parse_ok_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\x05", "u0x5"); // FIXME this will fail because pretty-printing - g_check_parse_failed_cxx(int_range_, (HParserBackend)GPOINTER_TO_INT(backend), "\xb"); -} - -static void test_whitespace_cxx(gconstpointer backend) { - hammer::Whitespace whitespace_ = hammer::Whitespace(hammer::Ch('a')); - hammer::Whitespace whitespace_end = hammer::Whitespace(hammer::End()); - - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_ok_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "\ta", "u0x61"); - g_check_parse_failed_cxx(whitespace_, (HParserBackend)GPOINTER_TO_INT(backend), "_a"); - - g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend), "", "NULL"); - g_check_parse_ok_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," ", "NULL"); - g_check_parse_failed_cxx(whitespace_end, (HParserBackend)GPOINTER_TO_INT(backend)," x"); -} - -static void test_left_cxx(gconstpointer backend) { - hammer::Left left_ = hammer::Left(hammer::Ch('a'), hammer::Ch(' ')); - - g_check_parse_ok_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a ", "u0x61"); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(left_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); -} - -static void test_right_cxx(gconstpointer backend) { - hammer::Right right_ = hammer::Right(hammer::Ch(' '), hammer::Ch('a')); - - g_check_parse_ok_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " a", "u0x61"); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(right_, (HParserBackend)GPOINTER_TO_INT(backend), "ba"); -} - -static void test_middle_cxx(gconstpointer backend) { - hammer::Middle middle_ = hammer::Middle(hammer::Ch(' '), hammer::Ch('a'), hammer::Ch(' ')); - - g_check_parse_ok_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a ", "u0x61"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " a"); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "a "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " b "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), "ba "); - g_check_parse_failed_cxx(middle_, (HParserBackend)GPOINTER_TO_INT(backend), " ab"); -} - -static void test_in_cxx(gconstpointer backend) { - std::string options = "abc"; - hammer::In in_ = hammer::In(options); - g_check_parse_ok_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); - g_check_parse_failed_cxx(in_, (HParserBackend)GPOINTER_TO_INT(backend), "d"); - -} - -static void test_not_in_cxx(gconstpointer backend) { - std::string options = "abc"; - hammer::NotIn not_in_ = hammer::NotIn(options); - g_check_parse_ok_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "u0x64"); - g_check_parse_failed_cxx(not_in_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - -} - -static void test_end_cxx(gconstpointer backend) { - hammer::End end_ = hammer::Sequence(hammer::Ch('a'), hammer::End()); - g_check_parse_ok_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_failed_cxx(end_, (HParserBackend)GPOINTER_TO_INT(backend), "aa"); -} - -static void test_nothing_cxx(gconstpointer backend) { - hammer::Nothing nothing_ = hammer::Nothing(); - g_check_parse_failed_cxx(nothing_, (HParserBackend)GPOINTER_TO_INT(backend),"a"); -} - -static void test_sequence_cxx(gconstpointer backend) { - hammer::Sequence sequence_1 = hammer::Sequence(hammer::Ch('a'), hammer::Ch('b')); - hammer::Sequence sequence_2 = hammer::Sequence(hammer::Ch('a'), hammer::Whitespace(hammer::Ch('b'))); - - g_check_parse_ok_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "a"); - g_check_parse_failed_cxx(sequence_1, (HParserBackend)GPOINTER_TO_INT(backend), "b"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(sequence_2, (HParserBackend)GPOINTER_TO_INT(backend), "a b", "(u0x61 u0x62)"); -} - -static void test_choice_cxx(gconstpointer backend) { - hammer::Choice choice_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')); - - g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "u0x62"); - g_check_parse_failed_cxx(choice_, (HParserBackend)GPOINTER_TO_INT(backend), "c"); -} - -static void test_butnot_cxx(gconstpointer backend) { - hammer::ButNot butnot_1 = hammer::ButNot(hammer::Ch('a'), hammer::Token("ab")); - hammer::ButNot butnot_2 = hammer::ButNot(hammer::ChRange('0', '9'), hammer::Ch('6')); - - g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_failed_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); - g_check_parse_ok_cxx(butnot_1, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "u0x61"); - g_check_parse_failed_cxx(butnot_2, (HParserBackend)GPOINTER_TO_INT(backend), "6"); -} - -static void test_difference_cxx(gconstpointer backend) { - hammer::Difference difference_ = hammer::Difference(hammer::Token("ab"), hammer::Ch('a')); - - g_check_parse_ok_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "<61.62>"); - g_check_parse_failed_cxx(difference_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); -} - -static void test_xor_cxx(gconstpointer backend) { - hammer::Xor xor_ = hammer::Xor(hammer::ChRange('0', '6'), hammer::ChRange('5', '9')); - - g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "0", "u0x30"); - g_check_parse_ok_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "9", "u0x39"); - g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "5"); - g_check_parse_failed_cxx(xor_, (HParserBackend)GPOINTER_TO_INT(backend), "a"); -} - -static void test_many_cxx(gconstpointer backend) { - hammer::Many many_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many(); - - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); - g_check_parse_ok_cxx(many_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); -} - -static void test_many1_cxx(gconstpointer backend) { - hammer::Many1 many1_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b')).many1(); - - g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "b", "(u0x62)"); - g_check_parse_ok_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "aabbaba", "(u0x61 u0x61 u0x62 u0x62 u0x61 u0x62 u0x61)"); - g_check_parse_failed_cxx(many1_, (HParserBackend)GPOINTER_TO_INT(backend), "daabbabadef"); -} - -static void test_repeat_n_cxx(gconstpointer backend) { - hammer::RepeatN repeat_n_ = hammer::Choice(hammer::Ch('a'), hammer::Ch('b'))[2]; - - g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "adef"); - g_check_parse_ok_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "abdef", "(u0x61 u0x62)"); - g_check_parse_failed_cxx(repeat_n_, (HParserBackend)GPOINTER_TO_INT(backend), "dabdef"); -} - -static void test_optional_cxx(gconstpointer backend) { - hammer::Optional optional_ = hammer::Sequence(hammer::Ch('a'), - hammer::Choice(hammer::Ch('b'), hammer::Ch('c')).optional(), - hammer::ch('d')); - - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "abd", "(u0x61 u0x62 u0x64)"); - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "acd", "(u0x61 u0x63 u0x64)"); - g_check_parse_ok_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ad", "(u0x61 null u0x64)"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "aed"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ab"); - g_check_parse_failed_cxx(optional_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); -} - -static void test_ignore_cxx(gconstpointer backend) { - hammer::Ignore ignore_ = hammer::Sequence(hammer::Ch('a'), - hammer::ch('b').ignore(), - hammer::ch('c')); - - g_check_parse_ok_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "abc", "(u0x61 u0x63)"); - g_check_parse_failed_cxx(ignore_, (HParserBackend)GPOINTER_TO_INT(backend), "ac"); -} - -static void test_sepBy_cxx(gconstpointer backend) { - hammer::SepBy sepBy_ = hammer::SepBy(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); - - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); - g_check_parse_ok_cxx(sepBy_, (HParserBackend)GPOINTER_TO_INT(backend), "", "()"); -} - -static void test_sepBy1_cxx(gconstpointer backend) { - hammer::SepBy1 sepBy1_ = hammer::SepBy1(hammer::Choice(hammer::Ch('1'), hammer::Ch('2'), hammer::Ch('3')), hammer::Ch(',')); - - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,2,3", "(u0x31 u0x32 u0x33)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3,2", "(u0x31 u0x33 u0x32)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "1,3", "(u0x31 u0x33)"); - g_check_parse_ok_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), "3", "(u0x33)"); - g_check_parse_failed_cxx(sepBy1_, (HParserBackend)GPOINTER_TO_INT(backend), ""); -} - -static void test_epsilon_cxx(gconstpointer backend) { - hammer::Epsilon epsilon_p_1 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon(), hammer::Ch('b')); - hammer::Epsilon epsilon_p_2 = hammer::Sequence(hammer::Epsilon(), hammer::Ch('a')); - hammer::Epsilon epsilon_p_3 = hammer::Sequence(hammer::Ch('a'), hammer::Epsilon()); - - g_check_parse_ok_cxx(epsilon_p_1, (HParserBackend)GPOINTER_TO_INT(backend), "ab", "(u0x61 u0x62)"); - g_check_parse_ok_cxx(epsilon_p_2, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); -} - -static void test_and_cxx(gconstpointer backend) { - hammer::And and_1 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('0')); - hammer::And and_2 = hammer::Sequence(hammer::And(hammer::Ch('0')), hammer::Ch('1')); - hammer::And and_3 = hammer::Sequence(hammer::Ch('1'), hammer::And(hammer::Ch('2'))); - - g_check_parse_ok_cxx(and_1, (HParserBackend)GPOINTER_TO_INT(backend), "0", "(u0x30)"); - g_check_parse_failed_cxx(and_2, (HParserBackend)GPOINTER_TO_INT(backend), "0"); - g_check_parse_ok_cxx(and_3, (HParserBackend)GPOINTER_TO_INT(backend), "12", "(u0x31)"); -} - -static void test_not_cxx(gconstpointer backend) { - hammer::Not not_1 = hammer::Sequence(hammer::Ch('a'), hammer::Choice(hammer::Ch('+'), hammer::Token("++")), hammer::Ch('b')); - hammer::Not not_2 = hammer::Sequence(hammer::Ch('a'), - hammer::Choice(hammer::Sequence(hammer::Ch('+'), - hammer::Not(hammer::Ch('+'))), - hammer::Token("++")), - hammer::Ch('b')); - - g_check_parse_ok_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 u0x2b u0x62)"); - g_check_parse_failed_cxx(not_1, (HParserBackend)GPOINTER_TO_INT(backend), "a++b"); - g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a+b", "(u0x61 (u0x2b) u0x62)"); - g_check_parse_ok_cxx(not_2, (HParserBackend)GPOINTER_TO_INT(backend), "a++b", "(u0x61 <2b.2b> u0x62)"); -} - -static void test_leftrec(gconstpointer backend) { - hammer::Ch a_ = hammer::Ch('a'); - - hammer::Indirect lr_ = hammer::Indirect(); - - lr_.bind(hammer::Choice(hammer::Sequence(lr_, a_), a_)); - - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "u0x61"); - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 u0x61)"); - g_check_parse_ok_cxx(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "((u0x61 u0x61) u0x61)"); -} - -static void test_rightrec(gconstpointer backend) { - hammer::Ch a_ = hammer::Ch('a'); - - hammer::Indirect rr_ = hammer::Indirect(); - rr_.bind(hammer::Choice(hammer::Sequence(a_, rr_), hammer::Epsilon())); - - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", "(u0x61)"); - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", "(u0x61 (u0x61))"); - g_check_parse_ok_cxx(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", "(u0x61 (u0x61 (u0x61)))"); -} - -// FIXME won't be ready until action is ready -/* -static void test_ambiguous(gconstpointer backend) { - hammer::Ch d_ = hammer::Ch('d'); - hammer::Ch p_ = hammer::Ch('+'); - hammer::Indirect E_ = hammer::Indirect(); - E_.bind(hammer::Choice(hammer::Sequence(E_, p_, E_), d_)); - hammer::Action expr_ = hammer::Action(E_, act_flatten); - - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", "(u0x64)"); - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", "(u0x64 u0x2b u0x64)"); - g_check_parse_ok_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d+d", "(u0x64 u0x2b u0x64 u0x2b u0x64)"); - g_check_parse_failed_cxx(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+"); -} -*/ - -void register_cxx_tests(void) { - g_test_add_data_func("/core/c++/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range_cxx); - g_test_add_data_func("/core/c++/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left_cxx); - g_test_add_data_func("/core/c++/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right_cxx); - g_test_add_data_func("/core/c++/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle_cxx); - g_test_add_data_func("/core/c++/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in_cxx); - g_test_add_data_func("/core/c++/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/packrat/end", GINT_TO_POINTER(PB_PACKRAT), test_end_cxx); - g_test_add_data_func("/core/c++/parser/packrat/nothing", GINT_TO_POINTER(PB_PACKRAT), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot_cxx); - g_test_add_data_func("/core/c++/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference_cxx); - g_test_add_data_func("/core/c++/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor_cxx); - g_test_add_data_func("/core/c++/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many_cxx); - g_test_add_data_func("/core/c++/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n_cxx); - g_test_add_data_func("/core/c++/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/packrat/epsilon", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_cxx); - g_test_add_data_func("/core/c++/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and_cxx); - g_test_add_data_func("/core/c++/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not_cxx); - // g_test_add_data_func("/core/c++/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec_cxx); - - g_test_add_data_Func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); - g_test_add_data_func("/core/c++/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token_cxx); - g_test_add_data_func("/core/c++/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left_cxx); - g_test_add_data_func("/core/c++/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right_cxx); - g_test_add_data_func("/core/c++/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle_cxx); - //g_test_add_data_func("/core/c++/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action_cxx); - g_test_add_data_func("/core/c++/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in_cxx); - g_test_add_data_func("/core/c++/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/llk/end", GINT_TO_POINTER(PB_LLk), test_end_cxx); - g_test_add_data_func("/core/c++/parser/llk/nothing", GINT_TO_POINTER(PB_LLk), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many_cxx); - g_test_add_data_func("/core/c++/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/llk/epsilon", GINT_TO_POINTER(PB_LLk), test_epsilon_cxx); - //g_test_add_data_func("/core/c++/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore_cxx); - //g_test_add_data_func("/core/c++/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec_cxx); - - g_test_add_data_func("/core/c++/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle_cxx); - // g_test_add_data_func("/core/c++/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/regex/end", GINT_TO_POINTER(PB_REGULAR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/regex/nothing", GINT_TO_POINTER(PB_REGULAR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n_cxx); - g_test_add_data_func("/core/c++/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/regex/epsilon", GINT_TO_POINTER(PB_REGULAR), test_epsilon_cxx); - // g_test_add_data_func("/core/c++/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore_cxx); - - g_test_add_data_func("/core/c++/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle_cxx); - // g_test_add_data_func("/core/c++/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/lalr/end", GINT_TO_POINTER(PB_LALR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/lalr/nothing", GINT_TO_POINTER(PB_LALR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/lalr/epsilon", GINT_TO_POINTER(PB_LALR), test_epsilon_cxx); - // g_test_add_data_func("/core/c++/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec_cxx); - - g_test_add_data_func("/core/c++/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token_cxx); - g_test_add_data_func("/core/c++/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch_cxx); - g_test_add_data_func("/core/c++/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range_cxx); - g_test_add_data_func("/core/c++/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64_cxx); - g_test_add_data_func("/core/c++/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32_cxx); - g_test_add_data_func("/core/c++/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16_cxx); - g_test_add_data_func("/core/c++/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16_cxx); - g_test_add_data_func("/core/c++/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8_cxx); - g_test_add_data_func("/core/c++/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range_cxx); -#if 0 - g_test_add_data_func("/core/c++/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64_cxx); - g_test_add_data_func("/core/c++/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32_cxx); -#endif - g_test_add_data_func("/core/c++/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace_cxx); - g_test_add_data_func("/core/c++/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left_cxx); - g_test_add_data_func("/core/c++/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right_cxx); - g_test_add_data_func("/core/c++/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle_cxx); - g_test_add_data_func("/core/c++/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action_cxx); - g_test_add_data_func("/core/c++/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in_cxx); - g_test_add_data_func("/core/c++/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in_cxx); - g_test_add_data_func("/core/c++/parser/glr/end", GINT_TO_POINTER(PB_GLR), test_end_cxx); - g_test_add_data_func("/core/c++/parser/glr/nothing", GINT_TO_POINTER(PB_GLR), test_nothing_cxx); - g_test_add_data_func("/core/c++/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence_cxx); - g_test_add_data_func("/core/c++/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice_cxx); - g_test_add_data_func("/core/c++/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many_cxx); - g_test_add_data_func("/core/c++/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1_cxx); - g_test_add_data_func("/core/c++/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional_cxx); - g_test_add_data_func("/core/c++/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy_cxx); - g_test_add_data_func("/core/c++/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1_cxx); - g_test_add_data_func("/core/c++/parser/glr/epsilon", GINT_TO_POINTER(PB_GLR), test_epsilon_cxx); - g_test_add_data_func("/core/c++/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool_cxx); - g_test_add_data_func("/core/c++/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore_cxx); - g_test_add_data_func("/core/c++/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec_cxx); - g_test_add_data_func("/core/c++/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec_cxx); - g_test_add_data_func("/core/c++/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous_cxx); - -} - -/* -int main(int argc, char **argv) { - hammer::Ch p = hammer::Ch('a'); - std::string test = "a"; - hammer::UintResult r = p.parse(test); - std::cerr << r.result() << std::endl; - hammer::SequenceResult s = p.many().parse(test); - hammer::UintResult u = boost::get(s.result()[0]); - std::cerr << u.result() << std::endl; - return 0; -} -*/ diff --git a/src/t_parser.c b/src/t_parser.c index c49e443..59adf36 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -442,199 +442,199 @@ static void test_ambiguous(gconstpointer backend) { } void register_parser_tests(void) { - g_test_add_data_func("/core/c/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); - g_test_add_data_func("/core/c/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); - g_test_add_data_func("/core/c/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); - g_test_add_data_func("/core/c/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); - g_test_add_data_func("/core/c/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); - g_test_add_data_func("/core/c/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); - g_test_add_data_func("/core/c/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); - g_test_add_data_func("/core/c/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); - g_test_add_data_func("/core/c/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); - g_test_add_data_func("/core/c/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); - g_test_add_data_func("/core/c/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); - g_test_add_data_func("/core/c/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); + g_test_add_data_func("/core/parser/packrat/token", GINT_TO_POINTER(PB_PACKRAT), test_token); + g_test_add_data_func("/core/parser/packrat/ch", GINT_TO_POINTER(PB_PACKRAT), test_ch); + g_test_add_data_func("/core/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range); + g_test_add_data_func("/core/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64); + g_test_add_data_func("/core/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32); + g_test_add_data_func("/core/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16); + g_test_add_data_func("/core/parser/packrat/int8", GINT_TO_POINTER(PB_PACKRAT), test_int8); + g_test_add_data_func("/core/parser/packrat/uint64", GINT_TO_POINTER(PB_PACKRAT), test_uint64); + g_test_add_data_func("/core/parser/packrat/uint32", GINT_TO_POINTER(PB_PACKRAT), test_uint32); + g_test_add_data_func("/core/parser/packrat/uint16", GINT_TO_POINTER(PB_PACKRAT), test_uint16); + g_test_add_data_func("/core/parser/packrat/uint8", GINT_TO_POINTER(PB_PACKRAT), test_uint8); + g_test_add_data_func("/core/parser/packrat/int_range", GINT_TO_POINTER(PB_PACKRAT), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); - g_test_add_data_func("/core/c/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); + g_test_add_data_func("/core/parser/packrat/float64", GINT_TO_POINTER(PB_PACKRAT), test_float64); + g_test_add_data_func("/core/parser/packrat/float32", GINT_TO_POINTER(PB_PACKRAT), test_float32); #endif - g_test_add_data_func("/core/c/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); - g_test_add_data_func("/core/c/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); - g_test_add_data_func("/core/c/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); - g_test_add_data_func("/core/c/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); - g_test_add_data_func("/core/c/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); - g_test_add_data_func("/core/c/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); - g_test_add_data_func("/core/c/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); - g_test_add_data_func("/core/c/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); - g_test_add_data_func("/core/c/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); - g_test_add_data_func("/core/c/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); - g_test_add_data_func("/core/c/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); - g_test_add_data_func("/core/c/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); - g_test_add_data_func("/core/c/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); - g_test_add_data_func("/core/c/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); - g_test_add_data_func("/core/c/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); - g_test_add_data_func("/core/c/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); - g_test_add_data_func("/core/c/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); - g_test_add_data_func("/core/c/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); - g_test_add_data_func("/core/c/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); - g_test_add_data_func("/core/c/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); - g_test_add_data_func("/core/c/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); - g_test_add_data_func("/core/c/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); - g_test_add_data_func("/core/c/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); - g_test_add_data_func("/core/c/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); - g_test_add_data_func("/core/c/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/c/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); - g_test_add_data_func("/core/c/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); + g_test_add_data_func("/core/parser/packrat/whitespace", GINT_TO_POINTER(PB_PACKRAT), test_whitespace); + g_test_add_data_func("/core/parser/packrat/left", GINT_TO_POINTER(PB_PACKRAT), test_left); + g_test_add_data_func("/core/parser/packrat/right", GINT_TO_POINTER(PB_PACKRAT), test_right); + g_test_add_data_func("/core/parser/packrat/middle", GINT_TO_POINTER(PB_PACKRAT), test_middle); + g_test_add_data_func("/core/parser/packrat/action", GINT_TO_POINTER(PB_PACKRAT), test_action); + g_test_add_data_func("/core/parser/packrat/in", GINT_TO_POINTER(PB_PACKRAT), test_in); + g_test_add_data_func("/core/parser/packrat/not_in", GINT_TO_POINTER(PB_PACKRAT), test_not_in); + g_test_add_data_func("/core/parser/packrat/end_p", GINT_TO_POINTER(PB_PACKRAT), test_end_p); + g_test_add_data_func("/core/parser/packrat/nothing_p", GINT_TO_POINTER(PB_PACKRAT), test_nothing_p); + g_test_add_data_func("/core/parser/packrat/sequence", GINT_TO_POINTER(PB_PACKRAT), test_sequence); + g_test_add_data_func("/core/parser/packrat/choice", GINT_TO_POINTER(PB_PACKRAT), test_choice); + g_test_add_data_func("/core/parser/packrat/butnot", GINT_TO_POINTER(PB_PACKRAT), test_butnot); + g_test_add_data_func("/core/parser/packrat/difference", GINT_TO_POINTER(PB_PACKRAT), test_difference); + g_test_add_data_func("/core/parser/packrat/xor", GINT_TO_POINTER(PB_PACKRAT), test_xor); + g_test_add_data_func("/core/parser/packrat/many", GINT_TO_POINTER(PB_PACKRAT), test_many); + g_test_add_data_func("/core/parser/packrat/many1", GINT_TO_POINTER(PB_PACKRAT), test_many1); + g_test_add_data_func("/core/parser/packrat/repeat_n", GINT_TO_POINTER(PB_PACKRAT), test_repeat_n); + g_test_add_data_func("/core/parser/packrat/optional", GINT_TO_POINTER(PB_PACKRAT), test_optional); + g_test_add_data_func("/core/parser/packrat/sepBy", GINT_TO_POINTER(PB_PACKRAT), test_sepBy); + g_test_add_data_func("/core/parser/packrat/sepBy1", GINT_TO_POINTER(PB_PACKRAT), test_sepBy1); + g_test_add_data_func("/core/parser/packrat/epsilon_p", GINT_TO_POINTER(PB_PACKRAT), test_epsilon_p); + g_test_add_data_func("/core/parser/packrat/attr_bool", GINT_TO_POINTER(PB_PACKRAT), test_attr_bool); + g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); + g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); + g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); + // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); - g_test_add_data_func("/core/c/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); - g_test_add_data_func("/core/c/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); - g_test_add_data_func("/core/c/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); - g_test_add_data_func("/core/c/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); - g_test_add_data_func("/core/c/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); - g_test_add_data_func("/core/c/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); - g_test_add_data_func("/core/c/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); - g_test_add_data_func("/core/c/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); - g_test_add_data_func("/core/c/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); - g_test_add_data_func("/core/c/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); - g_test_add_data_func("/core/c/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); - g_test_add_data_func("/core/c/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); + g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); + g_test_add_data_func("/core/parser/llk/ch", GINT_TO_POINTER(PB_LLk), test_ch); + g_test_add_data_func("/core/parser/llk/ch_range", GINT_TO_POINTER(PB_LLk), test_ch_range); + g_test_add_data_func("/core/parser/llk/int64", GINT_TO_POINTER(PB_LLk), test_int64); + g_test_add_data_func("/core/parser/llk/int32", GINT_TO_POINTER(PB_LLk), test_int32); + g_test_add_data_func("/core/parser/llk/int16", GINT_TO_POINTER(PB_LLk), test_int16); + g_test_add_data_func("/core/parser/llk/int8", GINT_TO_POINTER(PB_LLk), test_int8); + g_test_add_data_func("/core/parser/llk/uint64", GINT_TO_POINTER(PB_LLk), test_uint64); + g_test_add_data_func("/core/parser/llk/uint32", GINT_TO_POINTER(PB_LLk), test_uint32); + g_test_add_data_func("/core/parser/llk/uint16", GINT_TO_POINTER(PB_LLk), test_uint16); + g_test_add_data_func("/core/parser/llk/uint8", GINT_TO_POINTER(PB_LLk), test_uint8); + g_test_add_data_func("/core/parser/llk/int_range", GINT_TO_POINTER(PB_LLk), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); - g_test_add_data_func("/core/c/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); + g_test_add_data_func("/core/parser/llk/float64", GINT_TO_POINTER(PB_LLk), test_float64); + g_test_add_data_func("/core/parser/llk/float32", GINT_TO_POINTER(PB_LLk), test_float32); #endif - g_test_add_data_func("/core/c/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); - g_test_add_data_func("/core/c/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); - g_test_add_data_func("/core/c/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); - g_test_add_data_func("/core/c/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); - g_test_add_data_func("/core/c/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); - g_test_add_data_func("/core/c/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); - g_test_add_data_func("/core/c/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); - g_test_add_data_func("/core/c/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); - g_test_add_data_func("/core/c/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); - g_test_add_data_func("/core/c/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); - g_test_add_data_func("/core/c/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); - g_test_add_data_func("/core/c/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); - g_test_add_data_func("/core/c/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); - g_test_add_data_func("/core/c/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); - g_test_add_data_func("/core/c/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); - g_test_add_data_func("/core/c/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); - g_test_add_data_func("/core/c/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); - g_test_add_data_func("/core/c/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); - g_test_add_data_func("/core/c/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); - //g_test_add_data_func("/core/c/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); - g_test_add_data_func("/core/c/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); + g_test_add_data_func("/core/parser/llk/whitespace", GINT_TO_POINTER(PB_LLk), test_whitespace); + g_test_add_data_func("/core/parser/llk/left", GINT_TO_POINTER(PB_LLk), test_left); + g_test_add_data_func("/core/parser/llk/right", GINT_TO_POINTER(PB_LLk), test_right); + g_test_add_data_func("/core/parser/llk/middle", GINT_TO_POINTER(PB_LLk), test_middle); + g_test_add_data_func("/core/parser/llk/action", GINT_TO_POINTER(PB_LLk), test_action); + g_test_add_data_func("/core/parser/llk/in", GINT_TO_POINTER(PB_LLk), test_in); + g_test_add_data_func("/core/parser/llk/not_in", GINT_TO_POINTER(PB_LLk), test_not_in); + g_test_add_data_func("/core/parser/llk/end_p", GINT_TO_POINTER(PB_LLk), test_end_p); + g_test_add_data_func("/core/parser/llk/nothing_p", GINT_TO_POINTER(PB_LLk), test_nothing_p); + g_test_add_data_func("/core/parser/llk/sequence", GINT_TO_POINTER(PB_LLk), test_sequence); + g_test_add_data_func("/core/parser/llk/choice", GINT_TO_POINTER(PB_LLk), test_choice); + g_test_add_data_func("/core/parser/llk/many", GINT_TO_POINTER(PB_LLk), test_many); + g_test_add_data_func("/core/parser/llk/many1", GINT_TO_POINTER(PB_LLk), test_many1); + g_test_add_data_func("/core/parser/llk/optional", GINT_TO_POINTER(PB_LLk), test_optional); + g_test_add_data_func("/core/parser/llk/sepBy", GINT_TO_POINTER(PB_LLk), test_sepBy); + g_test_add_data_func("/core/parser/llk/sepBy1", GINT_TO_POINTER(PB_LLk), test_sepBy1); + g_test_add_data_func("/core/parser/llk/epsilon_p", GINT_TO_POINTER(PB_LLk), test_epsilon_p); + g_test_add_data_func("/core/parser/llk/attr_bool", GINT_TO_POINTER(PB_LLk), test_attr_bool); + g_test_add_data_func("/core/parser/llk/ignore", GINT_TO_POINTER(PB_LLk), test_ignore); + //g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec); + g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec); - g_test_add_data_func("/core/c/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); - g_test_add_data_func("/core/c/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); - g_test_add_data_func("/core/c/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); - g_test_add_data_func("/core/c/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); - g_test_add_data_func("/core/c/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); - g_test_add_data_func("/core/c/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); - g_test_add_data_func("/core/c/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); - g_test_add_data_func("/core/c/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); - g_test_add_data_func("/core/c/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); - g_test_add_data_func("/core/c/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); - g_test_add_data_func("/core/c/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); - g_test_add_data_func("/core/c/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); + g_test_add_data_func("/core/parser/regex/token", GINT_TO_POINTER(PB_REGULAR), test_token); + g_test_add_data_func("/core/parser/regex/ch", GINT_TO_POINTER(PB_REGULAR), test_ch); + g_test_add_data_func("/core/parser/regex/ch_range", GINT_TO_POINTER(PB_REGULAR), test_ch_range); + g_test_add_data_func("/core/parser/regex/int64", GINT_TO_POINTER(PB_REGULAR), test_int64); + g_test_add_data_func("/core/parser/regex/int32", GINT_TO_POINTER(PB_REGULAR), test_int32); + g_test_add_data_func("/core/parser/regex/int16", GINT_TO_POINTER(PB_REGULAR), test_int16); + g_test_add_data_func("/core/parser/regex/int8", GINT_TO_POINTER(PB_REGULAR), test_int8); + g_test_add_data_func("/core/parser/regex/uint64", GINT_TO_POINTER(PB_REGULAR), test_uint64); + g_test_add_data_func("/core/parser/regex/uint32", GINT_TO_POINTER(PB_REGULAR), test_uint32); + g_test_add_data_func("/core/parser/regex/uint16", GINT_TO_POINTER(PB_REGULAR), test_uint16); + g_test_add_data_func("/core/parser/regex/uint8", GINT_TO_POINTER(PB_REGULAR), test_uint8); + g_test_add_data_func("/core/parser/regex/int_range", GINT_TO_POINTER(PB_REGULAR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); - g_test_add_data_func("/core/c/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); + g_test_add_data_func("/core/parser/regex/float64", GINT_TO_POINTER(PB_REGULAR), test_float64); + g_test_add_data_func("/core/parser/regex/float32", GINT_TO_POINTER(PB_REGULAR), test_float32); #endif - g_test_add_data_func("/core/c/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); - g_test_add_data_func("/core/c/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); - g_test_add_data_func("/core/c/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); - g_test_add_data_func("/core/c/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); - g_test_add_data_func("/core/c/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); - g_test_add_data_func("/core/c/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); - g_test_add_data_func("/core/c/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); - g_test_add_data_func("/core/c/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); - g_test_add_data_func("/core/c/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); - g_test_add_data_func("/core/c/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); - g_test_add_data_func("/core/c/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); - g_test_add_data_func("/core/c/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); - g_test_add_data_func("/core/c/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); - g_test_add_data_func("/core/c/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); - g_test_add_data_func("/core/c/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); - g_test_add_data_func("/core/c/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); - g_test_add_data_func("/core/c/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); - g_test_add_data_func("/core/c/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); - g_test_add_data_func("/core/c/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); + g_test_add_data_func("/core/parser/regex/whitespace", GINT_TO_POINTER(PB_REGULAR), test_whitespace); + g_test_add_data_func("/core/parser/regex/left", GINT_TO_POINTER(PB_REGULAR), test_left); + g_test_add_data_func("/core/parser/regex/right", GINT_TO_POINTER(PB_REGULAR), test_right); + g_test_add_data_func("/core/parser/regex/middle", GINT_TO_POINTER(PB_REGULAR), test_middle); + g_test_add_data_func("/core/parser/regex/action", GINT_TO_POINTER(PB_REGULAR), test_action); + g_test_add_data_func("/core/parser/regex/in", GINT_TO_POINTER(PB_REGULAR), test_in); + g_test_add_data_func("/core/parser/regex/not_in", GINT_TO_POINTER(PB_REGULAR), test_not_in); + g_test_add_data_func("/core/parser/regex/end_p", GINT_TO_POINTER(PB_REGULAR), test_end_p); + g_test_add_data_func("/core/parser/regex/nothing_p", GINT_TO_POINTER(PB_REGULAR), test_nothing_p); + g_test_add_data_func("/core/parser/regex/sequence", GINT_TO_POINTER(PB_REGULAR), test_sequence); + g_test_add_data_func("/core/parser/regex/choice", GINT_TO_POINTER(PB_REGULAR), test_choice); + g_test_add_data_func("/core/parser/regex/many", GINT_TO_POINTER(PB_REGULAR), test_many); + g_test_add_data_func("/core/parser/regex/many1", GINT_TO_POINTER(PB_REGULAR), test_many1); + g_test_add_data_func("/core/parser/regex/repeat_n", GINT_TO_POINTER(PB_REGULAR), test_repeat_n); + g_test_add_data_func("/core/parser/regex/optional", GINT_TO_POINTER(PB_REGULAR), test_optional); + g_test_add_data_func("/core/parser/regex/sepBy", GINT_TO_POINTER(PB_REGULAR), test_sepBy); + g_test_add_data_func("/core/parser/regex/sepBy1", GINT_TO_POINTER(PB_REGULAR), test_sepBy1); + g_test_add_data_func("/core/parser/regex/epsilon_p", GINT_TO_POINTER(PB_REGULAR), test_epsilon_p); + g_test_add_data_func("/core/parser/regex/attr_bool", GINT_TO_POINTER(PB_REGULAR), test_attr_bool); + g_test_add_data_func("/core/parser/regex/ignore", GINT_TO_POINTER(PB_REGULAR), test_ignore); - g_test_add_data_func("/core/c/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); - g_test_add_data_func("/core/c/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); - g_test_add_data_func("/core/c/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); - g_test_add_data_func("/core/c/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); - g_test_add_data_func("/core/c/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); - g_test_add_data_func("/core/c/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); - g_test_add_data_func("/core/c/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); - g_test_add_data_func("/core/c/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); - g_test_add_data_func("/core/c/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); - g_test_add_data_func("/core/c/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); - g_test_add_data_func("/core/c/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); - g_test_add_data_func("/core/c/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); + g_test_add_data_func("/core/parser/lalr/token", GINT_TO_POINTER(PB_LALR), test_token); + g_test_add_data_func("/core/parser/lalr/ch", GINT_TO_POINTER(PB_LALR), test_ch); + g_test_add_data_func("/core/parser/lalr/ch_range", GINT_TO_POINTER(PB_LALR), test_ch_range); + g_test_add_data_func("/core/parser/lalr/int64", GINT_TO_POINTER(PB_LALR), test_int64); + g_test_add_data_func("/core/parser/lalr/int32", GINT_TO_POINTER(PB_LALR), test_int32); + g_test_add_data_func("/core/parser/lalr/int16", GINT_TO_POINTER(PB_LALR), test_int16); + g_test_add_data_func("/core/parser/lalr/int8", GINT_TO_POINTER(PB_LALR), test_int8); + g_test_add_data_func("/core/parser/lalr/uint64", GINT_TO_POINTER(PB_LALR), test_uint64); + g_test_add_data_func("/core/parser/lalr/uint32", GINT_TO_POINTER(PB_LALR), test_uint32); + g_test_add_data_func("/core/parser/lalr/uint16", GINT_TO_POINTER(PB_LALR), test_uint16); + g_test_add_data_func("/core/parser/lalr/uint8", GINT_TO_POINTER(PB_LALR), test_uint8); + g_test_add_data_func("/core/parser/lalr/int_range", GINT_TO_POINTER(PB_LALR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); - g_test_add_data_func("/core/c/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); + g_test_add_data_func("/core/parser/lalr/float64", GINT_TO_POINTER(PB_LALR), test_float64); + g_test_add_data_func("/core/parser/lalr/float32", GINT_TO_POINTER(PB_LALR), test_float32); #endif - g_test_add_data_func("/core/c/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); - g_test_add_data_func("/core/c/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); - g_test_add_data_func("/core/c/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); - g_test_add_data_func("/core/c/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); - g_test_add_data_func("/core/c/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); - g_test_add_data_func("/core/c/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); - g_test_add_data_func("/core/c/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); - g_test_add_data_func("/core/c/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); - g_test_add_data_func("/core/c/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); - g_test_add_data_func("/core/c/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); - g_test_add_data_func("/core/c/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); - g_test_add_data_func("/core/c/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); - g_test_add_data_func("/core/c/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); - g_test_add_data_func("/core/c/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); - g_test_add_data_func("/core/c/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); - g_test_add_data_func("/core/c/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); - g_test_add_data_func("/core/c/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); - g_test_add_data_func("/core/c/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); - g_test_add_data_func("/core/c/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); - g_test_add_data_func("/core/c/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); + g_test_add_data_func("/core/parser/lalr/whitespace", GINT_TO_POINTER(PB_LALR), test_whitespace); + g_test_add_data_func("/core/parser/lalr/left", GINT_TO_POINTER(PB_LALR), test_left); + g_test_add_data_func("/core/parser/lalr/right", GINT_TO_POINTER(PB_LALR), test_right); + g_test_add_data_func("/core/parser/lalr/middle", GINT_TO_POINTER(PB_LALR), test_middle); + g_test_add_data_func("/core/parser/lalr/action", GINT_TO_POINTER(PB_LALR), test_action); + g_test_add_data_func("/core/parser/lalr/in", GINT_TO_POINTER(PB_LALR), test_in); + g_test_add_data_func("/core/parser/lalr/not_in", GINT_TO_POINTER(PB_LALR), test_not_in); + g_test_add_data_func("/core/parser/lalr/end_p", GINT_TO_POINTER(PB_LALR), test_end_p); + g_test_add_data_func("/core/parser/lalr/nothing_p", GINT_TO_POINTER(PB_LALR), test_nothing_p); + g_test_add_data_func("/core/parser/lalr/sequence", GINT_TO_POINTER(PB_LALR), test_sequence); + g_test_add_data_func("/core/parser/lalr/choice", GINT_TO_POINTER(PB_LALR), test_choice); + g_test_add_data_func("/core/parser/lalr/many", GINT_TO_POINTER(PB_LALR), test_many); + g_test_add_data_func("/core/parser/lalr/many1", GINT_TO_POINTER(PB_LALR), test_many1); + g_test_add_data_func("/core/parser/lalr/optional", GINT_TO_POINTER(PB_LALR), test_optional); + g_test_add_data_func("/core/parser/lalr/sepBy", GINT_TO_POINTER(PB_LALR), test_sepBy); + g_test_add_data_func("/core/parser/lalr/sepBy1", GINT_TO_POINTER(PB_LALR), test_sepBy1); + g_test_add_data_func("/core/parser/lalr/epsilon_p", GINT_TO_POINTER(PB_LALR), test_epsilon_p); + g_test_add_data_func("/core/parser/lalr/attr_bool", GINT_TO_POINTER(PB_LALR), test_attr_bool); + g_test_add_data_func("/core/parser/lalr/ignore", GINT_TO_POINTER(PB_LALR), test_ignore); + g_test_add_data_func("/core/parser/lalr/leftrec", GINT_TO_POINTER(PB_LALR), test_leftrec); + g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec); - g_test_add_data_func("/core/c/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); - g_test_add_data_func("/core/c/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); - g_test_add_data_func("/core/c/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); - g_test_add_data_func("/core/c/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); - g_test_add_data_func("/core/c/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); - g_test_add_data_func("/core/c/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); - g_test_add_data_func("/core/c/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); - g_test_add_data_func("/core/c/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); - g_test_add_data_func("/core/c/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); - g_test_add_data_func("/core/c/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); - g_test_add_data_func("/core/c/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); - g_test_add_data_func("/core/c/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); + g_test_add_data_func("/core/parser/glr/token", GINT_TO_POINTER(PB_GLR), test_token); + g_test_add_data_func("/core/parser/glr/ch", GINT_TO_POINTER(PB_GLR), test_ch); + g_test_add_data_func("/core/parser/glr/ch_range", GINT_TO_POINTER(PB_GLR), test_ch_range); + g_test_add_data_func("/core/parser/glr/int64", GINT_TO_POINTER(PB_GLR), test_int64); + g_test_add_data_func("/core/parser/glr/int32", GINT_TO_POINTER(PB_GLR), test_int32); + g_test_add_data_func("/core/parser/glr/int16", GINT_TO_POINTER(PB_GLR), test_int16); + g_test_add_data_func("/core/parser/glr/int8", GINT_TO_POINTER(PB_GLR), test_int8); + g_test_add_data_func("/core/parser/glr/uint64", GINT_TO_POINTER(PB_GLR), test_uint64); + g_test_add_data_func("/core/parser/glr/uint32", GINT_TO_POINTER(PB_GLR), test_uint32); + g_test_add_data_func("/core/parser/glr/uint16", GINT_TO_POINTER(PB_GLR), test_uint16); + g_test_add_data_func("/core/parser/glr/uint8", GINT_TO_POINTER(PB_GLR), test_uint8); + g_test_add_data_func("/core/parser/glr/int_range", GINT_TO_POINTER(PB_GLR), test_int_range); #if 0 - g_test_add_data_func("/core/c/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); - g_test_add_data_func("/core/c/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); + g_test_add_data_func("/core/parser/glr/float64", GINT_TO_POINTER(PB_GLR), test_float64); + g_test_add_data_func("/core/parser/glr/float32", GINT_TO_POINTER(PB_GLR), test_float32); #endif - g_test_add_data_func("/core/c/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); - g_test_add_data_func("/core/c/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); - g_test_add_data_func("/core/c/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); - g_test_add_data_func("/core/c/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); - g_test_add_data_func("/core/c/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); - g_test_add_data_func("/core/c/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); - g_test_add_data_func("/core/c/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); - g_test_add_data_func("/core/c/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); - g_test_add_data_func("/core/c/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); - g_test_add_data_func("/core/c/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); - g_test_add_data_func("/core/c/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); - g_test_add_data_func("/core/c/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); - g_test_add_data_func("/core/c/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); - g_test_add_data_func("/core/c/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); - g_test_add_data_func("/core/c/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); - g_test_add_data_func("/core/c/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); - g_test_add_data_func("/core/c/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); - g_test_add_data_func("/core/c/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); - g_test_add_data_func("/core/c/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); - g_test_add_data_func("/core/c/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); - g_test_add_data_func("/core/c/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); - g_test_add_data_func("/core/c/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); + g_test_add_data_func("/core/parser/glr/whitespace", GINT_TO_POINTER(PB_GLR), test_whitespace); + g_test_add_data_func("/core/parser/glr/left", GINT_TO_POINTER(PB_GLR), test_left); + g_test_add_data_func("/core/parser/glr/right", GINT_TO_POINTER(PB_GLR), test_right); + g_test_add_data_func("/core/parser/glr/middle", GINT_TO_POINTER(PB_GLR), test_middle); + g_test_add_data_func("/core/parser/glr/action", GINT_TO_POINTER(PB_GLR), test_action); + g_test_add_data_func("/core/parser/glr/in", GINT_TO_POINTER(PB_GLR), test_in); + g_test_add_data_func("/core/parser/glr/not_in", GINT_TO_POINTER(PB_GLR), test_not_in); + g_test_add_data_func("/core/parser/glr/end_p", GINT_TO_POINTER(PB_GLR), test_end_p); + g_test_add_data_func("/core/parser/glr/nothing_p", GINT_TO_POINTER(PB_GLR), test_nothing_p); + g_test_add_data_func("/core/parser/glr/sequence", GINT_TO_POINTER(PB_GLR), test_sequence); + g_test_add_data_func("/core/parser/glr/choice", GINT_TO_POINTER(PB_GLR), test_choice); + g_test_add_data_func("/core/parser/glr/many", GINT_TO_POINTER(PB_GLR), test_many); + g_test_add_data_func("/core/parser/glr/many1", GINT_TO_POINTER(PB_GLR), test_many1); + g_test_add_data_func("/core/parser/glr/optional", GINT_TO_POINTER(PB_GLR), test_optional); + g_test_add_data_func("/core/parser/glr/sepBy", GINT_TO_POINTER(PB_GLR), test_sepBy); + g_test_add_data_func("/core/parser/glr/sepBy1", GINT_TO_POINTER(PB_GLR), test_sepBy1); + g_test_add_data_func("/core/parser/glr/epsilon_p", GINT_TO_POINTER(PB_GLR), test_epsilon_p); + g_test_add_data_func("/core/parser/glr/attr_bool", GINT_TO_POINTER(PB_GLR), test_attr_bool); + g_test_add_data_func("/core/parser/glr/ignore", GINT_TO_POINTER(PB_GLR), test_ignore); + g_test_add_data_func("/core/parser/glr/leftrec", GINT_TO_POINTER(PB_GLR), test_leftrec); + g_test_add_data_func("/core/parser/glr/rightrec", GINT_TO_POINTER(PB_GLR), test_rightrec); + g_test_add_data_func("/core/parser/glr/ambiguous", GINT_TO_POINTER(PB_GLR), test_ambiguous); } diff --git a/src/test_suite.c b/src/test_suite.c index 9191f93..e065f13 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -25,9 +25,6 @@ extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); extern void register_benchmark_tests(); -#ifdef __cplusplus -extern void register_cxx_tests(); -#endif int main(int argc, char** argv) { g_test_init(&argc, &argv, NULL); @@ -39,9 +36,6 @@ int main(int argc, char** argv) { register_grammar_tests(); register_misc_tests(); register_benchmark_tests(); - #ifdef __cplusplus - register_cxx_tests(); - #endif g_test_run(); } diff --git a/src/test_suite.h b/src/test_suite.h index 76c6d78..fc008e7 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -19,16 +19,6 @@ #define HAMMER_TEST_SUITE__H #include -#ifdef __cplusplus -extern "C" { -#include -#include -#endif - -#ifndef __cplusplus -typedef int bool; -#endif - // Equivalent to g_assert_*, but not using g_assert... #define g_check_inttype(fmt, typ, n1, op, n2) do { \ typ _n1 = (n1); \ @@ -62,19 +52,6 @@ typedef int bool; } \ } while(0) -#ifdef __cplusplus -#define g_check_string_cxx(n1, op, n2) do { \ - std::string _n1 = (n1); \ - std::string _n2 = (n2); \ - if (!(_n1.compare(_n2) op 0)) { \ - g_test_message("Check failed: (%s) (%s %s %s)", \ - #n1 " " #op " " #n2, \ - _n1.c_str(), #op, _n2.c_str()); \ - g_test_fail(); \ - } \ - } while(0) -#endif - #define g_check_regular(lang) do { \ if (!lang->isValidRegular(lang->env)) { \ g_test_message("Language is not regular"); \ @@ -118,21 +95,6 @@ typedef int bool; g_test_fail(); \ } \ } while(0) - -#ifdef __cplusplus -#define g_check_parse_failed_cxx(parser, backend, input) do { \ - int skip = parser.compile((HParserBackend)backend, NULL); \ - if(skip != 0) { \ - g_test_message("Backend not applicable, skipping test"); \ - break; \ - } \ - parser::result_type result = parser.parse(input); \ - if (NULL != result) { \ - g_test_message("Check failed: shouldn't have succeeded, but did"); \ - g_test_fail(); \ - } \ - } while(0) -#endif #define g_check_parse_ok(parser, backend, input, inp_len, result) do { \ int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \ @@ -158,34 +120,6 @@ typedef int bool; } \ } while(0) -#ifdef __cplusplus -#define g_check_parse_ok_cxx(parser, backend, input, result) do { \ - int skip = parser.compile((HParserBackend)backend, NULL); \ - if(skip) { \ - g_test_message("Backend not applicable, skipping test"); \ - break; \ - } \ - parser::result_type res = parser.parse((std::string)input); \ - if (!res) { \ - g_test_message("Parse failed on line %d", __LINE__); \ - g_test_fail(); \ - } else { \ - std::string& cres = h_write_result_unamb(res); \ - g_check_string_cxx(cres, ==, (std::string)result); \ - /* - system_allocator.free(&system_allocator, cres); \ - HArenaStats stats; \ - h_allocator_stats(res->arena, &stats); \ - g_test_message("Parse used %zd bytes, wasted %zd bytes. " \ - "Inefficiency: %5f%%", \ - stats.used, stats.wasted, \ - stats.wasted * 100. / (stats.used+stats.wasted)); \ - h_delete_arena(res->arena); \ - */ // FIXME might've made a memory leak there - } \ - } while(0) -#endif - #define g_check_hashtable_present(table, key) do { \ if(!h_hashtable_present(table, key)) { \ g_test_message("Check failed: key should have been in table, but wasn't"); \ @@ -263,8 +197,6 @@ typedef int bool; #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) -#ifdef __cplusplus -} -#endif + #endif // #ifndef HAMMER_TEST_SUITE__H From 78b7904c5a74fd8044f557355e2e2485060dc414 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 06/88] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 487fa97..37a273b 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -124,6 +124,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -570,7 +571,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From f8604f5c7658d1bc051267ed36c0b12e67350da1 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 07/88] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From 8928c04456e8bb190d02d06869fc13278aa9f3bf Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 08/88] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..7a05636 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From 390bbd32f12ef45fd801fcb633ff836e26c5e565 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 09/88] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From 06aa19e206ea0155aa17825dbcf5645d1be57685 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 10/88] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From c9c282dcb468daebc45073193b568f5f55ae4f01 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 11/88] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 9061318dd691359578322736fa131c90305c7005 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 12/88] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index f5a9e13..e435587 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -29,7 +29,9 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H typedef struct HParseState_ HParseState; @@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From 529aa3e1a3d67272e2c8c0dea4d620252e5c8447 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 13/88] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index e435587..a7046d0 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -70,8 +70,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -81,6 +94,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -144,12 +160,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From c54f63871af927d238f361bf3a3565a32e8ff0a6 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 14/88] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From fc63ee5d1bd1451db8bb80829be07772e452422f Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 15/88] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index a7046d0..d8c613d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} #endif // #ifndef HAMMER_HAMMER__H From a5da5804570a5b2449fcd2b3d06ffbff5b55efe9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 16/88] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From 804b8f7bd1f0c4667e63d1e93de98bd0f6a69e66 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 17/88] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 9b5c868..cb5d7f6 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From 6effa7ae3ebfc8acdd7858facdf456cb1ec50a52 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 18/88] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1458ecd..dd66860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 0eab9112587315b7e9089fd022e434412cb22e7a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 19/88] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From f1e6e1fa9532043ed515fff6523252e9e5e6a5c9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 20/88] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From 83fdfd93fda0d583562e35c62ebbe2726efa3da5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 21/88] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From e8b2c17026070e47d00835da3f30a29235d91bb4 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 22/88] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From d1b71779e130b4d1dfeab98f2958fa300cc589b5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 23/88] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From f3ce2fc7de48a7be27f7c9cabfa710ec56a9fe41 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 24/88] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From cec4659baa148de18403db5c305655453b1b019b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 25/88] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From 8681600365ae02ed3ff5c209520419a8be5d81b8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:29:44 -0600 Subject: [PATCH 26/88] working on fixing leftrec for packrat --- src/t_parser.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/t_parser.c b/src/t_parser.c index 59adf36..0a7aede 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -410,11 +410,11 @@ static void test_leftrec(gconstpointer backend) { HParser *a_ = h_ch('a'); HParser *lr_ = h_indirect(); - h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL)); + h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), h_epsilon_p(), NULL)); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "u0x61"); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)"); - g_check_parse_ok(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "((u0x61 u0x61) u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "((u0x61) u0x61)"); + g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "(((u0x61) u0x61) u0x61)"); } static void test_rightrec(gconstpointer backend) { @@ -483,7 +483,7 @@ void register_parser_tests(void) { g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and); g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not); g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore); - // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); + //g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec); g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec); g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token); From 656f4546ddbdfb405ae66845484d1b43cd5fe250 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 27/88] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dd66860..2276989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 9e44bea92035b91c4e2764ad432b5f25c04ed81a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 12:37:30 -0600 Subject: [PATCH 28/88] commenting out assert for debugging purposes --- src/backends/packrat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/packrat.c b/src/backends/packrat.c index 8aa1f8e..87f166d 100644 --- a/src/backends/packrat.c +++ b/src/backends/packrat.c @@ -83,7 +83,7 @@ void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) { some->eval_set = NULL; rec_detect->head = some; } - assert(state->lr_stack->head != NULL); + //assert(state->lr_stack->head != NULL); HSlistNode *head = state->lr_stack->head; HLeftRec *lr; while (head && (lr = head->elem)->rule != p) { From 62af9f622233af22c0ac58b94e58157bfd5fabd5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 29/88] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 10c8b0bd22bd1f172fb04aa2434e40035accd626 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:00:58 -0600 Subject: [PATCH 30/88] fixed the segfault! hand-initialized HParser needed PB_MIN set. --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- src/parsers/choice.c | 4 +++- src/parsers/sequence.c | 4 +++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass diff --git a/src/parsers/choice.c b/src/parsers/choice.c index 6db1378..bfc3f90 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &choice_vt; ret->env = (void*)s; + ret->vtable = &choice_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index eff4610..281b78f 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -159,6 +159,8 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &sequence_vt; ret->env = (void*)s; + ret->vtable = &sequence_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } From c32c5cf5eae3a51891c4bae2a55f66c1f520db26 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:05:48 -0600 Subject: [PATCH 31/88] Fixed a potential segfault; hand-initialized HParsers in h_choice and h_sequence need PB_MIN set. Conflicts: src/bindings/python/SConscript src/bindings/python/hammer_tests.py --- src/parsers/choice.c | 4 +++- src/parsers/sequence.c | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/parsers/choice.c b/src/parsers/choice.c index 6db1378..bb55fa8 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &choice_vt; ret->env = (void*)s; + ret->vtable = &choice_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index eff4610..bdea8c0 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -145,20 +145,22 @@ HParser* h_sequence__a(void *args[]) { HParser* h_sequence__ma(HAllocator* mm__, void *args[]) { size_t len = -1; // because do...while const HParser *arg; - + do { arg=((HParser **)args)[++len]; } while(arg); - + HSequence *s = h_new(HSequence, 1); s->p_array = h_new(HParser *, len); for (size_t i = 0; i < len; i++) { s->p_array[i] = ((HParser **)args)[i]; } - + s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &sequence_vt; ret->env = (void*)s; + ret->vtable = &sequence_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } From 87581e09cbdfde5261bb33c6bffacf5c3b078145 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 32/88] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 9091df8..a4e828d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -132,6 +132,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -592,7 +593,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From 42270b613dcbbf34bcea48b488cf759683c3b359 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 33/88] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From a4dbfc61f224510d7e75ea66ab5dc90e58e3b204 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 34/88] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..7a05636 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From a99d7a18d15362e97f9210231b40be7b28d41b85 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 35/88] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From c58555d6a960ef9552c0ea18d53b4aa97aeace3e Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 36/88] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From 4330a426e927fee43375d27fce998fcb96bdd5c8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 37/88] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 34062d50a20e289218d21c238c112ef0548bb61a Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 38/88] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index a4e828d..6928cc9 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -34,7 +34,9 @@ extern "C" { #endif #ifndef __cplusplus +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H #endif typedef struct HParseState_ HParseState; @@ -183,7 +185,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -201,17 +203,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From eb04069817f445f66e1f46dd077e1a81d131c3b9 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 39/88] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 6928cc9..5dac8bf 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -76,8 +76,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -87,6 +100,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -150,12 +166,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From 21f16ebefdbe8a5ff149188ad51f4fcdd2d61250 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 40/88] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From 902ca46b4d9c741dbe2c64f151d021fb043e3a78 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 41/88] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 5dac8bf..b4fc823 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -666,7 +666,7 @@ HParsedToken *h_act_ignore(const HParseResult *p); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} #ifdef __cplusplus From 551645e98069a906fd7f5ba336615b3bc38c563d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 42/88] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From c58a722aef9d6ad67de4683fb22c0b39c77d8ab4 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 43/88] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 9b5c868..cb5d7f6 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From 55af8ac28751469851a93b98254d1a43beb0920d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 44/88] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1458ecd..dd66860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 150e91537b3b5a0ce31302ba42609f1ace7a197f Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 45/88] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From 8fcad1e72ac22ce32843c434b2a58298e3a97fc2 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 46/88] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From 6746df20dbb2cc0a842e6c0bb4825bc26318380d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 47/88] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From b979f66c21ad8dbdbb1e60d78248b659e6557cfc Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 48/88] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From 2d459282d45bec0927ca81b68ff3cac398d06685 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 49/88] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From 5ced9fe9d5cb1d93fe9fb34dc156f6d6bc4fae04 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 50/88] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From 2e0c56f403bf4298d128584b714ea5a97c4ca75e Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 51/88] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From eb1cee7dee214abb0b6c431f3d52370aa08d619d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 52/88] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dd66860..2276989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 6e456e3daac1499ae017e44bad04f0d412f4f949 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 53/88] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 31f283e72023a5c5136810b91bf38103ec625140 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:10:56 -0600 Subject: [PATCH 54/88] Fixed TestRightrec --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass From de6f6115a80db68d5d5b887f12d7b8cec1e19e23 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 55/88] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 541e38d..08819f5 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -126,6 +126,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -586,7 +587,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From 28bee303f9d1885c69b0428548ca89e528d6e201 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 56/88] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From 9d43244c2e50dd3c92df2ad8b46a885cf340c272 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 57/88] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 0a43f8d..d36ec54 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -80,6 +80,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From a31f2d31a4f3b5ebae5df16aa254c5a12689d9b9 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 58/88] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From 1841c9d77ecc006046c164293f3ba67c17cad161 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 59/88] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From 75453d8b2f88336131e31130a89fc25c327cb421 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 60/88] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 49f22e2afffe70c93f5cc8ecc86b5a8039f5dac5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 61/88] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index 08819f5..02e4085 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -29,7 +29,9 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H typedef struct HParseState_ HParseState; @@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From 8f427b5a6d4d9b8b36552c51f20ff1cebd3a88d2 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 62/88] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 02e4085..c8a2bf3 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -70,8 +70,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -81,6 +94,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -144,12 +160,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From 0e4253d0efc2f13b43e35f28e51e68def7a4b763 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 63/88] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From 73eabc7bf9e5eaccaa3175a16c331d7c697c01c8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 64/88] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index c8a2bf3..2aded7a 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p, void* userdata); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} // {{{ Token type registry From ada7bf89ee61d12b2f624cb6e82168f78d647379 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 65/88] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From 77d48e2b7c939e9066e8a899cce47aaeec0c7ed1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 66/88] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 1d9ca76..0cb41af 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From dc9124724af7ffe4b837e6777c96080fe2d0b4d7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 67/88] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 246a3d6..f23f01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 27f94dbe610e7c30573c741a3a1be09b360e95a7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 68/88] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From 4bd2fc9e72f4149b9398d827c15ed3d4e7eb2a33 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 69/88] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From eba8ecc6c39f38efaebd76aba206cc98f5936164 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 70/88] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From 073d6d8a275ad4826b0f6a301f63b45920ca6efb Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 71/88] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From f685f9ea4ed8ad14178ad7f48b5237171c94162b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 72/88] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From 3228a86b90682115decf12be64338be3582283b3 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 73/88] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From 96760f1dafe792105cf18cc3dae1ef03b21fae39 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 74/88] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From 0aac5f4622d52b9df8f01769a6a6d81a63baceb1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 75/88] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f23f01c..0e406ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 12bb5cac36c5225d09bdd029147ea46eb1337133 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 76/88] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 8b6d6084c5ed15630d19a059e037525193835ef8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:00:58 -0600 Subject: [PATCH 77/88] fixed the segfault! hand-initialized HParser needed PB_MIN set. --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass From 954b6e32d3757fd8d1cf7bffc76ea4018ebcbb4c Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 22 Nov 2013 19:42:02 -0600 Subject: [PATCH 78/88] Changed typemap for strings to also parse lengths --- src/bindings/python/hammer_tests.py | 341 ++++++++++++++-------------- src/bindings/swig/hammer.i | 25 +- 2 files changed, 195 insertions(+), 171 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 41cd13f..a5967e6 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,11 +4,11 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2", 3) + cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95", 2), None) + self.assertEqual(h.h_parse(self.parser, "95"), None) class TestChParser(unittest.TestCase): @classmethod @@ -16,201 +16,202 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\xfe"), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a", 2), None) + self.assertEqual(h.h_parse(self.parser, "_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x", 3), None) + self.assertEqual(h.h_parse(self.parser, " x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ba", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, " a", 2), None) - self.assertEqual(h.h_parse(self.parser, "a ", 2), None) - self.assertEqual(h.h_parse(self.parser, " b ", 3), None) - self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) - self.assertEqual(h.h_parse(self.parser, " ab", 3), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, " a"), None) + self.assertEqual(h.h_parse(self.parser, "a "), None) + self.assertEqual(h.h_parse(self.parser, " b "), None) + self.assertEqual(h.h_parse(self.parser, "ba "), None) + self.assertEqual(h.h_parse(self.parser, " ab"), None) -# class TestAction(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) -# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +@unittest.skip("Action not implemented yet") +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc", 3) + cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc", 3) + cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2), None) + self.assertEqual(h.h_parse(self.parser, "aa"), None) class TestNothingP(unittest.TestCase): @classmethod @@ -219,243 +220,243 @@ class TestNothingP(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, "b", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, "b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c", 4), None) + self.assertEqual(h.h_parse(self.parser, "a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c", 1), None) + self.assertEqual(h.h_parse(self.parser, "c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6", 1), None) + self.assertEqual(h.h_parse(self.parser, "6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) + cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5", 1), None) - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "5"), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) + self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef", 4), None) - self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) + self.assertEqual(h.h_parse(self.parser, "adef"), None) + self.assertEqual(h.h_parse(self.parser, "dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed", 3), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "aed"), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass # class TestAttrBool(unittest.TestCase): # @classmethod # def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) # def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + self.assertEqual(h.h_parse(self.parser, "0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) + self.assertEqual(h.h_parse(self.parser, "a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "a+b").ast.token_data.seq tree[1] = tree[1].token_data.seq[0] self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree = h.h_parse(self.parser, "a++b").ast.token_data.seq tree[0] = chr(tree[0].token_data.uint) tree[1] = tree[1].token_data.bytes tree[2] = chr(tree[2].token_data.uint) @@ -471,9 +472,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -482,14 +483,14 @@ class TestRightrec(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_indirect() a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + tree = h.h_parse(self.parser, "a").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) @@ -506,9 +507,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81a9dd1..4fb30c5 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -14,13 +14,35 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } + +%typemap(newfree) HParseResult* { + h_parse_result_free($1); + } + +%newobject h_parse +%delobject h_parse_result_free + + /* +%typemap(in) (uint8_t* str, size_t len) { + if (PyString_Check($input) || + PyUnicode_Check($input)) { + PyString_AsStringAndSize($input, (char**)&$1, &$2); + } else { + PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); + } + } +*/ +%apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} +%apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); int size = PyList_Size($input); int i = 0; int res = 0; - $1 = (void**)malloc(size*sizeof(HParser*)); + $1 = (void**)malloc((size+1)*sizeof(HParser*)); for (i=0; i Date: Sat, 23 Nov 2013 16:40:57 -0600 Subject: [PATCH 79/88] Got rid of accessors in Python; made h_action work --- src/bindings/python/hammer.py | 488 ---------------------------- src/bindings/python/hammer_tests.py | 181 +++++------ src/bindings/swig/hammer.i | 134 ++++++-- 3 files changed, 194 insertions(+), 609 deletions(-) delete mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py deleted file mode 100644 index 36b78c8..0000000 --- a/src/bindings/python/hammer.py +++ /dev/null @@ -1,488 +0,0 @@ -from cffi import FFI -import threading -import sys - -_ffi = FFI() - -# {{{ Types - -_ffi.cdef("typedef struct HAllocator_ HAllocator;") -_ffi.cdef("typedef struct HArena_ HArena;") -_ffi.cdef("typedef int bool;") -_ffi.cdef("typedef struct HParseState_ HParseState;") -_ffi.cdef(""" -typedef enum HParserBackend_ { - PB_MIN = 0, - PB_PACKRAT = 0, // PB_MIN is always the default. - PB_REGULAR, - PB_LLk, - PB_LALR, - PB_GLR -// TODO: support PB_MAX -} HParserBackend; -""") -_ffi.cdef(""" -typedef enum HTokenType_ { - // Before you change the explicit values of these, think of the poor bindings ;_; - TT_NONE = 1, - TT_BYTES = 2, - TT_SINT = 4, - TT_UINT = 8, - TT_SEQUENCE = 16, - TT_RESERVED_1, // reserved for backend-specific internal use - TT_ERR = 32, - TT_USER = 64, - TT_MAX -} HTokenType; -""") -_ffi.cdef(""" -typedef struct HCountedArray_ { - size_t capacity; - size_t used; - HArena * arena; - struct HParsedToken_ **elements; -} HCountedArray; -""") -_ffi.cdef(""" -typedef struct HBytes_ { - const uint8_t *token; - size_t len; -} HBytes; -""") -_ffi.cdef(""" -typedef struct HParsedToken_ { - HTokenType token_type; - union { - HBytes bytes; - int64_t sint; - uint64_t uint; - double dbl; - float flt; - HCountedArray *seq; // a sequence of HParsedToken's - void *user; - }; - size_t index; - char bit_offset; -} HParsedToken; -""") -_ffi.cdef(""" -typedef struct HParseResult_ { - const HParsedToken *ast; - long long bit_length; - HArena * arena; -} HParseResult; -""") - -_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -_ffi.cdef(""" -typedef struct HCFChoice_ HCFChoice; -typedef struct HRVMProg_ HRVMProg; -typedef struct HParserVtable_ HParserVtable; -""") - -_ffi.cdef("typedef struct HParser_ HParser;") -_ffi.cdef(""" -typedef struct HParserTestcase_ { - unsigned char* input; - size_t length; - char* output_unambiguous; -} HParserTestcase; - -typedef struct HCaseResult_ { - bool success; - union { - const char* actual_results; // on failure, filled in with the results of h_write_result_unamb - size_t parse_time; // on success, filled in with time for a single parse, in nsec - }; -} HCaseResult; - -typedef struct HBackendResults_ { - HParserBackend backend; - bool compile_success; - size_t n_testcases; - size_t failed_testcases; // actually a count... - HCaseResult *cases; -} HBackendResults; - -typedef struct HBenchmarkResults_ { - size_t len; - HBackendResults *results; -} HBenchmarkResults; -""") - -# }}} -# {{{ Arena functions -_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") -_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -# }}} -# {{{ cdefs -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' -_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_ch(const uint8_t c);") -_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_bits(size_t len, bool sign);") -_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -_ffi.cdef("HParser* h_int64(void);") -_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int32(void);") -_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int16(void);") -_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int8(void);") -_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint64(void);") -_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint32(void);") -_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint16(void);") -_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint8(void);") -_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_whitespace(const HParser* p);") -_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_end_p(void);") -_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_nothing_p(void);") -_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_sequence(HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__a(void* args);") -_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_choice(HParser* p, ...);") -_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_choice__a(void* args);") -_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_many(const HParser* p);") -_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_many1(const HParser* p);") -_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_optional(const HParser* p);") -_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_ignore(const HParser* p);") -_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_epsilon_p(void);") -_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_and(const HParser* p);") -_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_not(const HParser* p);") -_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_indirect(void);") -_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_parse_result_free(HParseResult *result);") -_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") - -_lib = _ffi.verify("#include ", - libraries=['hammer']) - -_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# }}} -class _DynamicScopeHolder(threading.local): - """A dynamically-scoped holder of python objects, which may or may not - otherwise appear in the object graph. Intended for use with CFFI """ - def __init__(self): - self._ctxstack = [] - def __enter__(self): - self._ctxstack.append([]) - def __exit__(self, exc_type, exc_value, traceback): - self._ctxstack.pop() - return False - def stash(self, *objs): - if len(self._ctxstack) < 1: - raise Exception("Not in any dynamic scope") - for obj in objs: - self._ctxstack[-1].append(obj) -def _fromHParsedToken(cobj): - # TODO: Free the toplevel parser - tt = cobj.token_type - - if cobj.token_type == _lib.TT_BYTES: - return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == _lib.TT_ERR: - # I have no idea what this is for - pass - elif cobj.token_type == _lib.TT_NONE: - return None - elif cobj.token_type == _lib.TT_SEQUENCE: - return [_fromHParsedToken(cobj.seq.elements[i]) - for i in range(cobj.seq.used)] - elif cobj.token_type == _lib.TT_SINT: - return cobj.sint - elif cobj.token_type == _lib.TT_UINT: - return cobj.uint - elif cobj.token_type == _lib.TT_PYTHON: - return _ffi.from_handle(cobj.user) - -_parser_result_holder = _DynamicScopeHolder() -def _toHParsedToken(arena, pyobj): - if pyobj is None: - return _ffi.NULL - cobj = _ffi.new_handle(pyobj) - _parser_result_holder.stash(cobj) - - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) - hpt.token_type = _lib.TT_PYTHON - hpt.user = cobj - hpt.bit_offset = chr(127) - hpt.index = 0 - return hpt - -def _fromParseResult(cobj): - ret = _fromHParsedToken(cobj.ast) - _lib.h_parse_result_free(cobj) - return ret - -def _to_haction(fn): - """Turn a function that transforms a parsed value into an HAction""" - def action(parse_result): - res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) - if res != _ffi.NULL and parse_result.ast != _ffi.NULL: - res.index = parse_result.ast.index - res.bit_offset = parse_result.ast.bit_offset - return res - return _ffi.callback("HParsedToken*(HParseResult*)", action) - -def _to_hpredicate(fn): - """Turn a function that transforms a parsed value into an HAction""" - def predicate(parse_result): - res = fn(_fromParseResult(parse_result)) - # TODO: Handle exceptions; parse should fail. - if type(res) != bool: - raise TypeError("Predicates should return a bool") - return res - return _ffi.callback("bool(HParseResult*)", predicate) - -class Parser(object): - # TODO: Map these to individually garbage-collected blocks of - # memory. Perhaps with an arena allocator with block size of 1? - # There has to be something more efficient than that, though. - - # TODO: How do we handle encodings? By default, we're using UTF-8 - def __init__(self, internal, deps): - """Create a new parser from an FFI object. Not for user code""" - self._parser = internal - self._deps = deps - - def parse(self, string): - with _parser_result_holder: - pres = _lib.h_parse(self._parser, string, len(string)) - if pres: - return _fromParseResult(pres) - else: - return None - - def __mul__(self, count): - return repeat_n(self, count) - - - -class IndirectParser(Parser): - def bind(self, inner): - _lib.h_bind_indirect(self._parser, inner._parser) - self._deps = (inner,) - -class BitsParser(Parser): - pass - -def token(token): - # TODO: Does not clone argument. - if isinstance(token, unicode): - token = token.encode("utf-8") - return Parser(_lib.h_token(token, len(token)), ()) - -def ch(char): - """Returns either a token or an int, depending on the type of the - argument""" - if isinstance(char, int): - return Parser(_lib.h_ch(char), ()) - else: - return token(char) - -def ch_range(chr1, chr2): - if not isinstance(chr1, str) or not isinstance(chr2, str): - raise TypeError("ch_range can't handle unicode") - def my_action(pr): - # print "In action: ", pr - return pr - return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) - -def int_range(parser, i1, i2): - if type(parser) != BitsParser: - raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) - -def bits(length, signedp): - return BitsParser(_lib.h_bits(length, signedp), ()) - -def int64(): return bits(64, True) -def int32(): return bits(32, True) -def int16(): return bits(16, True) -def int8 (): return bits(8, True) -def uint64(): return bits(64, False) -def uint32(): return bits(32, False) -def uint16(): return bits(16, False) -def uint8 (): return bits(8, False) - -def whitespace(p): - return Parser(_lib.h_whitespace(p._parser), (p,)) -def left(p1, p2): - return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) -def right(p1, p2): - return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) -def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) -def action(parser, action): - caction = _to_haction(action) - return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) - -def in_(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_in(charset, len(charset)), ()) -def not_in(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_not_in(charset, len(charset)), ()) -def end_p(): - return Parser(_lib.h_end_p(), ()) -def nothing_p(): - return Parser(_lib.h_nothing_p(), ()) -def sequence(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_sequence(*plist), (plist,)) -def choice(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_choice(*plist), (plist,)) -def butnot(p1, p2): - return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) -def difference(p1, p2): - return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) -def xor(p1, p2): - return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) -def many(p1): - return Parser(_lib.h_many(p1._parser), (p1,)) -def many1(p1): - return Parser(_lib.h_many1(p1._parser), (p1,)) -def repeat_n(p1, n): - return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) -def optional(p1): - return Parser(_lib.h_optional(p1._parser), (p1,)) -def ignore(p1): - return Parser(_lib.h_ignore(p1._parser), (p1,)) -def sepBy(p, sep): - return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) -def sepBy1(p, sep): - return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) -def epsilon_p(): - return Parser(_lib.h_epsilon_p(), ()) -def length_value(p_len, p_value): - return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) -def attr_bool(parser, predicate): - cpredicate = _to_hpredicate(predicate) - return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) -def and_(parser): - return Parser(_lib.h_and(parser._parser), (parser,)) -def not_(parser): - return Parser(_lib.h_not(parser._parser), (parser,)) -def indirect(): - return IndirectParser(_lib.h_indirect(), ()) -def bind_indirect(indirect, inner): - indirect.bind(inner) - -def parse(parser): - return parser.parse() - -# Unfortunately, "in", "and", and "not" are keywords. This makes them -# show up in the module namespace for the use of automated tools. Do -# not attempt to use them by hand; only use the mangled forms (with -# the '_') -sys.modules[__name__].__dict__["in"] = in_ -sys.modules[__name__].__dict__["and"] = and_ -sys.modules[__name__].__dict__["not"] = not_ - -def run_test(): - p_test = sepBy1(choice(ch('1'), - ch('2'), - ch('3')), - ch(',')) - return p_test.parse("1,2,3") - -# {{{ Automatic parser construction... python specific - -# TODO: Implement Parsable metaclass, which requires the existence of -# a "parse" method. - -# This is expected to be extended by user code. As a general rule, -# only provide auto-parsers for your own types. -AUTO_PARSERS = { - str: token, - unicode: token, -} - -def _auto_seq(lst): - return sequence(*(auto_1(p, default_method=_auto_choice) - for p in lst)) - -def _auto_choice(lst): - return choice(*(auto_1(p, default_method=_auto_seq) - for p in lst)) - -def auto_1(arg, default_method=_auto_choice): - if isinstance(arg, Parser): - return arg - elif type(arg) in AUTO_PARSERS: - return AUTO_PARSERS[type(arg)](arg) - else: - return default_method(arg) - -def auto(*args): - return auto_1(args, default_method=_auto_choice) - -# }}} diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a5967e6..82f4084 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95"), None) @@ -16,8 +16,8 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -36,7 +36,7 @@ class TestInt64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) @@ -45,8 +45,8 @@ class TestInt32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -56,8 +56,8 @@ class TestInt16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xfe"), None) self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -67,7 +67,7 @@ class TestInt8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -76,7 +76,7 @@ class TestUint64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) @@ -85,7 +85,7 @@ class TestUint32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -94,7 +94,7 @@ class TestUint16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -103,7 +103,7 @@ class TestUint8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -112,7 +112,7 @@ class TestIntRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05"), 5) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x0b"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -133,8 +133,8 @@ class TestWhitespaceEnd(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, ""), None) # empty string + self.assertEqual(h.h_parse(self.parser, " "), None) # empty string def test_failure(self): self.assertEqual(h.h_parse(self.parser, " x"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,14 +175,16 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -@unittest.skip("Action not implemented yet") +#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), + h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + lambda x: [chr(y).upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "XX"), None) @@ -191,7 +193,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -200,7 +202,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -209,7 +211,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -227,7 +229,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -237,9 +239,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -248,8 +250,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -258,8 +260,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -268,7 +270,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -277,7 +279,7 @@ class TestDifference(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab"), "ab") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -286,8 +288,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) + self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -297,10 +299,10 @@ class TestMany(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) def test_failure(self): pass @@ -309,9 +311,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -321,7 +323,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -331,10 +333,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) + self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -345,7 +346,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -354,11 +355,11 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -367,10 +368,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -380,7 +381,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) def test_failure(self): pass @@ -389,7 +390,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -398,7 +399,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -407,8 +408,8 @@ class TestEpsilonP3(unittest.TestCase): # def setUpClass(cls): # cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) # def test_failure(self): # self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -417,7 +418,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) def test_failure(self): pass @@ -435,7 +436,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) def test_failure(self): pass @@ -444,7 +445,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -453,14 +454,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b").ast.token_data.seq - tree[1] = tree[1].token_data.seq[0] - self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b").ast.token_data.seq - tree[0] = chr(tree[0].token_data.uint) - tree[1] = tree[1].token_data.bytes - tree[2] = chr(tree[2].token_data.uint) - self.assertEqual(tree, ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) def test_failure(self): pass @@ -472,9 +467,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a"), "a") +# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -485,15 +480,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) + self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) + self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) def test_failure(self): pass @@ -507,9 +496,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): # # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 4fb30c5..23bcc15 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,6 @@ %module hammer %nodefaultctor; +//%nodefaultdtor; %include "stdint.i" //%include "typemaps.i" @@ -7,31 +8,6 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; -%typemap(in) uint8_t* { - Py_INCREF($input); - $1 = (uint8_t*)PyString_AsString($input); - } -%typemap(out) uint8_t* { - $result = PyString_FromString((char*)$1); - } - -%typemap(newfree) HParseResult* { - h_parse_result_free($1); - } - -%newobject h_parse -%delobject h_parse_result_free - - /* -%typemap(in) (uint8_t* str, size_t len) { - if (PyString_Check($input) || - PyUnicode_Check($input)) { - PyString_AsStringAndSize($input, (char**)&$1, &$2); - } else { - PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); - } - } -*/ %apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} @@ -79,6 +55,43 @@ PyList_SetItem($result, i, o); } } +%typemap(out) struct HParseResult_* { + if ($1 == NULL) { + // TODO: raise parse failure + Py_INCREF(Py_None); + $result = Py_None; + } else { + $result = hpt_to_python($1->ast); + } + } + +%inline %{ + static int h_tt_python; + %} +%init %{ + h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); + %} + +/* +%typemap(in) (HPredicate* pred, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_predicate; + } +*/ +%typemap(in) (const HAction a, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_action; + } + +%inline { + struct HParsedToken_; + struct HParseResult_; + static PyObject* hpt_to_python(const struct HParsedToken_ *token); + + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); + } #else #warning no uint8_t* typemaps defined #endif @@ -92,3 +105,74 @@ %include "allocator.h" %include "hammer.h" + +%extend HArena_ { + ~HArena_() { + h_delete_arena($self); + } + }; +%extend HParseResult_ { + ~HParseResult_() { + h_parse_result_free($self); + } +}; + +%newobject h_parse; +%delobject h_parse_result_free; +%newobject h_new_arena; +%delobject h_delete_arena; + +#ifdef SWIGPYTHON +%inline { + static PyObject* hpt_to_python(const HParsedToken *token) { + // Caller holds a reference to returned object + PyObject *ret; + if (token == NULL) { + Py_RETURN_NONE; + } + switch (token->token_type) { + case TT_NONE: + Py_RETURN_NONE; + break; + case TT_BYTES: + return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); + case TT_SINT: + // TODO: return PyINT if appropriate + return PyLong_FromLong(token->token_data.sint); + case TT_UINT: + // TODO: return PyINT if appropriate + return PyLong_FromUnsignedLong(token->token_data.uint); + case TT_SEQUENCE: + ret = PyTuple_New(token->token_data.seq->used); + for (int i = 0; i < token->token_data.seq->used; i++) { + PyTuple_SET_ITEM(ret, i, hpt_to_python(token->token_data.seq->elements[i])); + } + return ret; + default: + if (token->token_type == h_tt_python) { + ret = (PyObject*)token->token_data.user; + Py_INCREF(ret); + return ret; + } else { + return SWIG_NewPointerObj((void*)token, SWIGTYPE_p_HParsedToken_, 0 | 0); + // TODO: support registry + } + + } + } + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + if (ret == NULL) { + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + HParsedToken *tok = h_make(p->arena, h_tt_python, ret); + return tok; + } + + } +#endif From 294bca3fce98ead2a3383ab578e39c9dfde0d124 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 19:53:58 -0600 Subject: [PATCH 80/88] Cleaned up python bindings interface a bit; now returns strings rather than character codes if appropriate --- src/bindings/python/SConscript | 2 +- src/bindings/python/hammer_tests.py | 109 ++++++++++++++-------------- src/bindings/swig/hammer.i | 77 ++++++++++++++++++-- 3 files changed, 126 insertions(+), 62 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 0bde353..fd03a3f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -18,6 +18,6 @@ libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") +pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 82f4084..7a7d5f1 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, "\ta"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,13 +175,12 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), - lambda x: [chr(y).upper() for y in x]) + lambda x: [y.upper() for y in x]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) @@ -193,7 +192,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -202,7 +201,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), "d") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -211,7 +210,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -229,7 +228,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -239,9 +238,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -250,8 +249,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -260,8 +259,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "aa"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -270,7 +269,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), "4") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -288,8 +287,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) - self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), "0") + self.assertEqual(h.h_parse(self.parser, "9"), "9") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -300,9 +299,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass @@ -311,9 +310,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -323,7 +322,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -333,9 +332,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) + self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) + self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) + self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -346,7 +345,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) + self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -355,10 +354,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -368,10 +367,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -381,7 +380,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) + self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) def test_failure(self): pass @@ -390,7 +389,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -399,7 +398,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -418,7 +417,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) + self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) def test_failure(self): pass @@ -436,7 +435,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) + self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) def test_failure(self): pass @@ -445,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) + self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -454,8 +453,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) - self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) + self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -480,9 +479,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) - self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) - self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) + self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 23bcc15..a2a8940 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,8 @@ %module hammer + %nodefaultctor; -//%nodefaultdtor; %include "stdint.i" - //%include "typemaps.i" - //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) %ignore HCountedArray_; @@ -12,6 +10,67 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} + + +%rename(_h_ch) h_ch; +%pythoncode %{ + def h_ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return h_token(ch) + else: + return _h_ch(ch) +%} + +%rename(_h_ch_range) h_ch_range; +%pythoncode %{ + def h_ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return h_action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +%} + +%rename(_h_in) h_in; +%rename(_h_not_in) h_not_in; +%pythoncode %{ + def h_in(charset): + return h_action(_h_in(charset), chr) + def h_not_in(charset): + return h_action(_h_not_in(charset), chr) + %} + +%inline { + static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; + + static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + _helper_ParseError = parse_error; + _helper_Placeholder = placeholder; + } + } + +%pythoncode %{ + class Placeholder(object): + """The python equivalent of TT_NONE""" + def __str__(self): + return "Placeholder" + def __repr__(self): + return "Placeholder" + def __eq__(self, other): + return type(self) == type(other) + class ParseError(Exception): + """The parse failed; the message may have more information""" + pass + + _hammer._register_helpers(ParseError, + Placeholder) + %} + %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); @@ -64,7 +123,9 @@ $result = hpt_to_python($1->ast); } } - +%typemap(newfree) struct HParseResult_* { + h_parse_result_free($input); + } %inline %{ static int h_tt_python; %} @@ -72,6 +133,8 @@ h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); %} + + /* %typemap(in) (HPredicate* pred, void* user_data) { Py_INCREF($input); @@ -132,7 +195,7 @@ } switch (token->token_type) { case TT_NONE: - Py_RETURN_NONE; + return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL); break; case TT_BYTES: return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); @@ -173,6 +236,8 @@ HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } - + } + + #endif From 43357bbcda19ff22b509ecbd134b0b14bbe1b0ef Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sun, 24 Nov 2013 18:43:14 -0600 Subject: [PATCH 81/88] AttrBool working --- src/bindings/python/hammer_tests.py | 18 ++++++++-------- src/bindings/swig/hammer.i | 32 ++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 7a7d5f1..587fbd5 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -402,15 +402,15 @@ class TestEpsilonP3(unittest.TestCase): def test_failure(self): pass -# class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) + self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index a2a8940..20661c8 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -135,26 +135,28 @@ -/* -%typemap(in) (HPredicate* pred, void* user_data) { + +%typemap(in) (HPredicate pred, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_predicate; } -*/ + %typemap(in) (const HAction a, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_action; } -%inline { +%inline %{ + struct HParsedToken_; struct HParseResult_; static PyObject* hpt_to_python(const struct HParsedToken_ *token); static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); - } + static int call_predicate(const struct HParseResult_ *p, void* user_data); + %} #else #warning no uint8_t* typemaps defined #endif @@ -164,6 +166,7 @@ #include "allocator.h" #include "hammer.h" #include "internal.h" +#include "glue.h" %} %include "allocator.h" %include "hammer.h" @@ -233,11 +236,30 @@ assert(ret != NULL); } // TODO: add reference to ret to parse-local data + // For now, just hold onto reference HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } + static int call_predicate(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + int rret = 0; + if (ret == NULL) { + // TODO: throw exception + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + rret = PyObject_IsTrue(ret); + Py_DECREF(ret); + return rret; + } + } + #endif From cdc7ea83816289231ef14d655c0c5f4138be8fb0 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Mon, 25 Nov 2013 02:49:02 -0600 Subject: [PATCH 82/88] Finished Python bindings --- src/bindings/python/hammer_tests.py | 398 ++++++++++++++-------------- src/bindings/swig/hammer.i | 145 +++++++--- 2 files changed, 315 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 587fbd5..9014d5c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,457 +4,462 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2") + cls.parser = h.token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95"), None) + self.assertEqual(self.parser.parse("95"), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.h_ch(0xa2) - cls.parser_chr = h.h_ch("\xa2") + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_ch_range("a", "c") + cls.parser = h.ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int64() + cls.parser = h.int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int32() + cls.parser = h.int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int16() + cls.parser = h.int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe"), None) - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int8() + cls.parser = h.int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) + self.assertEqual(self.parser.parse("\x88"), -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint64() + cls.parser = h.uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint32() + cls.parser = h.uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint16() + cls.parser = h.uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint8() + cls.parser = h.uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) + self.assertEqual(self.parser.parse("\x78"), 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int_range(h.h_uint8(), 3, 10) + cls.parser = h.int_range(h.uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05"), 5) + self.assertEqual(self.parser.parse("\x05"), 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b"), None) + self.assertEqual(self.parser.parse("\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_ch("a")) + cls.parser = h.whitespace(h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, "\ta"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a"), None) + self.assertEqual(self.parser.parse("_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_end_p()) + cls.parser = h.whitespace(h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), None) # empty string - self.assertEqual(h.h_parse(self.parser, " "), None) # empty string + self.assertEqual(self.parser.parse(""), None) # empty string + self.assertEqual(self.parser.parse(" "), None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x"), None) + self.assertEqual(self.parser.parse(" x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.left(h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), "a") + self.assertEqual(self.parser.parse("a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) + cls.parser = h.right(h.ch(" "), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ba"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), "a") + self.assertEqual(self.parser.parse(" a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, " a"), None) - self.assertEqual(h.h_parse(self.parser, "a "), None) - self.assertEqual(h.h_parse(self.parser, " b "), None) - self.assertEqual(h.h_parse(self.parser, "ba "), None) - self.assertEqual(h.h_parse(self.parser, " ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), - h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), + h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) + self.assertEqual(self.parser.parse("ab"), ["A", "B"]) + self.assertEqual(self.parser.parse("AB"), ["A", "B"]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX"), None) + self.assertEqual(self.parser.parse("XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc") + cls.parser = h.in_("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc") + cls.parser = h.not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), "d") + self.assertEqual(self.parser.parse("d"), "d") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) + cls.parser = h.sequence(h.ch("a"), h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa"), None) + self.assertEqual(self.parser.parse("aa"), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_nothing_p() + cls.parser = h.nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, "b"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c"), None) + self.assertEqual(self.parser.parse("a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.choice(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c"), None) + self.assertEqual(self.parser.parse("c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.butnot(h.ch("a"), h.token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "aa"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), "4") + self.assertEqual(self.parser.parse("4"), "4") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6"), None) + self.assertEqual(self.parser.parse("6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) + cls.parser = h.difference(h.token("ab"), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), "ab") + self.assertEqual(self.parser.parse("ab"), "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), "0") - self.assertEqual(h.h_parse(self.parser, "9"), "9") + self.assertEqual(self.parser.parse("0"), "0") + self.assertEqual(self.parser.parse("9"), "9") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5"), None) - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("5"), None) + self.assertEqual(self.parser.parse("a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) + self.assertEqual(self.parser.parse(""), ()) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("b"), ('b',)) + self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) - self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) + self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse("b"), ("b",)) + self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) + self.assertEqual(self.parser.parse("abdef"), ('a', 'b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef"), None) - self.assertEqual(h.h_parse(self.parser, "dabdef"), None) + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) - self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) - self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) + self.assertEqual(self.parser.parse("abd"), ('a','b','d')) + self.assertEqual(self.parser.parse("acd"), ('a','c','d')) + self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed"), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) + self.assertEqual(self.parser.parse("abc"), ("a","c")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) - self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) + self.assertEqual(self.parser.parse(""), ()) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) + self.assertEqual(self.parser.parse("ab"), ("a", "b")) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestAttrBool(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), + lambda x: x[0] == x[1]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) - self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + self.assertEqual(self.parser.parse("aa"), ("a", "a")) + self.assertEqual(self.parser.parse("bb"), ("b", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) + self.assertEqual(self.parser.parse("0"), ("0",)) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0"), None) + self.assertEqual(self.parser.parse("0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) + self.assertEqual(self.parser.parse("12"), ('1',)) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), + h.choice(h.ch("+"), h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) + self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b"), None) + self.assertEqual(self.parser.parse("a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), + h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) - self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) + self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b')) + self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -462,26 +467,29 @@ class TestNot2(unittest.TestCase): # #class TestLeftrec(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # a = h.h_ch("a") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # cls.parser = h.indirect() +# # a = h.ch("a") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a"), "a") -# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) +# # self.assertEqual(self.parser.parse("a"), "a") +# # self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass + class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) + #raise unittest.SkipTest("Bind doesn't work right now") + cls.parser = h.indirect() + a = h.ch("a") + cls.parser.bind(h.choice(h.sequence(a, cls.parser), + h.epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) - self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("aa"), ('a', ('a',))) + self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass @@ -489,15 +497,15 @@ class TestRightrec(unittest.TestCase): # #class TestAmbiguous(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # d = h.h_ch("d") -# # p = h.h_ch("+") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # cls.parser = h.indirect() +# # d = h.ch("d") +# # p = h.ch("+") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d"), ["d"]) +# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+"), None) +# # self.assertEqual(self.parser.parse("d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 20661c8..e251904 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -12,43 +12,13 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} -%rename(_h_ch) h_ch; -%pythoncode %{ - def h_ch(ch): - if isinstance(ch, str) or isinstance(ch, unicode): - return h_token(ch) - else: - return _h_ch(ch) -%} - -%rename(_h_ch_range) h_ch_range; -%pythoncode %{ - def h_ch_range(c1, c2): - dostr = isinstance(c1, str) - dostr2 = isinstance(c2, str) - if isinstance(c1, unicode) or isinstance(c2, unicode): - raise TypeError("ch_range only works on bytes") - if dostr != dostr2: - raise TypeError("Both arguments to ch_range must be the same type") - if dostr: - return h_action(_h_ch_range(c1, c2), chr) - else: - return _h_ch_range(c1, c2) -%} - -%rename(_h_in) h_in; -%rename(_h_not_in) h_not_in; -%pythoncode %{ - def h_in(charset): - return h_action(_h_in(charset), chr) - def h_not_in(charset): - return h_action(_h_not_in(charset), chr) - %} +%rename("_%s") ""; +// %rename(_h_ch) h_ch; %inline { static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; - static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + static void register_helpers(PyObject* parse_error, PyObject *placeholder) { _helper_ParseError = parse_error; _helper_Placeholder = placeholder; } @@ -260,6 +230,115 @@ } +%rename("%s") ""; +%extend HParser_ { + HParseResult* parse(const uint8_t* input, size_t length) { + return h_parse($self, input, length); + } + bool compile(HParserBackend backend) { + return h_compile($self, backend, NULL) == 0; + } + PyObject* __dir__() { + PyObject* ret = PyList_New(2); + PyList_SET_ITEM(ret, 0, PyString_FromString("parse")); + PyList_SET_ITEM(ret, 1, PyString_FromString("compile")); + return ret; + } +} + +%pythoncode %{ + +def action(p, act): + return _h_action(p, act) +def attr_bool(p, pred): + return _h_attr_bool(p, pred) + +def ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return token(ch) + else: + return _h_ch(ch) + +def ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +def epsilon_p(): return _h_epsilon_p() +def end_p(): + return _h_end_p() +def in_(charset): + return action(_h_in(charset), chr) +def not_in(charset): + return action(_h_not_in(charset), chr) +def not_(p): return _h_not(p) +def int_range(p, i1, i2): + return _h_int_range(p, i1, i2) +def token(string): + return _h_token(string) +def whitespace(p): + return _h_whitespace(p) +def xor(p1, p2): + return _h_xor(p1, p2) +def butnot(p1, p2): + return _h_butnot(p1, p2) +def and_(p1): + return _h_and(p1) +def difference(p1, p2): + return _h_difference(p1, p2) + +def sepBy(p, sep): return _h_sepBy(p, sep) +def sepBy1(p, sep): return _h_sepBy1(p, sep) +def many(p): return _h_many(p) +def many1(p): return _h_many1(p) +def repeat_n(p, n): return _h_repeat_n(p, n) +def choice(*args): return _h_choice__a(list(args)) +def sequence(*args): return _h_sequence__a(list(args)) + +def optional(p): return _h_optional(p) +def nothing_p(): return _h_nothing_p() +def ignore(p): return _h_ignore(p) + +def left(p1, p2): return _h_left(p1, p2) +def middle(p1, p2, p3): return _h_middle(p1, p2, p3) +def right(p1, p2): return _h_right(p1, p2) + + +class HIndirectParser(_HParser_): + def __init__(self): + # Shoves the guts of an _HParser_ into a HIndirectParser. + tret = _h_indirect() + self.__dict__.clear() + self.__dict__.update(tret.__dict__) + + def __dir__(self): + return super(HIndirectParser, self).__dir__() + ['bind'] + def bind(self, parser): + _h_bind_indirect(self, parser) + +def indirect(): + return HIndirectParser() + +def bind_indirect(indirect, new_parser): + indirect.bind(new_parser) + +def uint8(): return _h_uint8() +def uint16(): return _h_uint16() +def uint32(): return _h_uint32() +def uint64(): return _h_uint64() +def int8(): return _h_int8() +def int16(): return _h_int16() +def int32(): return _h_int32() +def int64(): return _h_int64() + + +%} #endif From f21954eecd430ee3760366551ee2977d496335b0 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:29:28 -0800 Subject: [PATCH 83/88] everything's working for new-build except Clean() --- SConstruct | 19 ++++++++++++++----- src/SConscript | 25 ++++++++++++++----------- src/bindings/python/SConscript | 13 +++++++++---- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/SConstruct b/SConstruct index 50c185b..e30f6df 100644 --- a/SConstruct +++ b/SConstruct @@ -7,9 +7,13 @@ import sys vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept)) +vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['python'])) env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars, tools=['default', 'scanreplace'], toolpath=['tools']) +if not 'bindings' in env: + env['bindings'] = [] + def calcInstallPath(*elements): path = os.path.abspath(os.path.join(*map(env.subst, elements))) if 'DESTDIR' in env: @@ -90,18 +94,23 @@ env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') #env.Append(CPPPATH=os.path.join('#', "hammer")) +testruns = [] + Export('env') +Export('testruns') if not GetOption("in_place"): env['BUILD_BASE'] = 'build/$VARIANT' - env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') - env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples') + lib = env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') + env.Alias("examples", env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples')) else: env['BUILD_BASE'] = '.' - env.SConscript(["src/SConscript"]) - env.SConscript(["examples/SConscript"]) + lib = env.SConscript(["src/SConscript"]) + env.Alias(env.SConscript(["examples/SConscript"])) -env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') +#env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') + +env.Alias("test", testruns) env.Alias("install", "$libpath") env.Alias("install", "$incpath") diff --git a/src/SConscript b/src/SConscript index 0cb41af..0388393 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,6 @@ # -*- python -*- -Import('env') - -bindings = ['python'] +import os.path +Import('env testruns') dist_headers = [ "hammer.h", @@ -62,15 +61,16 @@ misc_hammer_parts = [ 'registry.c', 'system_allocator.c'] -tests = ['t_benchmark.c', - 't_bitreader.c', - 't_bitwriter.c', - 't_parser.c', - 't_grammar.c', - 't_misc.c'] +ctests = ['t_benchmark.c', + 't_bitreader.c', + 't_bitwriter.c', + 't_parser.c', + 't_grammar.c', + 't_misc.c'] libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts) libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts) +Default(libhammer_shared, libhammer_static) env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) @@ -81,9 +81,12 @@ env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') testenv.Append(LIBS=['hammer'], LIBPATH=['.']) -testenv.Program('test_suite', tests + ['test_suite.c']) +ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c']) +ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path])) +AlwaysBuild(ctest) +testruns.append(ctest) Export("libhammer_static libhammer_shared") -for b in bindings: +for b in env['bindings']: env.SConscript(["bindings/%s/SConscript" % b]) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index fd03a3f..5d78875 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,11 +1,12 @@ # -*- python -*- -import os.path -Import('env libhammer_shared') +import os, os.path +Import('env libhammer_shared testruns') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) -pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CPPPATH = ['../../']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.ParseConfig("pkg-config --cflags python") pythonenv.Append(LIBS = ['hammer']) pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) @@ -15,9 +16,13 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytestexec = pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytest = Alias("testpython", [pytestexec], pytestexec) +AlwaysBuild(pytest) +testruns.append(pytest) Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) From 1e7553d3dfd0e7f2634411f715aee590754a4a0e Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:40:15 -0800 Subject: [PATCH 84/88] build//src/bindings/python/hammer_tests.py still hangs around but everything else is cleaned up. --- src/bindings/python/SConscript | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 5d78875..0f1e9a9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -20,9 +20,10 @@ Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestexec = pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") +pytests = ['hammer_tests.py'] +pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "nosetests -vv $SOURCE") pytest = Alias("testpython", [pytestexec], pytestexec) AlwaysBuild(pytest) testruns.append(pytest) -Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) + From 5dea57c4d482929e43d6e9cfa9a496e518dc3dc5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:45:34 -0800 Subject: [PATCH 85/88] build matrix for python --- .travis.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e406ca..a594c74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,31 @@ language: c compiler: - gcc - clang +env: + - BINDINGS=none +matrix: + include: + - compiler: gcc + language: python + python: 2.7 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.7 + env: BINDINGS=python + - compiler: gcc + language: python + python: 2.6 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.6 + env: BINDINGS=python before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev python-nose + - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq swig python-dev; fi script: - - scons + - scons bindings=$BINDINGS test notifications: irc: channels: From bd1603812952b01852a910c7eae09a7ec1900306 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:53:30 -0800 Subject: [PATCH 86/88] Python 2.6 is unsupported because the PyCapsule API was backported from Python 3 to 2.7. Trying Python 3.3 and 3.2 just for grins. --- .travis.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a594c74..8717876 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,19 +8,27 @@ matrix: include: - compiler: gcc language: python - python: 2.7 + python: 3.3 env: BINDINGS=python - compiler: clang language: python - python: 2.7 + python: 3.3 env: BINDINGS=python - compiler: gcc language: python - python: 2.6 + python: 3.2 env: BINDINGS=python - compiler: clang language: python - python: 2.6 + python: 3.2 + env: BINDINGS=python + - compiler: gcc + language: python + python: 2.7 + env: BINDINGS=python + - compiler: clang + language: python + python: 2.7 env: BINDINGS=python before_install: - sudo apt-get update -qq From 75255d2e473943982c387dd63660d443020970d0 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 15:55:23 -0800 Subject: [PATCH 87/88] SCons doesn't support Python 3! So 2.7 only it is. Oh well. --- .travis.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8717876..8c5af52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,6 @@ env: - BINDINGS=none matrix: include: - - compiler: gcc - language: python - python: 3.3 - env: BINDINGS=python - - compiler: clang - language: python - python: 3.3 - env: BINDINGS=python - - compiler: gcc - language: python - python: 3.2 - env: BINDINGS=python - - compiler: clang - language: python - python: 3.2 - env: BINDINGS=python - compiler: gcc language: python python: 2.7 From f611a49ab2c2bcc9a1f0ce5101cd418822a6145c Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 26 Nov 2013 16:04:13 -0800 Subject: [PATCH 88/88] cpp stuff doesn't belong on python-bindings branch --- src/hammer.cxx | 164 ----------------- src/hammer.hxx | 491 ------------------------------------------------- 2 files changed, 655 deletions(-) delete mode 100644 src/hammer.cxx delete mode 100644 src/hammer.hxx diff --git a/src/hammer.cxx b/src/hammer.cxx deleted file mode 100644 index e7f3cb7..0000000 --- a/src/hammer.cxx +++ /dev/null @@ -1,164 +0,0 @@ -#include "hammer.hxx" - -namespace hammer { - - typedef variant AnyResult; - - const BytesResult::result_type BytesResult::result() { return _bytes; } - const UintResult::result_type UintResult::result() { return _uint; } - const IntResult::result_type IntResult::result() { return _sint; } - const SequenceResult::result_type SequenceResult::result() { return _seq; } - - template<> - BytesResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); - } - - template<> - BytesResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return BytesResult(vector(res->ast->bytes.token, res->ast->bytes.token+res->ast->bytes.len)); - } - - template<> - UintResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return UintResult(res->ast->uint); - } - - template<> - UintResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return UintResult(res->ast->uint); - } - - template<> - IntResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return IntResult(res->ast->sint); - } - - template<> - IntResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return IntResult(res->ast->sint); - } - - template<> - NullResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return NullResult(); - } - - template<> - NullResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return NullResult(); - } - - vector make_seq(HCountedArray *seq) { - vector ret; - for (size_t i=0; iused; ++i) { - switch(seq->elements[i]->token_type) { - case TT_NONE: - ret.push_back(NullResult()); - break; - case TT_BYTES: - ret.push_back(BytesResult(vector(seq->elements[i]->bytes.token, seq->elements[i]->bytes.token+seq->elements[i]->bytes.len))); - break; - case TT_SINT: - ret.push_back(IntResult(seq->elements[i]->sint)); - break; - case TT_UINT: - ret.push_back(UintResult(seq->elements[i]->uint)); - break; - case TT_SEQUENCE: - ret.push_back(make_seq(seq->elements[i]->seq)); - break; - default: - //TODO some kind of error - break; - } - } - return ret; - } - - template<> - SequenceResult Parser::parse(const string &input) { - HParseResult *res = h_parse(_parser, reinterpret_cast(input.c_str()), input.size()); - return SequenceResult(make_seq(res->ast->seq)); - } - - template<> - SequenceResult Parser::parse(const uint8_t *input, size_t length) { - HParseResult *res = h_parse(_parser, input, length); - return SequenceResult(make_seq(res->ast->seq)); - } - - template - Many Parser::many() { - return Many(*this); - } - - template<> - Many Parser::many() { - return Many(*this); - } - - template - RepeatN Parser::many(size_t n) { - return RepeatN(this, n); - } - - template - Optional Parser::optional() { - return Optional(this); - } - - template - RepeatN Parser::operator[](size_t n) { - return RepeatN(this, n); - } - - IntRange Int64::in_range(const int64_t lower, const int64_t upper) { - Int64 p = Int64(); - return IntRange(p, lower, upper); - } - - IntRange Int32::in_range(const int32_t lower, const int32_t upper) { - Int32 p = Int32(); - return IntRange(p, lower, upper); - } - - IntRange Int16::in_range(const int16_t lower, const int16_t upper) { - Int16 p = Int16(); - return IntRange(p, lower, upper); - } - - IntRange Int8::in_range(const int8_t lower, const int8_t upper) { - Int8 p = Int8(); - return IntRange(p, lower, upper); - } - - IntRange Uint64::in_range(const uint64_t lower, const uint64_t upper) { - Uint64 p = Uint64(); - return IntRange(p, lower, upper); - } - - IntRange Uint32::in_range(const uint32_t lower, const uint32_t upper) { - Uint32 p = Uint32(); - return IntRange(p, lower, upper); - } - - IntRange Uint16::in_range(const uint16_t lower, const uint16_t upper) { - Uint16 p = Uint16(); - return IntRange(p, lower, upper); - } - - IntRange Uint8::in_range(const uint8_t lower, const uint8_t upper) { - Uint8 p = Uint8(); - return IntRange(p, lower, upper); - } - -} diff --git a/src/hammer.hxx b/src/hammer.hxx deleted file mode 100644 index 8f3200c..0000000 --- a/src/hammer.hxx +++ /dev/null @@ -1,491 +0,0 @@ -#ifndef HAMMER_HAMMER__HXX -#define HAMMER_HAMMER__HXX - -#include "hammer.h" -#include -#include -#include -#include - -using std::list; using std::string; using std::vector; -using boost::variant; - -namespace hammer { - - template - class ParseResult { - public: - typedef T result_type; - protected: - ParseResult() { } - }; - - class BytesResult : public ParseResult > { - public: - typedef vector result_type; - BytesResult(const vector res) : _bytes(res) { } - const result_type result(); - private: - BytesResult() { } - result_type _bytes; - }; - - class UintResult : public ParseResult { - public: - typedef uint64_t result_type; - UintResult(const uint64_t res) : _uint(res) { } - const result_type result(); - private: - UintResult() { } - result_type _uint; - }; - - class IntResult : public ParseResult { - public: - typedef int64_t result_type; - IntResult(const int64_t res) : _sint(res) { } - const result_type result(); - private: - IntResult() { } - result_type _sint; - }; - - class NullResult: public ParseResult { - public: - NullResult() { } - typedef void* result_type; - const result_type result() { return NULL; } - }; - - class SequenceResult : public ParseResult > > { - public: - typedef vector > result_type; - SequenceResult(result_type res) : _seq(res) { } - const result_type result(); - private: - SequenceResult() { } - result_type _seq; - }; - - /* forward declarations */ - template class Many; - class Many1; - template class Optional; - class RepeatN; - template class IntRange; - - template - class Parser { - public: - typedef T result_type; - result_type parse(const string &input); - result_type parse(const uint8_t *input, size_t length); - Many many(); - RepeatN many(size_t n); - Optional optional(); - RepeatN operator[](size_t n); - HParser* parser() { return _parser; } - protected: - HParser* _parser; - Parser() { } - // Parser(const Parser &p) : _parser(p.parser()) { } // hopefully we don't need a copy constructor... - }; - - class Token : public Parser { - public: - Token(string &str) : _tok(str) { - _parser = h_token(reinterpret_cast(str.c_str()), str.size()); - } - Token(const uint8_t *str, size_t length) : _tok(reinterpret_cast(str), length) { - _parser = h_token(str, length); - } - private: - string _tok; - }; - - class Ch : public Parser { - public: - friend class Parser; - Ch(const uint8_t c) : _c(c) { - _parser = h_ch(c); - } - private: - uint8_t _c; - }; - - class ChRange : public Parser { - public: - ChRange(const uint8_t lower, const uint8_t upper) : _lower(lower), _upper(upper) { - _parser = h_ch_range(lower, upper); - } - private: - uint8_t _lower, _upper; - }; - - class SignedBits : public Parser { - public: - SignedBits(size_t len) : _len(len) { - _parser = h_bits(len, true); - } - private: - size_t _len; - }; - - class UnsignedBits : public Parser { - public: - UnsignedBits(size_t len) : _len(len) { - _parser = h_bits(len, false); - } - private: - size_t _len; - }; - - class Int64 : public Parser { - public: - Int64() { - _parser = h_int64(); - } - IntRange in_range(const int64_t lower, const int64_t upper); - }; - - class Int32 : public Parser { - public: - Int32() { - _parser = h_int32(); - } - IntRange in_range(const int32_t lower, const int32_t upper); - }; - - class Int16 : public Parser { - public: - Int16() { - _parser = h_int16(); - } - IntRange in_range(const int16_t lower, const int16_t upper); - }; - - class Int8 : public Parser { - public: - Int8() { - _parser = h_int8(); - } - IntRange in_range(const int8_t lower, const int8_t upper); - }; - - class Uint64 : public Parser { - public: - Uint64() { - _parser = h_uint64(); - } - IntRange in_range(const uint64_t lower, const uint64_t upper); - }; - - class Uint32 : public Parser { - public: - Uint32() { - _parser = h_uint32(); - } - IntRange in_range(const uint32_t lower, const uint32_t upper); - }; - - class Uint16 : public Parser { - public: - Uint16() { - _parser = h_uint16(); - } - IntRange in_range(const uint16_t lower, const uint16_t upper); - }; - - class Uint8 : public Parser { - public: - Uint8() { - _parser = h_uint8(); - } - IntRange in_range(const uint8_t lower, const uint8_t upper); - }; - - template - class IntRange : public Parser { - public: - IntRange(Parser &p, const int64_t lower, const int64_t upper) : _p(p), _lower(lower), _upper(upper) { - this->_parser = h_int_range(p.parser(), lower, upper); - } - private: - Parser _p; - int64_t _lower, _upper; - }; - - template - class Whitespace : public Parser { - public: - typedef typename T::result_type result_type; - Whitespace(Parser &p) : _p(p) { - this->_parser = h_whitespace(p.parser()); - } - private: - Parser _p; - }; - - template - class Left : public Parser { - public: - typedef typename T::result_type result_type; - Left(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_left(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Right : public Parser { - public: - typedef typename U::result_type result_type; - Right(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_right(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Middle : public Parser { - public: - typedef typename U::result_type result_type; - Middle(Parser &p, Parser &x, Parser &q) : _p(p), _x(x), _q(q) { - this->_parser = h_middle(p.parser(), x.parser(), q.parser()); - } - private: - Parser _p; - Parser _x; - Parser _q; - }; - - /* what are we doing about h_action? */ - - class In : public Parser { - public: - In(string &charset) : _charset(charset) { - _parser = h_in(reinterpret_cast(charset.c_str()), charset.size()); - } - In(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { - _parser = h_in(charset, length); - } - private: - string _charset; - }; - - class NotIn : public Parser { - public: - NotIn(string &charset) : _charset(charset) { - _parser = h_not_in(reinterpret_cast(charset.c_str()), charset.size()); - } - NotIn(const uint8_t *charset, size_t length) : _charset(reinterpret_cast(charset), length) { - _parser = h_not_in(charset, length); - } - private: - string _charset; - }; - - class End : public Parser { - public: - End() { - _parser = h_end_p(); - } - }; - - class Nothing : public Parser { - public: - Nothing() { - _parser = h_nothing_p(); - } - }; - - class Sequence : public Parser { - friend class Parser; - public: - Sequence(list &ps) : _ps(ps) { - void *parsers[ps.size()]; - size_t i = 0; - for (list::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = const_cast(it->parser()); - } - _parser = h_sequence__a(parsers); - } - // maybe also a begin and end iterator version - private: - list _ps; - }; - - class Choice : public Parser { - public: - Choice(list > &ps) : _ps(ps) { - void *parsers[ps.size()]; - size_t i = 0; - for (list >::iterator it=ps.begin(); it != ps.end(); ++it, ++i) { - parsers[i] = const_cast(it->parser()); - } - _parser = h_choice__a(parsers); - } - private: - list > _ps; - }; - - template - class ButNot : public Parser { - public: - typedef typename T::result_type result_type; - ButNot(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_butnot(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Difference : public Parser { - public: - typedef typename T::result_type result_type; - Difference(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_difference(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Xor : public Parser > { - public: - typedef variant result_type; - Xor(Parser &p, Parser &q) : _p(p), _q(q) { - this->_parser = h_xor(p.parser(), q.parser()); - } - private: - Parser _p; - Parser _q; - }; - - template - class Many : public Parser { - public: - Many(Parser &p) : _p(p) { - _parser = h_many(p.parser()); - } - private: - Parser _p; - }; - - class Many1 : public Parser { - public: - Many1(Parser &p) : _p(p) { - _parser = h_many1(p.parser()); - } - private: - Parser _p; - }; - - class RepeatN: public Parser { - public: - RepeatN(Parser &p, const size_t n) : _p(p), _n(n) { - _parser = h_repeat_n(p.parser(), n); - } - private: - Parser _p; - size_t _n; - }; - - template - class Optional : public Parser { - public: - typedef typename T::result_type result_type; - Optional(Parser &p) : _p(p) { - this->_parser = h_optional(p.parser()); - } - private: - Parser _p; - }; - - class Ignore : public Parser { - public: - Ignore(Parser &p) : _p(p) { - _parser = h_ignore(p.parser()); - } - private: - Parser _p; - }; - - class SepBy : public Parser { - public: - SepBy(Parser &p, Parser &sep) : _p(p), _sep(sep) { - _parser = h_sepBy(p.parser(), sep.parser()); - } - private: - Parser _p, _sep; - }; - - class SepBy1 : public Parser { - public: - SepBy1(Parser &p, Parser &sep) : _p(p), _sep(sep) { - _parser = h_sepBy1(p.parser(), sep.parser()); - } - private: - Parser _p, _sep; - }; - - class Epsilon : public Parser { - public: - Epsilon() { - _parser = h_epsilon_p(); - } - }; - - template - class LengthValue : public Parser { - public: - LengthValue(Parser &length, Parser &value) : _length(length), _value(value) { - _parser = h_length_value(length.parser(), value.parser()); - } - private: - Parser _length; - Parser _value; - }; - - /* FIXME attr_bool */ - - class And : public Parser { - public: - And(Parser &p) : _p(p) { - _parser = h_and(p.parser()); - } - private: - Parser _p; - }; - - class Not : public Parser { - public: - Not(Parser &p) : _p(p) { - _parser = h_not(p.parser()); - } - private: - Parser _p; - }; - - template - class Indirect : public Parser { - public: - typedef typename T::result_type result_type; - Indirect(Parser &p) : _p(p) { - this->_parser = h_indirect(); - h_bind_indirect(this->_parser, p.parser()); - } - private: - Parser _p; - }; - -} - -#endif