Got rid of accessors in Python; made h_action work
This commit is contained in:
Dan Hirsch
parent
954b6e32d3
commit
26c358accf
3 changed files with 194 additions and 609 deletions
|
|
@ -1,488 +0,0 @@
|
|||
from cffi import FFI
|
||||
import threading
|
||||
import sys
|
||||
|
||||
_ffi = FFI()
|
||||
|
||||
# {{{ Types
|
||||
|
||||
_ffi.cdef("typedef struct HAllocator_ HAllocator;")
|
||||
_ffi.cdef("typedef struct HArena_ HArena;")
|
||||
_ffi.cdef("typedef int bool;")
|
||||
_ffi.cdef("typedef struct HParseState_ HParseState;")
|
||||
_ffi.cdef("""
|
||||
typedef enum HParserBackend_ {
|
||||
PB_MIN = 0,
|
||||
PB_PACKRAT = 0, // PB_MIN is always the default.
|
||||
PB_REGULAR,
|
||||
PB_LLk,
|
||||
PB_LALR,
|
||||
PB_GLR
|
||||
// TODO: support PB_MAX
|
||||
} HParserBackend;
|
||||
""")
|
||||
_ffi.cdef("""
|
||||
typedef enum HTokenType_ {
|
||||
// Before you change the explicit values of these, think of the poor bindings ;_;
|
||||
TT_NONE = 1,
|
||||
TT_BYTES = 2,
|
||||
TT_SINT = 4,
|
||||
TT_UINT = 8,
|
||||
TT_SEQUENCE = 16,
|
||||
TT_RESERVED_1, // reserved for backend-specific internal use
|
||||
TT_ERR = 32,
|
||||
TT_USER = 64,
|
||||
TT_MAX
|
||||
} HTokenType;
|
||||
""")
|
||||
_ffi.cdef("""
|
||||
typedef struct HCountedArray_ {
|
||||
size_t capacity;
|
||||
size_t used;
|
||||
HArena * arena;
|
||||
struct HParsedToken_ **elements;
|
||||
} HCountedArray;
|
||||
""")
|
||||
_ffi.cdef("""
|
||||
typedef struct HBytes_ {
|
||||
const uint8_t *token;
|
||||
size_t len;
|
||||
} HBytes;
|
||||
""")
|
||||
_ffi.cdef("""
|
||||
typedef struct HParsedToken_ {
|
||||
HTokenType token_type;
|
||||
union {
|
||||
HBytes bytes;
|
||||
int64_t sint;
|
||||
uint64_t uint;
|
||||
double dbl;
|
||||
float flt;
|
||||
HCountedArray *seq; // a sequence of HParsedToken's
|
||||
void *user;
|
||||
};
|
||||
size_t index;
|
||||
char bit_offset;
|
||||
} HParsedToken;
|
||||
""")
|
||||
_ffi.cdef("""
|
||||
typedef struct HParseResult_ {
|
||||
const HParsedToken *ast;
|
||||
long long bit_length;
|
||||
HArena * arena;
|
||||
} HParseResult;
|
||||
""")
|
||||
|
||||
_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
|
||||
_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
|
||||
_ffi.cdef("""
|
||||
typedef struct HCFChoice_ HCFChoice;
|
||||
typedef struct HRVMProg_ HRVMProg;
|
||||
typedef struct HParserVtable_ HParserVtable;
|
||||
""")
|
||||
|
||||
_ffi.cdef("typedef struct HParser_ HParser;")
|
||||
_ffi.cdef("""
|
||||
typedef struct HParserTestcase_ {
|
||||
unsigned char* input;
|
||||
size_t length;
|
||||
char* output_unambiguous;
|
||||
} HParserTestcase;
|
||||
|
||||
typedef struct HCaseResult_ {
|
||||
bool success;
|
||||
union {
|
||||
const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
|
||||
size_t parse_time; // on success, filled in with time for a single parse, in nsec
|
||||
};
|
||||
} HCaseResult;
|
||||
|
||||
typedef struct HBackendResults_ {
|
||||
HParserBackend backend;
|
||||
bool compile_success;
|
||||
size_t n_testcases;
|
||||
size_t failed_testcases; // actually a count...
|
||||
HCaseResult *cases;
|
||||
} HBackendResults;
|
||||
|
||||
typedef struct HBenchmarkResults_ {
|
||||
size_t len;
|
||||
HBackendResults *results;
|
||||
} HBenchmarkResults;
|
||||
""")
|
||||
|
||||
# }}}
|
||||
# {{{ Arena functions
|
||||
_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);")
|
||||
_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);")
|
||||
# }}}
|
||||
# {{{ cdefs
|
||||
## The following section was generated by
|
||||
## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/'
|
||||
_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
|
||||
_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
|
||||
_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
|
||||
_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
|
||||
_ffi.cdef("HParser* h_ch(const uint8_t c);")
|
||||
_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
|
||||
_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
|
||||
_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
|
||||
_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
|
||||
_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
|
||||
_ffi.cdef("HParser* h_bits(size_t len, bool sign);")
|
||||
_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
|
||||
_ffi.cdef("HParser* h_int64(void);")
|
||||
_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_int32(void);")
|
||||
_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_int16(void);")
|
||||
_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_int8(void);")
|
||||
_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_uint64(void);")
|
||||
_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_uint32(void);")
|
||||
_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_uint16(void);")
|
||||
_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_uint8(void);")
|
||||
_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_whitespace(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
|
||||
_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
|
||||
_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
|
||||
_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
|
||||
_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
|
||||
_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
|
||||
_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
|
||||
_ffi.cdef("HParser* h_end_p(void);")
|
||||
_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_nothing_p(void);")
|
||||
_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_sequence(HParser* p, ...);")
|
||||
_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
|
||||
_ffi.cdef("HParser* h_sequence__a(void* args);")
|
||||
_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
|
||||
_ffi.cdef("HParser* h_choice(HParser* p, ...);")
|
||||
_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
|
||||
_ffi.cdef("HParser* h_choice__a(void* args);")
|
||||
_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
|
||||
_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
|
||||
_ffi.cdef("HParser* h_many(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_many1(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
|
||||
_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
|
||||
_ffi.cdef("HParser* h_optional(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_ignore(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
|
||||
_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
|
||||
_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
|
||||
_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
|
||||
_ffi.cdef("HParser* h_epsilon_p(void);")
|
||||
_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
|
||||
_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
|
||||
_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
|
||||
_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
|
||||
_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
|
||||
_ffi.cdef("HParser* h_and(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_not(const HParser* p);")
|
||||
_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
|
||||
_ffi.cdef("HParser* h_indirect(void);")
|
||||
_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
|
||||
_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
|
||||
_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
|
||||
_ffi.cdef("void h_parse_result_free(HParseResult *result);")
|
||||
_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
|
||||
_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
|
||||
_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
|
||||
_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
|
||||
_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
|
||||
_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
|
||||
|
||||
_lib = _ffi.verify("#include <hammer/hammer.h>",
|
||||
libraries=['hammer'])
|
||||
|
||||
_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45
|
||||
# }}}
|
||||
class _DynamicScopeHolder(threading.local):
|
||||
"""A dynamically-scoped holder of python objects, which may or may not
|
||||
otherwise appear in the object graph. Intended for use with CFFI """
|
||||
def __init__(self):
|
||||
self._ctxstack = []
|
||||
def __enter__(self):
|
||||
self._ctxstack.append([])
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self._ctxstack.pop()
|
||||
return False
|
||||
def stash(self, *objs):
|
||||
if len(self._ctxstack) < 1:
|
||||
raise Exception("Not in any dynamic scope")
|
||||
for obj in objs:
|
||||
self._ctxstack[-1].append(obj)
|
||||
def _fromHParsedToken(cobj):
|
||||
# TODO: Free the toplevel parser
|
||||
tt = cobj.token_type
|
||||
|
||||
if cobj.token_type == _lib.TT_BYTES:
|
||||
return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
|
||||
elif cobj.token_type == _lib.TT_ERR:
|
||||
# I have no idea what this is for
|
||||
pass
|
||||
elif cobj.token_type == _lib.TT_NONE:
|
||||
return None
|
||||
elif cobj.token_type == _lib.TT_SEQUENCE:
|
||||
return [_fromHParsedToken(cobj.seq.elements[i])
|
||||
for i in range(cobj.seq.used)]
|
||||
elif cobj.token_type == _lib.TT_SINT:
|
||||
return cobj.sint
|
||||
elif cobj.token_type == _lib.TT_UINT:
|
||||
return cobj.uint
|
||||
elif cobj.token_type == _lib.TT_PYTHON:
|
||||
return _ffi.from_handle(cobj.user)
|
||||
|
||||
_parser_result_holder = _DynamicScopeHolder()
|
||||
def _toHParsedToken(arena, pyobj):
|
||||
if pyobj is None:
|
||||
return _ffi.NULL
|
||||
cobj = _ffi.new_handle(pyobj)
|
||||
_parser_result_holder.stash(cobj)
|
||||
|
||||
hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken")))
|
||||
hpt.token_type = _lib.TT_PYTHON
|
||||
hpt.user = cobj
|
||||
hpt.bit_offset = chr(127)
|
||||
hpt.index = 0
|
||||
return hpt
|
||||
|
||||
def _fromParseResult(cobj):
|
||||
ret = _fromHParsedToken(cobj.ast)
|
||||
_lib.h_parse_result_free(cobj)
|
||||
return ret
|
||||
|
||||
def _to_haction(fn):
|
||||
"""Turn a function that transforms a parsed value into an HAction"""
|
||||
def action(parse_result):
|
||||
res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result)))
|
||||
if res != _ffi.NULL and parse_result.ast != _ffi.NULL:
|
||||
res.index = parse_result.ast.index
|
||||
res.bit_offset = parse_result.ast.bit_offset
|
||||
return res
|
||||
return _ffi.callback("HParsedToken*(HParseResult*)", action)
|
||||
|
||||
def _to_hpredicate(fn):
|
||||
"""Turn a function that transforms a parsed value into an HAction"""
|
||||
def predicate(parse_result):
|
||||
res = fn(_fromParseResult(parse_result))
|
||||
# TODO: Handle exceptions; parse should fail.
|
||||
if type(res) != bool:
|
||||
raise TypeError("Predicates should return a bool")
|
||||
return res
|
||||
return _ffi.callback("bool(HParseResult*)", predicate)
|
||||
|
||||
class Parser(object):
|
||||
# TODO: Map these to individually garbage-collected blocks of
|
||||
# memory. Perhaps with an arena allocator with block size of 1?
|
||||
# There has to be something more efficient than that, though.
|
||||
|
||||
# TODO: How do we handle encodings? By default, we're using UTF-8
|
||||
def __init__(self, internal, deps):
|
||||
"""Create a new parser from an FFI object. Not for user code"""
|
||||
self._parser = internal
|
||||
self._deps = deps
|
||||
|
||||
def parse(self, string):
|
||||
with _parser_result_holder:
|
||||
pres = _lib.h_parse(self._parser, string, len(string))
|
||||
if pres:
|
||||
return _fromParseResult(pres)
|
||||
else:
|
||||
return None
|
||||
|
||||
def __mul__(self, count):
|
||||
return repeat_n(self, count)
|
||||
|
||||
|
||||
|
||||
class IndirectParser(Parser):
|
||||
def bind(self, inner):
|
||||
_lib.h_bind_indirect(self._parser, inner._parser)
|
||||
self._deps = (inner,)
|
||||
|
||||
class BitsParser(Parser):
|
||||
pass
|
||||
|
||||
def token(token):
|
||||
# TODO: Does not clone argument.
|
||||
if isinstance(token, unicode):
|
||||
token = token.encode("utf-8")
|
||||
return Parser(_lib.h_token(token, len(token)), ())
|
||||
|
||||
def ch(char):
|
||||
"""Returns either a token or an int, depending on the type of the
|
||||
argument"""
|
||||
if isinstance(char, int):
|
||||
return Parser(_lib.h_ch(char), ())
|
||||
else:
|
||||
return token(char)
|
||||
|
||||
def ch_range(chr1, chr2):
|
||||
if not isinstance(chr1, str) or not isinstance(chr2, str):
|
||||
raise TypeError("ch_range can't handle unicode")
|
||||
def my_action(pr):
|
||||
# print "In action: ", pr
|
||||
return pr
|
||||
return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action)
|
||||
|
||||
def int_range(parser, i1, i2):
|
||||
if type(parser) != BitsParser:
|
||||
raise TypeError("int_range is only valid when used with a bits parser")
|
||||
return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,))
|
||||
|
||||
def bits(length, signedp):
|
||||
return BitsParser(_lib.h_bits(length, signedp), ())
|
||||
|
||||
def int64(): return bits(64, True)
|
||||
def int32(): return bits(32, True)
|
||||
def int16(): return bits(16, True)
|
||||
def int8 (): return bits(8, True)
|
||||
def uint64(): return bits(64, False)
|
||||
def uint32(): return bits(32, False)
|
||||
def uint16(): return bits(16, False)
|
||||
def uint8 (): return bits(8, False)
|
||||
|
||||
def whitespace(p):
|
||||
return Parser(_lib.h_whitespace(p._parser), (p,))
|
||||
def left(p1, p2):
|
||||
return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2))
|
||||
def right(p1, p2):
|
||||
return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2))
|
||||
def middle(p1, p2, p3):
|
||||
return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3))
|
||||
def action(parser, action):
|
||||
caction = _to_haction(action)
|
||||
return Parser(_lib.h_action(parser._parser, caction), (parser, caction))
|
||||
|
||||
def in_(charset):
|
||||
if not isinstance(charset, str):
|
||||
# TODO/Python3: change str to bytes
|
||||
raise TypeError("in_ can't deal with unicode")
|
||||
return Parser(_lib.h_in(charset, len(charset)), ())
|
||||
def not_in(charset):
|
||||
if not isinstance(charset, str):
|
||||
# TODO/Python3: change str to bytes
|
||||
raise TypeError("in_ can't deal with unicode")
|
||||
return Parser(_lib.h_not_in(charset, len(charset)), ())
|
||||
def end_p():
|
||||
return Parser(_lib.h_end_p(), ())
|
||||
def nothing_p():
|
||||
return Parser(_lib.h_nothing_p(), ())
|
||||
def sequence(*parsers):
|
||||
plist = [p._parser for p in parsers]
|
||||
plist.append(_ffi.NULL)
|
||||
return Parser(_lib.h_sequence(*plist), (plist,))
|
||||
def choice(*parsers):
|
||||
plist = [p._parser for p in parsers]
|
||||
plist.append(_ffi.NULL)
|
||||
return Parser(_lib.h_choice(*plist), (plist,))
|
||||
def butnot(p1, p2):
|
||||
return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2))
|
||||
def difference(p1, p2):
|
||||
return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2))
|
||||
def xor(p1, p2):
|
||||
return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2))
|
||||
def many(p1):
|
||||
return Parser(_lib.h_many(p1._parser), (p1,))
|
||||
def many1(p1):
|
||||
return Parser(_lib.h_many1(p1._parser), (p1,))
|
||||
def repeat_n(p1, n):
|
||||
return Parser(_lib.h_repeat_n(p1._parser, n), (p1,))
|
||||
def optional(p1):
|
||||
return Parser(_lib.h_optional(p1._parser), (p1,))
|
||||
def ignore(p1):
|
||||
return Parser(_lib.h_ignore(p1._parser), (p1,))
|
||||
def sepBy(p, sep):
|
||||
return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep))
|
||||
def sepBy1(p, sep):
|
||||
return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep))
|
||||
def epsilon_p():
|
||||
return Parser(_lib.h_epsilon_p(), ())
|
||||
def length_value(p_len, p_value):
|
||||
return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value))
|
||||
def attr_bool(parser, predicate):
|
||||
cpredicate = _to_hpredicate(predicate)
|
||||
return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate))
|
||||
def and_(parser):
|
||||
return Parser(_lib.h_and(parser._parser), (parser,))
|
||||
def not_(parser):
|
||||
return Parser(_lib.h_not(parser._parser), (parser,))
|
||||
def indirect():
|
||||
return IndirectParser(_lib.h_indirect(), ())
|
||||
def bind_indirect(indirect, inner):
|
||||
indirect.bind(inner)
|
||||
|
||||
def parse(parser):
|
||||
return parser.parse()
|
||||
|
||||
# Unfortunately, "in", "and", and "not" are keywords. This makes them
|
||||
# show up in the module namespace for the use of automated tools. Do
|
||||
# not attempt to use them by hand; only use the mangled forms (with
|
||||
# the '_')
|
||||
sys.modules[__name__].__dict__["in"] = in_
|
||||
sys.modules[__name__].__dict__["and"] = and_
|
||||
sys.modules[__name__].__dict__["not"] = not_
|
||||
|
||||
def run_test():
|
||||
p_test = sepBy1(choice(ch('1'),
|
||||
ch('2'),
|
||||
ch('3')),
|
||||
ch(','))
|
||||
return p_test.parse("1,2,3")
|
||||
|
||||
# {{{ Automatic parser construction... python specific
|
||||
|
||||
# TODO: Implement Parsable metaclass, which requires the existence of
|
||||
# a "parse" method.
|
||||
|
||||
# This is expected to be extended by user code. As a general rule,
|
||||
# only provide auto-parsers for your own types.
|
||||
AUTO_PARSERS = {
|
||||
str: token,
|
||||
unicode: token,
|
||||
}
|
||||
|
||||
def _auto_seq(lst):
|
||||
return sequence(*(auto_1(p, default_method=_auto_choice)
|
||||
for p in lst))
|
||||
|
||||
def _auto_choice(lst):
|
||||
return choice(*(auto_1(p, default_method=_auto_seq)
|
||||
for p in lst))
|
||||
|
||||
def auto_1(arg, default_method=_auto_choice):
|
||||
if isinstance(arg, Parser):
|
||||
return arg
|
||||
elif type(arg) in AUTO_PARSERS:
|
||||
return AUTO_PARSERS[type(arg)](arg)
|
||||
else:
|
||||
return default_method(arg)
|
||||
|
||||
def auto(*args):
|
||||
return auto_1(args, default_method=_auto_choice)
|
||||
|
||||
# }}}
|
||||
|
|
@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_token("95\xa2")
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2")
|
||||
self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2")
|
||||
def test_partial_fails(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "95"), None)
|
||||
|
||||
|
|
@ -16,8 +16,8 @@ class TestChParser(unittest.TestCase):
|
|||
cls.parser_int = h.h_ch(0xa2)
|
||||
cls.parser_chr = h.h_ch("\xa2")
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2)
|
||||
self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2"))
|
||||
self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2)
|
||||
self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None)
|
||||
self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None)
|
||||
|
|
@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_ch_range("a", "c")
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b"))
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), ord("b"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "d"), None)
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ class TestInt64(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_int64()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None)
|
||||
|
||||
|
|
@ -45,8 +45,8 @@ class TestInt32(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_int32()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None)
|
||||
|
|
@ -56,8 +56,8 @@ class TestInt16(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_int16()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200)
|
||||
self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\xfe"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02"), None)
|
||||
|
|
@ -67,7 +67,7 @@ class TestInt8(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_int8()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, ""), None)
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ class TestUint64(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_uint64()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None)
|
||||
|
||||
|
|
@ -85,7 +85,7 @@ class TestUint32(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_uint32()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None)
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ class TestUint16(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_uint16()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x02"), None)
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ class TestUint8(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_uint8()
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, ""), None)
|
||||
|
||||
|
|
@ -112,7 +112,7 @@ class TestIntRange(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_int_range(h.h_uint8(), 3, 10)
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5)
|
||||
self.assertEqual(h.h_parse(self.parser, "\x05"), 5)
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "\x0b"), None)
|
||||
|
||||
|
|
@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_whitespace(h.h_ch("a"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a"), ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a"), ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "_a"), None)
|
||||
|
||||
|
|
@ -133,8 +133,8 @@ class TestWhitespaceEnd(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_whitespace(h.h_end_p())
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string
|
||||
self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string
|
||||
self.assertEqual(h.h_parse(self.parser, ""), None) # empty string
|
||||
self.assertEqual(h.h_parse(self.parser, " "), None) # empty string
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, " x"), None)
|
||||
|
||||
|
|
@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" "))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "a "), ord("a"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, " "), None)
|
||||
|
|
@ -154,7 +154,7 @@ class TestRight(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a"), ord("a"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, " "), None)
|
||||
|
|
@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" "))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, " a "), ord("a"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, " "), None)
|
||||
|
|
@ -175,14 +175,16 @@ class TestMiddle(unittest.TestCase):
|
|||
self.assertEqual(h.h_parse(self.parser, "ba "), None)
|
||||
self.assertEqual(h.h_parse(self.parser, " ab"), None)
|
||||
|
||||
@unittest.skip("Action not implemented yet")
|
||||
#@unittest.skip("Action not implemented yet")
|
||||
class TestAction(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x])
|
||||
cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]),
|
||||
h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]),
|
||||
lambda x: [chr(y).upper() for y in x])
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"])
|
||||
self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"])
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"])
|
||||
self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"])
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "XX"), None)
|
||||
|
||||
|
|
@ -191,7 +193,7 @@ class TestIn(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_in("abc")
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b"))
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), ord("b"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "d"), None)
|
||||
|
||||
|
|
@ -200,7 +202,7 @@ class TestNotIn(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_not_in("abc")
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d"))
|
||||
self.assertEqual(h.h_parse(self.parser, "d"), ord("d"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
|
||||
|
|
@ -209,7 +211,7 @@ class TestEndP(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"]))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "aa"), None)
|
||||
|
||||
|
|
@ -227,7 +229,7 @@ class TestSequence(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab")))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), None)
|
||||
|
|
@ -237,9 +239,9 @@ class TestSequenceWhitespace(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab")))
|
||||
self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab")))
|
||||
self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab")))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a c"), None)
|
||||
|
||||
|
|
@ -248,8 +250,8 @@ class TestChoice(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")])
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b"))
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), ord("b"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "c"), None)
|
||||
|
||||
|
|
@ -258,8 +260,8 @@ class TestButNot(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "aa"), ord("a"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), None)
|
||||
|
||||
|
|
@ -268,7 +270,7 @@ class TestButNotRange(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4"))
|
||||
self.assertEqual(h.h_parse(self.parser, "4"), ord("4"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "6"), None)
|
||||
|
||||
|
|
@ -277,7 +279,7 @@ class TestDifference(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab")
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), "ab")
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
|
||||
|
|
@ -286,8 +288,8 @@ class TestXor(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9"))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0"))
|
||||
self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9"))
|
||||
self.assertEqual(h.h_parse(self.parser, "0"), ord("0"))
|
||||
self.assertEqual(h.h_parse(self.parser, "9"), ord("9"))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "5"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), None)
|
||||
|
|
@ -297,10 +299,10 @@ class TestMany(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]))
|
||||
def test_success(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, [])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]])
|
||||
self.assertEqual(h.h_parse(self.parser, ""), ())
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a")))
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b")))
|
||||
self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba")))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -309,9 +311,9 @@ class TestMany1(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]))
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"]))
|
||||
self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"]))
|
||||
self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, ""), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None)
|
||||
|
|
@ -321,7 +323,7 @@ class TestRepeatN(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2)
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b')))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "adef"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "dabdef"), None)
|
||||
|
|
@ -331,10 +333,9 @@ class TestOptional(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]])
|
||||
### FIXME check this out in repl, what does tree look like
|
||||
#self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d')))
|
||||
self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d')))
|
||||
self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d')))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "aed"), None)
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), None)
|
||||
|
|
@ -345,7 +346,7 @@ class TestIgnore(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac")))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "ac"), None)
|
||||
|
||||
|
|
@ -354,11 +355,11 @@ class TestSepBy(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(","))
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, [])
|
||||
self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123")))
|
||||
self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132")))
|
||||
self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13")))
|
||||
self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),))
|
||||
self.assertEqual(h.h_parse(self.parser, ""), ())
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -367,10 +368,10 @@ class TestSepBy1(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(","))
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]])
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123")))
|
||||
self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132")))
|
||||
self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13")))
|
||||
self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, ""), None)
|
||||
|
||||
|
|
@ -380,7 +381,7 @@ class TestEpsilonP1(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"]))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -389,7 +390,7 @@ class TestEpsilonP2(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"]))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -398,7 +399,7 @@ class TestEpsilonP3(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"]))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -407,8 +408,8 @@ class TestEpsilonP3(unittest.TestCase):
|
|||
# def setUpClass(cls):
|
||||
# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1])
|
||||
# def test_success(self):
|
||||
# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"])
|
||||
# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"])
|
||||
# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"])
|
||||
# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"])
|
||||
# def test_failure(self):
|
||||
# self.assertEqual(h.h_parse(self.parser, "ab"), None)
|
||||
|
||||
|
|
@ -417,7 +418,7 @@ class TestAnd1(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "0"), (0x30,))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -435,7 +436,7 @@ class TestAnd3(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "12"), (0x31,))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -444,7 +445,7 @@ class TestNot1(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")])
|
||||
def test_success(self):
|
||||
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]])
|
||||
self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"]))
|
||||
def test_failure(self):
|
||||
self.assertEqual(h.h_parse(self.parser, "a++b"), None)
|
||||
|
||||
|
|
@ -453,14 +454,8 @@ class TestNot2(unittest.TestCase):
|
|||
def setUpClass(cls):
|
||||
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")])
|
||||
def test_success(self):
|
||||
tree = h.h_parse(self.parser, "a+b").ast.token_data.seq
|
||||
tree[1] = tree[1].token_data.seq[0]
|
||||
self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]])
|
||||
tree = h.h_parse(self.parser, "a++b").ast.token_data.seq
|
||||
tree[0] = chr(tree[0].token_data.uint)
|
||||
tree[1] = tree[1].token_data.bytes
|
||||
tree[2] = chr(tree[2].token_data.uint)
|
||||
self.assertEqual(tree, ["a", "++", "b"])
|
||||
self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b')))
|
||||
self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b')))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -472,9 +467,9 @@ class TestNot2(unittest.TestCase):
|
|||
# # a = h.h_ch("a")
|
||||
# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
|
||||
# # def test_success(self):
|
||||
# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a")
|
||||
# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "a"), "a")
|
||||
# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"])
|
||||
# # def test_failure(self):
|
||||
# # pass
|
||||
|
||||
|
|
@ -485,15 +480,9 @@ class TestRightrec(unittest.TestCase):
|
|||
a = h.h_ch("a")
|
||||
h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()]))
|
||||
def test_success(self):
|
||||
tree = h.h_parse(self.parser, "a").ast.token_data.seq
|
||||
self.assertEqual(tree[0].token_data.uint, ord("a"))
|
||||
tree = h.h_parse(self.parser, "aa").ast.token_data.seq
|
||||
self.assertEqual(tree[0].token_data.uint, ord("a"))
|
||||
self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
|
||||
tree = h.h_parse(self.parser, "aaa").ast.token_data.seq
|
||||
self.assertEqual(tree[0].token_data.uint, ord("a"))
|
||||
self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a"))
|
||||
self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a"))
|
||||
self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),))
|
||||
self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),)))
|
||||
self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),))))
|
||||
def test_failure(self):
|
||||
pass
|
||||
|
||||
|
|
@ -507,9 +496,9 @@ class TestRightrec(unittest.TestCase):
|
|||
# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
|
||||
# # # this is supposed to be flattened
|
||||
# # def test_success(self):
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"])
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"])
|
||||
# # def test_failure(self):
|
||||
# # self.assertEqual(h.h_parse(self.parser, "d+"), None)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
%module hammer
|
||||
%nodefaultctor;
|
||||
//%nodefaultdtor;
|
||||
|
||||
%include "stdint.i"
|
||||
//%include "typemaps.i"
|
||||
|
|
@ -7,31 +8,6 @@
|
|||
|
||||
#if defined(SWIGPYTHON)
|
||||
%ignore HCountedArray_;
|
||||
%typemap(in) uint8_t* {
|
||||
Py_INCREF($input);
|
||||
$1 = (uint8_t*)PyString_AsString($input);
|
||||
}
|
||||
%typemap(out) uint8_t* {
|
||||
$result = PyString_FromString((char*)$1);
|
||||
}
|
||||
|
||||
%typemap(newfree) HParseResult* {
|
||||
h_parse_result_free($1);
|
||||
}
|
||||
|
||||
%newobject h_parse
|
||||
%delobject h_parse_result_free
|
||||
|
||||
/*
|
||||
%typemap(in) (uint8_t* str, size_t len) {
|
||||
if (PyString_Check($input) ||
|
||||
PyUnicode_Check($input)) {
|
||||
PyString_AsStringAndSize($input, (char**)&$1, &$2);
|
||||
} else {
|
||||
PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode");
|
||||
}
|
||||
}
|
||||
*/
|
||||
%apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)}
|
||||
%apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)}
|
||||
%apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)}
|
||||
|
|
@ -79,6 +55,43 @@
|
|||
PyList_SetItem($result, i, o);
|
||||
}
|
||||
}
|
||||
%typemap(out) struct HParseResult_* {
|
||||
if ($1 == NULL) {
|
||||
// TODO: raise parse failure
|
||||
Py_INCREF(Py_None);
|
||||
$result = Py_None;
|
||||
} else {
|
||||
$result = hpt_to_python($1->ast);
|
||||
}
|
||||
}
|
||||
|
||||
%inline %{
|
||||
static int h_tt_python;
|
||||
%}
|
||||
%init %{
|
||||
h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python");
|
||||
%}
|
||||
|
||||
/*
|
||||
%typemap(in) (HPredicate* pred, void* user_data) {
|
||||
Py_INCREF($input);
|
||||
$2 = $input;
|
||||
$1 = call_predicate;
|
||||
}
|
||||
*/
|
||||
%typemap(in) (const HAction a, void* user_data) {
|
||||
Py_INCREF($input);
|
||||
$2 = $input;
|
||||
$1 = call_action;
|
||||
}
|
||||
|
||||
%inline {
|
||||
struct HParsedToken_;
|
||||
struct HParseResult_;
|
||||
static PyObject* hpt_to_python(const struct HParsedToken_ *token);
|
||||
|
||||
static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data);
|
||||
}
|
||||
#else
|
||||
#warning no uint8_t* typemaps defined
|
||||
#endif
|
||||
|
|
@ -92,3 +105,74 @@
|
|||
%include "allocator.h"
|
||||
%include "hammer.h"
|
||||
|
||||
|
||||
%extend HArena_ {
|
||||
~HArena_() {
|
||||
h_delete_arena($self);
|
||||
}
|
||||
};
|
||||
%extend HParseResult_ {
|
||||
~HParseResult_() {
|
||||
h_parse_result_free($self);
|
||||
}
|
||||
};
|
||||
|
||||
%newobject h_parse;
|
||||
%delobject h_parse_result_free;
|
||||
%newobject h_new_arena;
|
||||
%delobject h_delete_arena;
|
||||
|
||||
#ifdef SWIGPYTHON
|
||||
%inline {
|
||||
static PyObject* hpt_to_python(const HParsedToken *token) {
|
||||
// Caller holds a reference to returned object
|
||||
PyObject *ret;
|
||||
if (token == NULL) {
|
||||
Py_RETURN_NONE;
|
||||
}
|
||||
switch (token->token_type) {
|
||||
case TT_NONE:
|
||||
Py_RETURN_NONE;
|
||||
break;
|
||||
case TT_BYTES:
|
||||
return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
|
||||
case TT_SINT:
|
||||
// TODO: return PyINT if appropriate
|
||||
return PyLong_FromLong(token->token_data.sint);
|
||||
case TT_UINT:
|
||||
// TODO: return PyINT if appropriate
|
||||
return PyLong_FromUnsignedLong(token->token_data.uint);
|
||||
case TT_SEQUENCE:
|
||||
ret = PyTuple_New(token->token_data.seq->used);
|
||||
for (int i = 0; i < token->token_data.seq->used; i++) {
|
||||
PyTuple_SET_ITEM(ret, i, hpt_to_python(token->token_data.seq->elements[i]));
|
||||
}
|
||||
return ret;
|
||||
default:
|
||||
if (token->token_type == h_tt_python) {
|
||||
ret = (PyObject*)token->token_data.user;
|
||||
Py_INCREF(ret);
|
||||
return ret;
|
||||
} else {
|
||||
return SWIG_NewPointerObj((void*)token, SWIGTYPE_p_HParsedToken_, 0 | 0);
|
||||
// TODO: support registry
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data) {
|
||||
PyObject *callable = user_data;
|
||||
PyObject *ret = PyObject_CallFunctionObjArgs(callable,
|
||||
hpt_to_python(p->ast),
|
||||
NULL);
|
||||
if (ret == NULL) {
|
||||
PyErr_Print();
|
||||
assert(ret != NULL);
|
||||
}
|
||||
// TODO: add reference to ret to parse-local data
|
||||
HParsedToken *tok = h_make(p->arena, h_tt_python, ret);
|
||||
return tok;
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue