kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity.

This commit is contained in:
Meredith L. Patterson 2013-11-19 02:41:45 -06:00
parent f3ce2fc7de
commit cec4659baa
3 changed files with 116 additions and 93 deletions

View file

@ -16,4 +16,6 @@ swig = ['hammer.i']
libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
pytestenv = pythonenv.Clone()
pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE")
pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE")
Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc'])

View file

@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase):
self.assertEqual(h.h_parse(self.parser, "ba ", 3), None)
self.assertEqual(h.h_parse(self.parser, " ab", 3), None)
class TestAction(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x])
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"])
self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"])
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "XX", 2), None)
# class TestAction(unittest.TestCase):
# @classmethod
# def setUpClass(cls):
# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x])
# def test_success(self):
# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"])
# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"])
# def test_failure(self):
# self.assertEqual(h.h_parse(self.parser, "XX", 2), None)
class TestIn(unittest.TestCase):
@classmethod
@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase):
def setUpClass(cls):
cls.parser = h.h_not_in("abc", 3)
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes!
self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d"))
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "a", 1), None)
class TestEndP(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p())
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
def test_failure(self):
### failing: parses a single 'a', dunno why
self.assertEqual(h.h_parse(self.parser, "aa", 2), None)
class TestNothingP(unittest.TestCase):
@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase):
class TestSequence(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
def test_failure(self):
@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase):
class TestSequenceWhitespace(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b")))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase):
class TestChoice(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b"))
cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a"))
self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b"))
@ -295,7 +294,7 @@ class TestXor(unittest.TestCase):
class TestMany(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b")))
cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]))
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, [])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
@ -307,7 +306,7 @@ class TestMany(unittest.TestCase):
class TestMany1(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b")))
cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]))
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]])
@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase):
class TestRepeatN(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2)
cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2)
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "adef", 4), None)
self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None)
@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase):
class TestOptional(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]])
### FIXME check this out in repl, what does tree look like
#self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]])
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "aed", 3), None)
self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase):
class TestIgnore(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]])
def test_failure(self):
@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase):
class TestSepBy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(","))
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase):
class TestSepBy1(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(","))
cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(","))
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]])
@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase):
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "", 0), None)
### segfaults
class TestEpsilonP1(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]])
def test_failure(self):
@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase):
class TestEpsilonP2(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a"))
cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
def test_failure(self):
@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase):
class TestEpsilonP3(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p())
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
def test_failure(self):
pass
class TestAttrBool(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1])
def test_success(self):
self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"])
def test_failure(self):
self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
# class TestAttrBool(unittest.TestCase):
# @classmethod
# def setUpClass(cls):
# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1])
# def test_success(self):
# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"])
# def test_failure(self):
# self.assertEqual(h.h_parse(self.parser, "ab", 2), None)
class TestAnd1(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0"))
cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None])
def test_success(self):
### failing: [] != ["0"]. Token type is sequence.
self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]])
def test_failure(self):
pass
class TestAnd2(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1"))
cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None])
def test_success(self):
pass
def test_failure(self):
### failing: [] is not None, parse should have failed
self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None)
self.assertEqual(h.h_parse(self.parser, "0", 1), None)
class TestAnd3(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2")))
cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]])
def test_failure(self):
@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase):
class TestNot1(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]])
def test_failure(self):
@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase):
class TestNot2(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b"))
cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None])
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"])
self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"])
tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq
tree[1] = tree[1].token_data.seq[0]
self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]])
tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq
tree[0] = chr(tree[0].token_data.uint)
tree[1] = tree[1].token_data.bytes
tree[2] = chr(tree[2].token_data.uint)
self.assertEqual(tree, ["a", "++", "b"])
def test_failure(self):
pass
### this is commented out for packrat in C ...
#class TestLeftrec(unittest.TestCase):
# ### this is commented out for packrat in C ...
# #class TestLeftrec(unittest.TestCase):
# # @classmethod
# # def setUpClass(cls):
# # cls.parser = h.h_indirect()
# # a = h.h_ch("a")
# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
# # def test_success(self):
# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
# # def test_failure(self):
# # pass
# class TestRightrec(unittest.TestCase):
# @classmethod
# def setUpClass(cls):
# cls.parser = h.h_indirect()
# a = h.h_ch("a")
# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a))
# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()]))
# def test_success(self):
# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a")
# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"])
# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"])
# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
# def test_failure(self):
# pass
class TestRightrec(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.parser = h.h_indirect()
a = h.h_ch("a")
h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p()))
def test_success(self):
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]])
self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]])
def test_failure(self):
pass
### this is just for GLR
#class TestAmbiguous(unittest.TestCase):
# @classmethod
# def setUpClass(cls):
# cls.parser = h.h_indirect()
# d = h.h_ch("d")
# p = h.h_ch("+")
# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
# # this is supposed to be flattened
# def test_success(self):
# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
# def test_failure(self):
# self.assertEqual(h.h_parse(self.parser, "d+", 2), None)
# ### this is just for GLR
# #class TestAmbiguous(unittest.TestCase):
# # @classmethod
# # def setUpClass(cls):
# # cls.parser = h.h_indirect()
# # d = h.h_ch("d")
# # p = h.h_ch("+")
# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d))
# # # this is supposed to be flattened
# # def test_success(self):
# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"])
# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"])
# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"])
# # def test_failure(self):
# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None)

View file

@ -1,4 +1,5 @@
%module hammer
%nodefaultctor;
%include "stdint.i"
//%include "typemaps.i"
@ -7,11 +8,31 @@
#if defined(SWIGPYTHON)
%ignore HCountedArray_;
%typemap(in) uint8_t* {
Py_INCREF($input);
$1 = (uint8_t*)PyString_AsString($input);
}
%typemap(out) uint8_t* {
$result = PyString_FromString((char*)$1);
}
%typemap(in) void*[] {
if (PyList_Check($input)) {
Py_INCREF($input);
int size = PyList_Size($input);
int i = 0;
int res = 0;
$1 = (void**)malloc(size*sizeof(HParser*));
for (i=0; i<size; i++) {
PyObject *o = PyList_GetItem($input, i);
res = SWIG_ConvertPtr(o, &($1[i]), SWIGTYPE_p_HParser_, 0 | 0);
if (!SWIG_IsOK(res)) {
SWIG_exception_fail(SWIG_ArgError(res), "that wasn't an HParser" );
}
}
} else {
PyErr_SetString(PyExc_TypeError, "__a functions take lists of parsers as their argument");
return NULL;
}
}
%typemap(in) uint8_t {
if (PyInt_Check($input)) {
$1 = PyInt_AsLong($input);
@ -23,11 +44,6 @@
$1 = *(uint8_t*)PyString_AsString($input);
}
}
/*
%typemap(out) uint8_t {
$result = PyString_FromString(&$1);
}
*/
%typemap(out) HBytes* {
$result = PyString_FromStringAndSize((char*)$1->token, $1->len);
}