diff --git a/README.md b/README.md index ee62558..d122fc8 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ Installing * pkg-config (for `scons test`) * glib-2.0 (>= 2.29) (for `scons test`) * glib-2.0-dev (for `scons test`) -* [swig](http://swig.org/) (for Python/Perl/PHP bindings; Perl requires >= 2.0.8) -* python2.7-dev (for Python bindings) +* [swig](http://swig.org/) (for Python/Perl/PHP bindings; Perl requires >= 2.0.8; Python 3.x requires >= 3.0.0) +* python2.7-dev (for Python 2 bindings) +* python3-dev (>= 3.5) (for Python 3 bindings) * a JDK (for Java bindings) * a working [phpenv](https://github.com/CHH/phpenv) configuration (for PHP bindings) * [Ruby](https://www.ruby-lang.org/) >= 1.9.3 and bundler, for the Ruby bindings @@ -73,7 +74,7 @@ The `examples/` directory contains some simple examples, currently including: Known Issues ============ -The Python bindings only work with Python 2.7. SCons doesn't work with Python 3, and PyCapsule isn't available in 2.6 and below, so 2.7 is all you get. Sorry about that. +The Python bindings work with Python 2.7, and Python 3.5+. The requirement for SWIG >= 2.0.8 for Perl bindings is due to a [known bug](http://sourceforge.net/p/swig/patches/324/) in SWIG. [ppa:dns/irc](https://launchpad.net/~dns/+archive/irc) has backports of SWIG 2.0.8 for Ubuntu versions 10.04-12.10; you can also [build SWIG from source](http://www.swig.org/download.html). diff --git a/SConstruct b/SConstruct index a906390..8aa8c81 100644 --- a/SConstruct +++ b/SConstruct @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os import os.path import platform @@ -12,6 +15,7 @@ vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept)) vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby'])) +vars.Add('python', 'Python interpreter', 'python') tools = ['default', 'scanreplace'] if 'dotnet' in ARGUMENTS.get('bindings', []): @@ -43,9 +47,9 @@ env['prefix'] = os.path.abspath(env['prefix']) if 'DESTDIR' in env: env['DESTDIR'] = os.path.abspath(env['DESTDIR']) if rel_prefix: - print >>sys.stderr, '--!!-- You used a relative prefix with a DESTDIR. This is probably not what you' - print >>sys.stderr, '--!!-- you want; files will be installed in' - print >>sys.stderr, '--!!-- %s' % (calcInstallPath('$prefix'),) + print('--!!-- You used a relative prefix with a DESTDIR. This is probably not what you', file=sys.stderr) + print('--!!-- you want; files will be installed in', file=sys.stderr) + print('--!!-- %s' % (calcInstallPath('$prefix'),), file=sys.stderr) env['libpath'] = calcInstallPath('$prefix', 'lib') diff --git a/examples/SConscript b/examples/SConscript index 0694721..b34b85a 100644 --- a/examples/SConscript +++ b/examples/SConscript @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function + Import('env') example = env.Clone() diff --git a/examples/base64.py b/examples/base64.py index 3ffe304..f7b68ab 100644 --- a/examples/base64.py +++ b/examples/base64.py @@ -10,7 +10,7 @@ # base64_sem1.py and base64_sem2.py for examples how to attach appropriate # semantic actions to the grammar. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import sys @@ -23,13 +23,13 @@ def init_parser(): alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.ch('=') + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.ch(b'=') bsfdig = h.choice(alpha, digit, plus, slash) - bsfdig_4bit = h.in_('AEIMQUYcgkosw048') - bsfdig_2bit = h.in_('AQgw') + bsfdig_4bit = h.in_(b'AEIMQUYcgkosw048') + bsfdig_2bit = h.in_(b'AQgw') base64_3 = h.repeat_n(bsfdig, 4) base64_2 = h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals) base64_1 = h.sequence(bsfdig, bsfdig_2bit, equals, equals) diff --git a/examples/base64_sem1.py b/examples/base64_sem1.py index f0676eb..84cb2d7 100644 --- a/examples/base64_sem1.py +++ b/examples/base64_sem1.py @@ -13,7 +13,7 @@ # transform the parse tree in small steps in a bottom-up fashion. Compare # base64_sem2.py for an alternative approach using a single top-level action. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import sys @@ -26,7 +26,7 @@ import hammer as h def act_bsfdig(p, user_data=None): # FIXME See the note in init_parser() - c = p if isinstance(p, (int, long)) else ord(p) + c = p if isinstance(p, h.INTEGER_TYPES) else ord(p) if 0x41 <= c <= 0x5A: # A-Z return c - 0x41 @@ -34,9 +34,9 @@ def act_bsfdig(p, user_data=None): return c - 0x61 + 26 elif 0x30 <= c <= 0x39: # 0-9 return c - 0x30 + 52 - elif c == '+': + elif c == b'+': return 62 - elif c == '/': + elif c == b'/': return 63 else: raise ValueError @@ -65,14 +65,14 @@ def act_base64_n(n, p, user_data=None): x = 0 bits = 0 - for i in xrange(0, n+1): + for i in range(0, n+1): x <<= 6 x |= p[i] or 0 bits += 6 x >>= bits % 8 # align, i.e. cut off extra bits - for i in xrange(n): + for i in range(n): item = x & 0xFF res[n-1-i] = item # output the last byte and @@ -118,16 +118,16 @@ def init_parser(): # literals, or integers digit = h.ch_range(0x30, 0x39) alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) - space = h.in_(" \t\n\r\f\v") + space = h.in_(b" \t\n\r\f\v") # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.action(h.ch('='), act_equals) + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.action(h.ch(b'='), act_equals) bsfdig = h.action(h.choice(alpha, digit, plus, slash), act_bsfdig) - bsfdig_4bit = h.action(h.in_("AEIMQUYcgkosw048"), act_bsfdig_4bit) - bsfdig_2bit = h.action(h.in_("AQgw"), act_bsfdig_2bit) + bsfdig_4bit = h.action(h.in_(b"AEIMQUYcgkosw048"), act_bsfdig_4bit) + bsfdig_2bit = h.action(h.in_(b"AQgw"), act_bsfdig_2bit) base64_3 = h.action(h.repeat_n(bsfdig, 4), act_base64_3) base64_2 = h.action(h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals), act_base64_2) diff --git a/examples/base64_sem2.py b/examples/base64_sem2.py index 6b5f8db..2e33774 100644 --- a/examples/base64_sem2.py +++ b/examples/base64_sem2.py @@ -14,7 +14,7 @@ # for an alternative approach using a fine-grained piece-by-piece # transformation. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import sys @@ -28,7 +28,7 @@ import hammer as h def bsfdig_value(p): """Return the numeric value of a parsed base64 digit. """ - c = p if isinstance(p, (int, long)) else ord(p) + c = p if isinstance(p, h.INTEGER_TYPES) else ord(p) if c: if 0x41 <= c <= 0x5A: # A-Z return c - 0x41 @@ -36,9 +36,9 @@ def bsfdig_value(p): return c - 0x61 + 26 elif 0x30 <= c <= 0x39: # 0-9 return c - 0x30 + 52 - elif c == '+': + elif c == b'+': return 62 - elif c == '/': + elif c == b'/': return 63 return 0 @@ -109,16 +109,16 @@ def init_parser(): # CORE digit = h.ch_range(0x30, 0x39) alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) - space = h.in_(" \t\n\r\f\v") + space = h.in_(b" \t\n\r\f\v") # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.ch('=') + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.ch(b'=') bsfdig = h.choice(alpha, digit, plus, slash) - bsfdig_4bit = h.in_("AEIMQUYcgkosw048") - bsfdig_2bit = h.in_("AQgw") + bsfdig_4bit = h.in_(b"AEIMQUYcgkosw048") + bsfdig_2bit = h.in_(b"AQgw") base64_3 = h.repeat_n(bsfdig, 4) base64_2 = h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals) base64_1 = h.sequence(bsfdig, bsfdig_2bit, equals, equals) diff --git a/src/SConscript b/src/SConscript index a46ddac..0c4f81e 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import('env testruns') @@ -66,7 +69,8 @@ misc_hammer_parts = [ 'hammer.c', 'pprint.c', 'registry.c', - 'system_allocator.c'] + 'system_allocator.c', + 'sloballoc.c'] if env['PLATFORM'] == 'win32': misc_hammer_parts += [ @@ -82,7 +86,8 @@ ctests = ['t_benchmark.c', 't_parser.c', 't_grammar.c', 't_misc.c', - 't_regression.c'] + 't_mm.c', + 't_regression.c'] static_library_name = 'hammer' diff --git a/src/bindings/cpp/SConscript b/src/bindings/cpp/SConscript index 9555b98..385759e 100644 --- a/src/bindings/cpp/SConscript +++ b/src/bindings/cpp/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/dotnet/SConscript b/src/bindings/dotnet/SConscript index afa4c30..416f374 100644 --- a/src/bindings/dotnet/SConscript +++ b/src/bindings/dotnet/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/perl/SConscript b/src/bindings/perl/SConscript index 8a192a5..c918432 100644 --- a/src/bindings/perl/SConscript +++ b/src/bindings/perl/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/php/SConscript b/src/bindings/php/SConscript index 6791cbc..b404f17 100644 --- a/src/bindings/php/SConscript +++ b/src/bindings/php/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os, os.path Import('env libhammer_shared testruns') diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 5c7e474..2ca8ea0 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os, os.path Import('env libhammer_shared testruns targets') @@ -7,17 +10,18 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) swig = pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) setup = ['setup.py'] pydir = os.path.join(env['BUILD_BASE'], 'src/bindings/python') -libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --swig=swig3.0 --inplace') +pysetup = os.path.join(pydir, 'setup.py') +libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], '%s %s build_ext --inplace' % (env['python'], pysetup)) Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) pytests = ['hammer_tests.py'] -pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=" + os.path.dirname(str(libhammer_shared[0])) + " nosetests -vv $SOURCE") +pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=%s %s -mnose -vv $SOURCE" % (os.path.dirname(str(libhammer_shared[0])), env['python'])) pytest = Alias("testpython", [pytestexec], pytestexec) AlwaysBuild(pytestexec) testruns.append(pytest) -pyinstallexec = pythonenv.Command(None, libhammer_python, 'python ' + os.path.join(pydir, 'setup.py ') + ' install') +pyinstallexec = pythonenv.Command(None, libhammer_python, '%s %s install' % (env['python'], pysetup)) pyinstall = Alias("installpython", [pyinstallexec], pyinstallexec) targets.append(pyinstall) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 45a63f4..0aead60 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -1,218 +1,220 @@ +from __future__ import absolute_import, division, print_function + import unittest import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.token(b"95\xa2") def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(self.parser.parse(b"95\xa2"), b"95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(self.parser.parse(b"95"), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_chr = h.ch(b"\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(self.parser_int.parse(b"\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse(b"\xa2"), b"\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(self.parser_int.parse(b"\xa3"), None) + self.assertEqual(self.parser_chr.parse(b"\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") + cls.parser = h.ch_range(b"a", b"c") def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(self.parser.parse(b"d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(self.parser.parse(b"\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(self.parser.parse(b"\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse(b"\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse(b"\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(self.parser.parse(b"\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(self.parser.parse(b"\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse(b"\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(self.parser.parse(b"\xfe"), None) + self.assertEqual(self.parser.parse(b"\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(self.parser.parse(b"\x88"), -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(self.parser.parse(b"\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse(b"\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(self.parser.parse(b"\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(self.parser.parse(b"\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(self.parser.parse(b"\x78"), 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int_range(h.uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(self.parser.parse(b"\x05"), 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(self.parser.parse(b"\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.whitespace(h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b" a"), b"a") + self.assertEqual(self.parser.parse(b" a"), b"a") + self.assertEqual(self.parser.parse(b"\ta"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(self.parser.parse(b"_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.whitespace(h.end_p()) def test_success(self): - self.assertEqual(self.parser.parse(""), None) # empty string - self.assertEqual(self.parser.parse(" "), None) # empty string + self.assertEqual(self.parser.parse(b""), None) # empty string + self.assertEqual(self.parser.parse(b" "), None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(self.parser.parse(b" x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.left(h.ch(b"a"), h.ch(b" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(self.parser.parse(b"a "), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.right(h.ch(b" "), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(b" a"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b"ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.middle(h.ch(b" "), h.ch(b"a"), h.ch(b" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(self.parser.parse(b" a "), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b" a"), None) + self.assertEqual(self.parser.parse(b"a "), None) + self.assertEqual(self.parser.parse(b" b "), None) + self.assertEqual(self.parser.parse(b"ba "), None) + self.assertEqual(self.parser.parse(b" ab"), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), - h.choice(h.ch("b"), h.ch("B"))), + cls.parser = h.action(h.sequence(h.choice(h.ch(b"a"), h.ch(b"A")), + h.choice(h.ch(b"b"), h.ch(b"B"))), lambda x: [y.upper() for y in x]) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["A", "B"]) - self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + self.assertEqual(self.parser.parse(b"ab"), [b"A", b"B"]) + self.assertEqual(self.parser.parse(b"AB"), [b"A", b"B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(self.parser.parse(b"XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.in_(b"abc") def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(self.parser.parse(b"d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.not_in(b"abc") def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(self.parser.parse(b"d"), b"d") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.sequence(h.ch(b"a"), h.end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(self.parser.parse(b"aa"), None) class TestNothingP(unittest.TestCase): @classmethod @@ -221,244 +223,244 @@ class TestNothingP(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ('a','b')) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b"b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.sequence(h.ch(b"a"), h.whitespace(h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ('a','b')) - self.assertEqual(self.parser.parse("a b"), ('a','b')) - self.assertEqual(self.parser.parse("a b"), ('a','b')) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) + self.assertEqual(self.parser.parse(b"a b"), (b"a", b"b")) + self.assertEqual(self.parser.parse(b"a b"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(self.parser.parse(b"a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.choice(h.ch(b"a"), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(self.parser.parse(b"c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.butnot(h.ch(b"a"), h.token(b"ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b"aa"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) + cls.parser = h.butnot(h.ch_range(b"0", b"9"), h.ch(b"6")) def test_success(self): - self.assertEqual(self.parser.parse("4"), "4") + self.assertEqual(self.parser.parse(b"4"), b"4") def test_failure(self): - self.assertEqual(self.parser.parse("6"), None) + self.assertEqual(self.parser.parse(b"6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.difference(h.token(b"ab"), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(self.parser.parse(b"ab"), b"ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) + cls.parser = h.xor(h.ch_range(b"0", b"6"), h.ch_range(b"5", b"9")) def test_success(self): - self.assertEqual(self.parser.parse("0"), "0") - self.assertEqual(self.parser.parse("9"), "9") + self.assertEqual(self.parser.parse(b"0"), b"0") + self.assertEqual(self.parser.parse(b"9"), b"9") def test_failure(self): - self.assertEqual(self.parser.parse("5"), None) - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"5"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.many(h.choice(h.ch(b"a"), h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), ()) - self.assertEqual(self.parser.parse("a"), ('a',)) - self.assertEqual(self.parser.parse("b"), ('b',)) - self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a')) + self.assertEqual(self.parser.parse(b""), ()) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"b"), (b"b",)) + self.assertEqual(self.parser.parse(b"aabbaba"), (b"a", b"a", b"b", b"b", b"a", b"b", b"a")) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.many1(h.choice(h.ch(b"a"), h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) - self.assertEqual(self.parser.parse("b"), ("b",)) - self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"b"), (b"b",)) + self.assertEqual(self.parser.parse(b"aabbaba"), (b"a", b"a", b"b", b"b", b"a", b"b", b"a")) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(self.parser.parse(b""), None) + self.assertEqual(self.parser.parse(b"daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.repeat_n(h.choice(h.ch(b"a"), h.ch(b"b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ('a', 'b')) + self.assertEqual(self.parser.parse(b"abdef"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(self.parser.parse(b"adef"), None) + self.assertEqual(self.parser.parse(b"dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.sequence(h.ch(b"a"), h.optional(h.choice(h.ch(b"b"), h.ch(b"c"))), h.ch(b"d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ('a','b','d')) - self.assertEqual(self.parser.parse("acd"), ('a','c','d')) - self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd')) + self.assertEqual(self.parser.parse(b"abd"), (b"a", b"b", b"d")) + self.assertEqual(self.parser.parse(b"acd"), (b"a", b"c", b"d")) + self.assertEqual(self.parser.parse(b"ad"), (b"a", h.Placeholder(), b"d")) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(self.parser.parse(b"aed"), None) + self.assertEqual(self.parser.parse(b"ab"), None) + self.assertEqual(self.parser.parse(b"ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.sequence(h.ch(b"a"), h.ignore(h.ch(b"b")), h.ch(b"c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ("a","c")) + self.assertEqual(self.parser.parse(b"abc"), (b"a",b"c")) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(self.parser.parse(b"ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.sepBy(h.choice(h.ch(b"1"), h.ch(b"2"), h.ch(b"3")), h.ch(b",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) - self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) - self.assertEqual(self.parser.parse("1,3"), ('1','3')) - self.assertEqual(self.parser.parse("3"), ('3',)) - self.assertEqual(self.parser.parse(""), ()) + self.assertEqual(self.parser.parse(b"1,2,3"), (b"1", b"2", b"3")) + self.assertEqual(self.parser.parse(b"1,3,2"), (b"1", b"3", b"2")) + self.assertEqual(self.parser.parse(b"1,3"), (b"1", b"3")) + self.assertEqual(self.parser.parse(b"3"), (b"3",)) + self.assertEqual(self.parser.parse(b""), ()) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.sepBy1(h.choice(h.ch(b"1"), h.ch(b"2"), h.ch(b"3")), h.ch(b",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) - self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) - self.assertEqual(self.parser.parse("1,3"), ('1','3')) - self.assertEqual(self.parser.parse("3"), ('3',)) + self.assertEqual(self.parser.parse(b"1,2,3"), (b"1", b"2", b"3")) + self.assertEqual(self.parser.parse(b"1,3,2"), (b"1", b"3", b"2")) + self.assertEqual(self.parser.parse(b"1,3"), (b"1", b"3")) + self.assertEqual(self.parser.parse(b"3"), (b"3",)) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.epsilon_p(), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ("a", "b")) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.sequence(h.epsilon_p(), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.sequence(h.ch(b"a"), h.epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): pass class TestAttrBool(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), + cls.parser = h.attr_bool(h.many1(h.choice(h.ch(b"a"), h.ch(b"b"))), lambda x: x[0] == x[1]) def test_success(self): - self.assertEqual(self.parser.parse("aa"), ("a", "a")) - self.assertEqual(self.parser.parse("bb"), ("b", "b")) + self.assertEqual(self.parser.parse(b"aa"), (b"a", b"a")) + self.assertEqual(self.parser.parse(b"bb"), (b"b", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.sequence(h.and_(h.ch(b"0")), h.ch(b"0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ("0",)) + self.assertEqual(self.parser.parse(b"0"), (b"0",)) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.sequence(h.and_(h.ch(b"0")), h.ch(b"1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(self.parser.parse(b"0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.sequence(h.ch(b"1"), h.and_(h.ch(b"2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ('1',)) + self.assertEqual(self.parser.parse(b"12"), (b"1",)) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), - h.choice(h.ch("+"), h.token("++")), - h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), + h.choice(h.ch(b"+"), h.token(b"++")), + h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b")) + self.assertEqual(self.parser.parse(b"a+b"), (b"a", b"+", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(self.parser.parse(b"a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), - h.token("++")), - h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.choice(h.sequence(h.ch(b"+"), h.not_(h.ch(b"+"))), + h.token(b"++")), + h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b')) - self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b')) + self.assertEqual(self.parser.parse(b"a+b"), (b"a", (b"+",), b"b")) + self.assertEqual(self.parser.parse(b"a++b"), (b"a", b"++", b"b")) def test_failure(self): pass @@ -467,12 +469,12 @@ class TestNot2(unittest.TestCase): # # @classmethod # # def setUpClass(cls): # # cls.parser = h.indirect() -# # a = h.ch("a") +# # a = h.ch(b"a") # # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(self.parser.parse("a"), "a") -# # self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) +# # self.assertEqual(self.parser.parse(b"a"), b"a") +# # self.assertEqual(self.parser.parse(b"aa"), [b"a", b"a"]) +# # self.assertEqual(self.parser.parse(b"aaa"), [b"a", b"a", b"a"]) # # def test_failure(self): # # pass @@ -480,15 +482,15 @@ class TestNot2(unittest.TestCase): class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - #raise unittest.SkipTest("Bind doesn't work right now") + #raise unittest.SkipTest(b"Bind doesn't work right now") cls.parser = h.indirect() - a = h.ch("a") + a = h.ch(b"a") cls.parser.bind(h.choice(h.sequence(a, cls.parser), h.epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ('a',)) - self.assertEqual(self.parser.parse("aa"), ('a', ('a',))) - self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',)))) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"aa"), (b"a", (b"a",))) + self.assertEqual(self.parser.parse(b"aaa"), (b"a", (b"a", (b"a",)))) def test_failure(self): pass @@ -497,13 +499,13 @@ class TestRightrec(unittest.TestCase): # # @classmethod # # def setUpClass(cls): # # cls.parser = h.indirect() -# # d = h.ch("d") -# # p = h.ch("+") +# # d = h.ch(b"d") +# # p = h.ch(b"+") # # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(self.parser.parse("d"), ["d"]) -# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# # self.assertEqual(self.parser.parse(b"d"), [b"d"]) +# # self.assertEqual(self.parser.parse(b"d+d"), [b"d", b"+", b"d"]) +# # self.assertEqual(self.parser.parse(b"d+d+d"), [b"d", b"+", b"d", b"+", b"d"]) # # def test_failure(self): -# # self.assertEqual(self.parser.parse("d+"), None) +# # self.assertEqual(self.parser.parse(b"d+"), None) diff --git a/src/bindings/ruby/SConscript b/src/bindings/ruby/SConscript index 6d85a93..d50b644 100644 --- a/src/bindings/ruby/SConscript +++ b/src/bindings/ruby/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 122ffe4..0097bdc 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,7 @@ %module hammer +%begin %{ +#define SWIG_PYTHON_STRICT_BYTE_CHAR +%} %nodefaultctor; @@ -25,6 +28,20 @@ } %pythoncode %{ + try: + INTEGER_TYPES = (int, long) + except NameError: + INTEGER_TYPES = (int,) + + try: + TEXT_TYPE = unicode + def bchr(i): + return chr(i) + except NameError: + TEXT_TYPE = str + def bchr(i): + return bytes([i]) + class Placeholder(object): """The python equivalent of TT_NONE""" def __str__(self): @@ -69,11 +86,11 @@ PyErr_SetString(PyExc_ValueError, "Expecting a string"); return NULL; } else { - $1 = *(uint8_t*)PyString_AsString($input); + $1 = *(uint8_t*)PyBytes_AsString($input); } } %typemap(out) HBytes* { - $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + $result = PyBytes_FromStringAndSize((char*)$1->token, $1->len); } %typemap(out) struct HCountedArray_* { int i; @@ -173,7 +190,7 @@ return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL); break; case TT_BYTES: - return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); + return PyBytes_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); case TT_SINT: // TODO: return PyINT if appropriate return PyLong_FromLong(token->token_data.sint); @@ -250,36 +267,35 @@ } %pythoncode %{ - def action(p, act): return _h_action(p, act) def attr_bool(p, pred): return _h_attr_bool(p, pred) def ch(ch): - if isinstance(ch, str) or isinstance(ch, unicode): + if isinstance(ch, (bytes, TEXT_TYPE)): return token(ch) else: return _h_ch(ch) def ch_range(c1, c2): - dostr = isinstance(c1, str) - dostr2 = isinstance(c2, str) - if isinstance(c1, unicode) or isinstance(c2, unicode): + dostr = isinstance(c1, bytes) + dostr2 = isinstance(c2, bytes) + if isinstance(c1, TEXT_TYPE) or isinstance(c2, TEXT_TYPE): raise TypeError("ch_range only works on bytes") if dostr != dostr2: raise TypeError("Both arguments to ch_range must be the same type") if dostr: - return action(_h_ch_range(c1, c2), chr) + return action(_h_ch_range(c1, c2), bchr) else: return _h_ch_range(c1, c2) def epsilon_p(): return _h_epsilon_p() def end_p(): return _h_end_p() def in_(charset): - return action(_h_in(charset), chr) + return action(_h_in(charset), bchr) def not_in(charset): - return action(_h_not_in(charset), chr) + return action(_h_not_in(charset), bchr) def not_(p): return _h_not(p) def int_range(p, i1, i2): return _h_int_range(p, i1, i2) diff --git a/src/hammer.h b/src/hammer.h index 984df31..ad44fee 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -804,6 +804,9 @@ HTokenType h_get_token_type_number(const char* name); const char* h_get_token_type_name(HTokenType token_type); // }}} +/// Make an allocator that draws from the given memory area. +HAllocator *h_sloballoc(void *mem, size_t size); + #ifdef __cplusplus } #endif diff --git a/src/sloballoc.c b/src/sloballoc.c new file mode 100644 index 0000000..e164cab --- /dev/null +++ b/src/sloballoc.c @@ -0,0 +1,216 @@ +// first-fit SLOB (simple list of blocks) allocator + +#include "sloballoc.h" +#include +#include + +struct alloc { + size_t size; + uint8_t data[]; +}; + +struct block { + size_t size; // read: struct alloc + struct block *next; +}; + +struct slob { + size_t size; + struct block *head; + uint8_t data[]; +}; + + +SLOB *slobinit(void *mem, size_t size) +{ + SLOB *slob = mem; + + assert(size >= sizeof(SLOB) + sizeof(struct block)); + assert(size < UINTPTR_MAX - (uintptr_t)mem); + + slob = mem; + slob->size = size - sizeof(SLOB); + slob->head = (struct block *)((uint8_t *)mem + sizeof(SLOB)); + slob->head->size = slob->size - sizeof(struct alloc); + slob->head->next = NULL; + + return slob; +} + +void *sloballoc(SLOB *slob, size_t size) +{ + struct block *b, **p; + size_t fitblock, remblock; + + // size must be enough to extend to a struct block in case of free + fitblock = sizeof(struct block) - sizeof(struct alloc); + if(size < fitblock) size = fitblock; + + // need this much to fit another block in the remaining space + remblock = size + sizeof(struct block); + if(remblock < size) return NULL; // overflow + + // scan list for the first block of sufficient size + for(p=&slob->head; (b=*p); p=&b->next) { + if(b->size >= remblock) { + // cut from the end of the block + b->size -= sizeof(struct alloc) + size; + struct alloc *a = (struct alloc *)(((struct alloc *)b)->data + b->size); + a->size = size; + return a->data; + } else if(b->size >= size) { + // when a block fills, it converts directly to a struct alloc + *p = b->next; // unlink + return ((struct alloc *)b)->data; + } + } + + return NULL; +} + +void slobfree(SLOB *slob, void *a_) +{ + struct alloc *a = (struct alloc *)((uint8_t *)a_ - sizeof(struct alloc)); + struct block *b, **p, *left=NULL, *right=NULL, **rightp=NULL; + + // sanity check: a lies inside slob + assert((uint8_t *)a >= slob->data); + assert(a->data + a->size <= slob->data + slob->size); + + // scan list for blocks adjacent to a + for(p=&slob->head; (b=*p); p=&b->next) { + if((uint8_t *)a == ((struct alloc *)b)->data + b->size) { + assert(!left); + left = b; + } + if(a->data + a->size == (uint8_t *)b) { + assert(!right); + right = b; + rightp = p; + } + + if(left && right) { + // extend left and unlink right + left->size += sizeof(*a) + a->size + + sizeof(struct alloc) + right->size; + *rightp = right->next; + return; + } + } + + if(left) { + // extend left to absorb a + left->size += sizeof(*a) + a->size; + } else if(right) { + // shift and extend right to absorb a + right->size += sizeof(*a) + a->size; + *rightp = (struct block *)a; **rightp = *right; + } else { + // spawn new block over a + struct block *b = (struct block *)a; + b->next = slob->head; slob->head = b; + } +} + +int slobcheck(SLOB *slob) +{ + // invariants: + // 1. memory area is divided seamlessly and exactly into n blocks + // 2. every block is large enough to hold a 'struct block'. + // 3. free list has at most n elements. + // 4. every element of the free list is one of the valid blocks. + // 5. every block appears at most once in the free list. + + uint8_t *p; + size_t nblocks=0, nfree=0; + + #define FORBLOCKS \ + for(p = slob->data; \ + p != slob->data + slob->size; \ + p += sizeof(struct alloc) + ((struct alloc *)p)->size) + + // 1. memory area is divided seamlessly and exactly into n blocks + FORBLOCKS { + if(p < slob->data) + return 1; + if(p > slob->data + slob->size) + return 2; + nblocks++; + + struct alloc *a = (struct alloc *)p; + if(a->size > UINTPTR_MAX - (uintptr_t)p) + return 3; + + // 2. every block is large enough to hold a 'struct block'. + if(a->size + sizeof(struct alloc) < sizeof(struct block)) + return 4; + } + + // 3. free list has at most n elements. + for(struct block *b=slob->head; b; b=b->next) { + nfree++; + if(nfree > nblocks) + return 5; + + // 4. every element of the free list is one of the valid blocks. + FORBLOCKS + if(p == (uint8_t *)b) break; + if(!p) + return 6; + } + + // 5. every block appears at most once in the free list. + FORBLOCKS { + size_t count=0; + for(struct block *b=slob->head; b; b=b->next) + if(p == (uint8_t *)b) count++; + if(count > 1) + return 7; + } + + #undef FORBLOCKS + return 0; +} + + +// hammer interface + +#include "hammer.h" + +static void *h_slob_alloc(HAllocator *mm, size_t size) +{ + SLOB *slob = (SLOB *)(mm+1); + return sloballoc(slob, size); +} + +static void h_slob_free(HAllocator *mm, void *p) +{ + SLOB *slob = (SLOB *)(mm+1); + slobfree(slob, p); +} + +static void *h_slob_realloc(HAllocator *mm, void *p, size_t size) +{ + SLOB *slob = (SLOB *)(mm+1); + + assert(((void)"XXX need realloc for SLOB allocator", 0)); + return NULL; +} + +HAllocator *h_sloballoc(void *mem, size_t size) +{ + if(size < sizeof(HAllocator)) + return NULL; + + HAllocator *mm = mem; + SLOB *slob = slobinit((uint8_t *)mem + sizeof(HAllocator), size - sizeof(HAllocator)); + if(!slob) + return NULL; + assert(slob == (SLOB *)(mm+1)); + + mm->alloc = h_slob_alloc; + mm->realloc = h_slob_realloc; + mm->free = h_slob_free; + + return mm; +} diff --git a/src/sloballoc.h b/src/sloballoc.h new file mode 100644 index 0000000..ecdc479 --- /dev/null +++ b/src/sloballoc.h @@ -0,0 +1,15 @@ +#ifndef SLOBALLOC_H_SEEN +#define SLOBALLOC_H_SEEN + +#include + +typedef struct slob SLOB; + +SLOB *slobinit(void *mem, size_t size); +void *sloballoc(SLOB *slob, size_t size); +void slobfree(SLOB *slob, void *p); + +// consistency check (verify internal invariants); returns 0 on success +int slobcheck(SLOB *slob); + +#endif // SLOBALLOC_H_SEEN diff --git a/src/t_mm.c b/src/t_mm.c new file mode 100644 index 0000000..620d4e8 --- /dev/null +++ b/src/t_mm.c @@ -0,0 +1,148 @@ +#include +#include +#include "test_suite.h" +#include "sloballoc.h" +#include "hammer.h" + +#define check_sloballoc_invariants() do { \ + int err = slobcheck(slob); \ + if(err) { \ + g_test_message("SLOB invariant check failed on line %d, returned %d", \ + __LINE__, err); \ + g_test_fail(); \ + } \ + } while(0) + +#define check_sloballoc(VAR, SIZE, OFFSET) do { \ + check_sloballoc_invariants(); \ + VAR = sloballoc(slob, (SIZE)); \ + g_check_cmp_ptr(VAR, ==, mem + (OFFSET)); \ + } while(0) + +#define check_sloballoc_fail(SIZE) do { \ + check_sloballoc_invariants(); \ + void *p = sloballoc(slob, (SIZE)); \ + g_check_cmp_ptr(p, ==, NULL); \ + } while(0) + +#define check_slobfree(P) do { \ + check_sloballoc_invariants(); \ + slobfree(slob, P); \ + } while(0) + +#define N 1024 + +#define SLOBALLOC_FIXTURE \ + static uint8_t mem[N] = {0x58}; \ + SLOB *slob = slobinit(mem, N); \ + size_t max = N - 2*sizeof(size_t) - sizeof(void *); \ + (void)max; /* silence warning */ \ + if(!slob) { \ + g_test_message("SLOB allocator init failed on line %d", __LINE__); \ + g_test_fail(); \ + } + +static void test_sloballoc_size(void) +{ + SLOBALLOC_FIXTURE + void *p; + + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_fail(N); + check_sloballoc_fail(max+1); + + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_invariants(); +} + +static void test_sloballoc_merge(void) +{ + SLOBALLOC_FIXTURE + void *p, *q, *r; + + check_sloballoc(p, 100, N-100); + check_slobfree(p); + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc(p, 100, N-100); + check_sloballoc(q, 100, N-200-sizeof(size_t)); + check_slobfree(p); + check_sloballoc(p, 50, N-50); + check_sloballoc(r, 100, N-300-2*sizeof(size_t)); + check_slobfree(q); + check_sloballoc(q, 150, N-200-sizeof(size_t)); + check_slobfree(p); + check_slobfree(r); + check_slobfree(q); // merge left and right + + check_sloballoc_fail(max+1); + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_invariants(); +} + +static void test_sloballoc_small(void) +{ + SLOBALLOC_FIXTURE + void *p, *q, *r; + + check_sloballoc(p, 100, N-100); + check_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_sloballoc(r, 100, N-200-2*sizeof(size_t)-sizeof(void *)); + check_slobfree(q); + check_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_slobfree(p); + check_slobfree(r); + + check_sloballoc_invariants(); +} + +#define check_h_sloballoc(VAR, SIZE, OFFSET) do { \ + check_sloballoc_invariants(); \ + VAR = mm->alloc(mm, (SIZE)); \ + g_check_cmp_ptr(VAR, ==, mem + (OFFSET)); \ + } while(0) + +#define check_h_slobfree(P) do { \ + check_sloballoc_invariants(); \ + mm->free(mm, P); \ + } while(0) + +static void test_sloballoc_hammer(void) +{ + static uint8_t mem[N] = {0x58}; + HAllocator *mm = h_sloballoc(mem, N); int line = __LINE__; + SLOB *slob = ((void *)mm) + sizeof(HAllocator); + void *p, *q, *r; + + if(!mm) { + g_test_message("h_sloballoc() failed on line %d", line); + g_test_fail(); + } + + check_h_sloballoc(p, 100, N-100); + check_h_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_h_sloballoc(r, 100, N-200-2*sizeof(size_t)-sizeof(void *)); + check_h_slobfree(q); + check_h_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_h_slobfree(p); + check_h_slobfree(r); + + check_sloballoc_invariants(); +} + +#undef N + +void register_mm_tests(void) { + g_test_add_func("/core/mm/sloballoc/size", test_sloballoc_size); + g_test_add_func("/core/mm/sloballoc/merge", test_sloballoc_merge); + g_test_add_func("/core/mm/sloballoc/small", test_sloballoc_small); + g_test_add_func("/core/mm/sloballoc/hammer", test_sloballoc_hammer); +} + diff --git a/src/test_suite.c b/src/test_suite.c index cba18e8..f569644 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -24,6 +24,7 @@ extern void register_bitwriter_tests(); extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); +extern void register_mm_tests(); extern void register_benchmark_tests(); extern void register_regression_tests(); @@ -36,6 +37,7 @@ int main(int argc, char** argv) { register_parser_tests(); register_grammar_tests(); register_misc_tests(); + register_mm_tests(); register_regression_tests(); if (g_test_slow() || g_test_perf()) register_benchmark_tests(); diff --git a/src/test_suite.h b/src/test_suite.h index 83359f9..ed640fd 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -321,6 +321,7 @@ #define g_check_cmp_int64(n1, op, n2) g_check_inttype("%" PRId64, int64_t, n1, op, n2) #define g_check_cmp_uint32(n1, op, n2) g_check_inttype("%u", uint32_t, n1, op, n2) #define g_check_cmp_uint64(n1, op, n2) g_check_inttype("%" PRIu64, uint64_t, n1, op, n2) +#define g_check_cmp_ptr(n1, op, n2) g_check_inttype("%p", void *, n1, op, n2) #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) diff --git a/tools/csharp/__init__.py b/tools/csharp/__init__.py index af4f519..fbc9d30 100644 --- a/tools/csharp/__init__.py +++ b/tools/csharp/__init__.py @@ -21,4 +21,6 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from csharp import exists, generate +from __future__ import absolute_import, division, print_function + +from .csharp import exists, generate diff --git a/tools/csharp/csharp.py b/tools/csharp/csharp.py index 6b38b45..101d3f6 100644 --- a/tools/csharp/csharp.py +++ b/tools/csharp/csharp.py @@ -30,6 +30,8 @@ # This is an attempt to meld to two based initially on the Microsoft C# tool with amendmnets from the Mono # tool. +from __future__ import absolute_import, division, print_function + import os.path import SCons.Builder import SCons.Node.FS @@ -203,7 +205,7 @@ def AddToModPaths(env, files, **kw): def cscFlags(target, source, env, for_signature): listCmd = [] - if (env.has_key('WINEXE')): + if ('WINEXE' in env): if (env['WINEXE'] == 1): listCmd.append('-t:winexe') return listCmd @@ -243,7 +245,7 @@ def cscSourcesNoResources(target, source, env, for_signature): def cscRefs(target, source, env, for_signature): listCmd = [] - if (env.has_key('ASSEMBLYREFS')): + if ('ASSEMBLYREFS' in env): refs = SCons.Util.flatten(env['ASSEMBLYREFS']) for ref in refs: if SCons.Util.is_String(ref): @@ -256,7 +258,7 @@ def cscRefs(target, source, env, for_signature): def cscMods(target, source, env, for_signature): listCmd = [] - if (env.has_key('NETMODULES')): + if ('NETMODULES' in env): mods = SCons.Util.flatten(env['NETMODULES']) for mod in mods: listCmd.append('-addmodule:%s' % mod) @@ -274,7 +276,7 @@ def alLinkSources(target, source, env, for_signature): # just treat this as a generic unidentified source file listCmd.append('-link:%s' % s.get_string(for_signature)) - if env.has_key('VERSION'): + if 'VERSION' in env: version = parseVersion(env) listCmd.append('-version:%d.%d.%d.%d' % version) @@ -296,7 +298,7 @@ def cliLinkSources(target, source, env, for_signature): return listCmd def add_version(target, source, env): - if env.has_key('VERSION'): + if 'VERSION' in env: if SCons.Util.is_String(target[0]): versionfile = target[0] + '_VersionInfo.cs' else: @@ -319,14 +321,14 @@ def lib_emitter(target, source, env): def add_depends(target, source, env): """Add dependency information before the build order is established""" - if (env.has_key('NETMODULES')): + if ('NETMODULES' in env): mods = SCons.Util.flatten(env['NETMODULES']) for mod in mods: # add as dependency for t in target: env.Depends(t, mod) - if (env.has_key('ASSEMBLYREFS')): + if ('ASSEMBLYREFS' in env): refs = SCons.Util.flatten(env['ASSEMBLYREFS']) for ref in refs: # add as dependency @@ -417,7 +419,7 @@ res_action = SCons.Action.Action('$CLIRCCOM', '$CLIRCCOMSTR') def res_emitter(target, source, env): # prepend NAMESPACE if provided - if (env.has_key('NAMESPACE')): + if ('NAMESPACE' in env): newtargets = [] for t in target: tname = t.name diff --git a/tools/csharp/mono.py b/tools/csharp/mono.py index a2cc380..99c509c 100644 --- a/tools/csharp/mono.py +++ b/tools/csharp/mono.py @@ -25,6 +25,8 @@ # This C# Tool for Mono taken from http://www.scons.org/wiki/CsharpBuilder. +from __future__ import absolute_import, division, print_function + import os.path import SCons.Builder import SCons.Node.FS diff --git a/tools/scanreplace.py b/tools/scanreplace.py index 5321e48..81a167e 100644 --- a/tools/scanreplace.py +++ b/tools/scanreplace.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function + from string import Template def replace_action(target, source, env):