From 4811f583741ca8b24bb08c0fc173807a34a320f8 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 19 Nov 2013 21:14:39 -0600 Subject: [PATCH 01/44] Added token type registry. Closes #45 --- src/SConscript | 1 + src/hammer.h | 11 +++++++ src/registry.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/t_misc.c | 19 +++++++++++ 4 files changed, 118 insertions(+) create mode 100644 src/registry.c diff --git a/src/SConscript b/src/SConscript index 9b5c868..03308dd 100644 --- a/src/SConscript +++ b/src/SConscript @@ -49,6 +49,7 @@ misc_hammer_parts = [ 'glue.c', 'hammer.c', 'pprint.c', + 'registry.c', 'system_allocator.c'] tests = ['t_benchmark.c', diff --git a/src/hammer.h b/src/hammer.h index 508653d..0175142 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -633,4 +633,15 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results); void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} +// {{{ Token type registry +/// Allocate a new, unused (as far as this function knows) token type. +int h_allocate_token_type(const char* name); + +/// Get the token type associated with name. Returns -1 if name is unkown +int h_get_token_type_number(const char* name); + +/// Get the name associated with token_type. Returns NULL if the token type is unkown +const char* h_get_token_type_name(int token_type); +// }}} + #endif // #ifndef HAMMER_HAMMER__H diff --git a/src/registry.c b/src/registry.c new file mode 100644 index 0000000..c59b6ea --- /dev/null +++ b/src/registry.c @@ -0,0 +1,87 @@ +/* Parser combinators for binary formats. + * Copyright (C) 2012 Meredith L. Patterson, Dan "TQ" Hirsch + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include "hammer.h" +#include "internal.h" + +typedef struct Entry_ { + const char* name; + int value; +} Entry; + +static void *tt_registry = NULL; +static Entry** tt_by_id = NULL; +static int tt_by_id_sz = 0; +#define TT_START TT_USER +static int tt_next = TT_START; + +/* + // TODO: These are for the extension registry, which does not yet have a good name. +static void *ext_registry = NULL; +static Entry** ext_by_id = NULL; +static int ext_by_id_sz = 0; +static int ext_next = 0; +*/ + + +static int compare_entries(const void* v1, const void* v2) { + const Entry *e1 = (Entry*)v1, *e2 = (Entry*)v2; + return strcmp(e1->name, e2->name); +} + +int h_allocate_token_type(const char* name) { + Entry* new_entry = malloc(sizeof(*new_entry)); + new_entry->name = name; + new_entry->value = -1; + Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries); + if (probe->value != -1) { + // Token type already exists... + // TODO: treat this as a bug? + free(new_entry); + return probe->value; + } else { + // new value + probe->name = strdup(probe->name); // drop ownership of name + probe->value = tt_next++; + if ((probe->value - TT_START) >= tt_by_id_sz) { + if (tt_by_id_sz == 0) + tt_by_id = malloc(sizeof(*tt_by_id) * ((tt_by_id_sz = (tt_next - TT_START) * 16))); + else + tt_by_id = realloc(tt_by_id, sizeof(*tt_by_id) * ((tt_by_id_sz *= 2))); + } + assert(probe->value - TT_START < tt_by_id_sz); + tt_by_id[probe->value - TT_START] = probe; + return probe->value; + } +} +int h_get_token_type_number(const char* name) { + Entry e; + e.name = name; + Entry **ret = (Entry**)tfind(&e, &tt_registry, compare_entries); + if (ret == NULL) + return -1; + else + return (*ret)->value; +} +const char* h_get_token_type_name(int token_type) { + if (token_type >= tt_next || token_type < TT_START) + return NULL; + else + return tt_by_id[token_type - TT_START]->name; +} diff --git a/src/t_misc.c b/src/t_misc.c index c762c07..74a57ca 100644 --- a/src/t_misc.c +++ b/src/t_misc.c @@ -1,4 +1,5 @@ #include +#include #include "test_suite.h" #include "hammer.h" @@ -11,6 +12,24 @@ static void test_tt_user(void) { g_check_cmp_int32(TT_USER, >, TT_ERR); } +static void test_tt_registry(void) { + int id = h_allocate_token_type("com.upstandinghackers.test.token_type"); + g_check_cmp_int32(id, >=, TT_USER); + int id2 = h_allocate_token_type("com.upstandinghackers.test.token_type_2"); + g_check_cmp_int32(id2, !=, id); + g_check_cmp_int32(id2, >=, TT_USER); + g_check_cmp_int32(id, ==, h_get_token_type_number("com.upstandinghackers.test.token_type")); + g_check_cmp_int32(id2, ==, h_get_token_type_number("com.upstandinghackers.test.token_type_2")); + g_check_string("com.upstandinghackers.test.token_type", ==, h_get_token_type_name(id)); + g_check_string("com.upstandinghackers.test.token_type_2", ==, h_get_token_type_name(id2)); + if (h_get_token_type_name(0) != NULL) { + g_test_message("Unknown token type should not return a name"); + g_test_fail(); + } + g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, -1); +} + void register_misc_tests(void) { g_test_add_func("/core/misc/tt_user", test_tt_user); + g_test_add_func("/core/misc/tt_registry", test_tt_registry); } From b4e28ac021d50eaa20a8e2ccf01d557e85bb52ab Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:05:48 -0600 Subject: [PATCH 02/44] Fixed a potential segfault; hand-initialized HParsers in h_choice and h_sequence need PB_MIN set. Conflicts: src/bindings/python/SConscript src/bindings/python/hammer_tests.py --- src/parsers/choice.c | 4 +++- src/parsers/sequence.c | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/parsers/choice.c b/src/parsers/choice.c index 6db1378..bb55fa8 100644 --- a/src/parsers/choice.c +++ b/src/parsers/choice.c @@ -143,6 +143,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) { s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &choice_vt; ret->env = (void*)s; + ret->vtable = &choice_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index eff4610..bdea8c0 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -145,20 +145,22 @@ HParser* h_sequence__a(void *args[]) { HParser* h_sequence__ma(HAllocator* mm__, void *args[]) { size_t len = -1; // because do...while const HParser *arg; - + do { arg=((HParser **)args)[++len]; } while(arg); - + HSequence *s = h_new(HSequence, 1); s->p_array = h_new(HParser *, len); for (size_t i = 0; i < len; i++) { s->p_array[i] = ((HParser **)args)[i]; } - + s->len = len; HParser *ret = h_new(HParser, 1); - ret->vtable = &sequence_vt; ret->env = (void*)s; + ret->vtable = &sequence_vt; + ret->env = (void*)s; + ret->backend = PB_MIN; return ret; } From 91221ac2563b301ccc3bd3dcec8bdf4dd1356e8d Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 20:46:34 -0600 Subject: [PATCH 03/44] tweak SConstruct so that 'scan-build scons' works --- SConstruct | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SConstruct b/SConstruct index 623bc5d..78090c3 100644 --- a/SConstruct +++ b/SConstruct @@ -68,10 +68,15 @@ if GetOption("coverage"): LDFLAGS=["-fprofile-arcs", "-ftest-coverage"], LIBS=['gcov']) +env["CC"] = os.getenv("CC") or env["CC"] +env["CXX"] = os.getenv("CXX") or env["CXX"] + if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") +env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) + #rootpath = env['ROOTPATH'] = os.path.abspath('.') #env.Append(CPPPATH=os.path.join('#', "hammer")) From f220524b5a383a7945a1d834a3600bd07f7b2edc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:04:07 -0800 Subject: [PATCH 04/44] Generate pkg-config for Hammer Create a pkg-config file and install it to $prefix/lib/pkgconfig --- SConstruct | 6 ++++-- libhammer.pc.in | 10 ++++++++++ src/SConscript | 1 + tools/scanreplace.py | 15 +++++++++++++++ tools/scanreplace.pyc | Bin 0 -> 1263 bytes 5 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 libhammer.pc.in create mode 100644 tools/scanreplace.py create mode 100644 tools/scanreplace.pyc diff --git a/SConstruct b/SConstruct index 78090c3..422efe6 100644 --- a/SConstruct +++ b/SConstruct @@ -8,7 +8,7 @@ vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept)) -env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars) +env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars, tools=['default', 'scanreplace'], toolpath=['tools']) def calcInstallPath(*elements): path = os.path.abspath(os.path.join(*map(env.subst, elements))) @@ -28,7 +28,8 @@ if 'DESTDIR' in env: env['libpath'] = calcInstallPath("$prefix", "lib") env['incpath'] = calcInstallPath("$prefix", "include", "hammer") -# TODO: Add pkgconfig +env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig") +env.ScanReplace('libhammer.pc.in') env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes") @@ -89,3 +90,4 @@ env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/ env.Alias("install", "$libpath") env.Alias("install", "$incpath") +env.Alias("install", "$pkgconfigpath") diff --git a/libhammer.pc.in b/libhammer.pc.in new file mode 100644 index 0000000..97ded13 --- /dev/null +++ b/libhammer.pc.in @@ -0,0 +1,10 @@ +prefix=/usr +exec_prefix=${prefix} +includedir=${prefix}/include +libdir=${exec_prefix}/lib + +Name: libhammer +Description: The Hammer parsing library +Version: 0.9.0 +Cflags: -I${includedir}/hammer +Libs: -L${libdir} -lhammer diff --git a/src/SConscript b/src/SConscript index 03308dd..a06244b 100644 --- a/src/SConscript +++ b/src/SConscript @@ -64,6 +64,7 @@ libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_ env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) +env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() testenv.ParseConfig('pkg-config --cflags --libs glib-2.0') diff --git a/tools/scanreplace.py b/tools/scanreplace.py new file mode 100644 index 0000000..5321e48 --- /dev/null +++ b/tools/scanreplace.py @@ -0,0 +1,15 @@ +from string import Template + +def replace_action(target, source, env): + open(str(target[0]), 'w').write(Template(open(str(source[0]), 'r').read()).safe_substitute(env)) + return 0 + +def replace_string(target, source, env): + return "building '%s' from '%s'" % (str(target[0]), str(source[0])) + +def generate(env, **kw): + action = env.Action(replace_action, replace_string) + env['BUILDERS']['ScanReplace'] = env.Builder(action=action, src_suffix='.in', single_source=True) + +def exists(env): + return 1 diff --git a/tools/scanreplace.pyc b/tools/scanreplace.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c37386cee5c446284c300e734bcd48d69842fc26 GIT binary patch literal 1263 zcmZSn%*(Y>tT!l`0ScHH7#JKF7#NBz7#J8*7#LC*8FCmHav2$-7#SE-m>60Z7^0XM zQkcOEW`-0NFoT65g%!+TWk_LT2-aX{U|`7n|NsAg4MqkAh7t}228NK-+=86MlGJ2o z1_lNYW@BJra0c1r15(1kP{PPi1GZ1HnUNudiJ_T^A&ZHjgqfiPBF6}F3s{aBEXTso z%)rph$dJX#P{RabG=p5o$Plc-!oa}bmkIJv3CQ|#Fj=GlvakdsmtT;YR|0ZEaY<1L zD+2>Vc~NFbY7kfhNWLgFF{Okb8T}PW^sOL zQF1ESxYWEd4Ui#VqS%Omfk8jCIJKx)KPx{~zqlw_KO-?WH?>GNt0=!JFSSU&BtJi= zSid+qF|R1KASW?7Rj;73gpYxN0m_X}OfJdH&jZ<7400C(qX#2A_(1Lk2cHrsq8LG; z1PVST22c={Ffi1B0};eYV`2~s2D{y_Se$`@A*nPoCnYm4T|r&7SY08lC_fj>0tXm4 zzJfrOfi(nyVlIf2fq@~2i-Caw>@hGw4c`@)6lLb6gOW-yDAh4ACWB%N)S|?aRItB5@y5U?1PMq`V1h6>k$_}UKzRe?0KZI-IM`}Pe1kMYxS-@qu5&;c zBef#4xTF}AhM{rl2g!CIIY@kivl$}5g2X_H3zRGo4gj;rbpc<^ik# literal 0 HcmV?d00001 From dbe0785e2cbe41de8c77a8c584d40de2b7606b10 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:25:09 -0800 Subject: [PATCH 05/44] Remove pyc file --- tools/scanreplace.pyc | Bin 1263 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tools/scanreplace.pyc diff --git a/tools/scanreplace.pyc b/tools/scanreplace.pyc deleted file mode 100644 index c37386cee5c446284c300e734bcd48d69842fc26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1263 zcmZSn%*(Y>tT!l`0ScHH7#JKF7#NBz7#J8*7#LC*8FCmHav2$-7#SE-m>60Z7^0XM zQkcOEW`-0NFoT65g%!+TWk_LT2-aX{U|`7n|NsAg4MqkAh7t}228NK-+=86MlGJ2o z1_lNYW@BJra0c1r15(1kP{PPi1GZ1HnUNudiJ_T^A&ZHjgqfiPBF6}F3s{aBEXTso z%)rph$dJX#P{RabG=p5o$Plc-!oa}bmkIJv3CQ|#Fj=GlvakdsmtT;YR|0ZEaY<1L zD+2>Vc~NFbY7kfhNWLgFF{Okb8T}PW^sOL zQF1ESxYWEd4Ui#VqS%Omfk8jCIJKx)KPx{~zqlw_KO-?WH?>GNt0=!JFSSU&BtJi= zSid+qF|R1KASW?7Rj;73gpYxN0m_X}OfJdH&jZ<7400C(qX#2A_(1Lk2cHrsq8LG; z1PVST22c={Ffi1B0};eYV`2~s2D{y_Se$`@A*nPoCnYm4T|r&7SY08lC_fj>0tXm4 zzJfrOfi(nyVlIf2fq@~2i-Caw>@hGw4c`@)6lLb6gOW-yDAh4ACWB%N)S|?aRItB5@y5U?1PMq`V1h6>k$_}UKzRe?0KZI-IM`}Pe1kMYxS-@qu5&;c zBef#4xTF}AhM{rl2g!CIIY@kivl$}5g2X_H3zRGo4gj;rbpc<^ik# From f1a5844523b2424a784995dffcad54c4832764bc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Wed, 20 Nov 2013 13:26:00 -0800 Subject: [PATCH 06/44] Ignore pyc files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e05a551..16fdda1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ Session.vim cscope.out build/ .sconsign.dblite +*.pyc From a98272b8c0979c191c0bda17cd1fc116f580f4df Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Wed, 20 Nov 2013 15:37:24 -0600 Subject: [PATCH 07/44] Reduce IRC spam --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1458ecd..246a3d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,9 @@ compiler: script: - scons notifications: - irc: "irc.upstandinghackers.com#hammer" + irc: + channels: + - "irc.upstandinghackers.com#hammer" + use_notice: true + skip_join: true + From 76034ae4d7db051d0dd351e9880b8322c6db5883 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:02:18 -0800 Subject: [PATCH 08/44] Install internal headers for Go bindings Go, and likely other language bindings will, need access to create new combinators. This functionality is currently located src/parsers/parser_internal.h. Install this header to the system with dependent headers until a better solution can be found. --- SConstruct | 4 ++++ src/SConscript | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 422efe6..ebed8b4 100644 --- a/SConstruct +++ b/SConstruct @@ -28,6 +28,8 @@ if 'DESTDIR' in env: env['libpath'] = calcInstallPath("$prefix", "lib") env['incpath'] = calcInstallPath("$prefix", "include", "hammer") +env['parsersincpath'] = calcInstallPath("$prefix", "include", "hammer", "parsers") +env['backendsincpath'] = calcInstallPath("$prefix", "include", "hammer", "backends") env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig") env.ScanReplace('libhammer.pc.in') @@ -90,4 +92,6 @@ env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/ env.Alias("install", "$libpath") env.Alias("install", "$incpath") +env.Alias("install", "$parsersincpath") +env.Alias("install", "$backendsincpath") env.Alias("install", "$pkgconfigpath") diff --git a/src/SConscript b/src/SConscript index a06244b..1d9ca76 100644 --- a/src/SConscript +++ b/src/SConscript @@ -6,7 +6,17 @@ bindings = [] dist_headers = [ "hammer.h", "allocator.h", - "glue.h" + "glue.h", + "internal.h" +] + +parsers_headers = [ + "parsers/parser_internal.h" +] + +backends_headers = [ + "backends/regex.h", + "backends/contextfree.h" ] parsers = ['parsers/%s.c'%s for s in @@ -64,6 +74,8 @@ libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_ env.Install("$libpath", [libhammer_static, libhammer_shared]) env.Install("$incpath", dist_headers) +env.Install("$parsersincpath", parsers_headers) +env.Install("$backendsincpath", backends_headers) env.Install("$pkgconfigpath", "../../../libhammer.pc") testenv = env.Clone() From 1d80058c851fb31dcc46ddcc7ceac12297fa9ca6 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:37:36 -0800 Subject: [PATCH 09/44] Properly name the lib Set the proper path for the shared library so that anything linking against it will be able to find the library. --- SConstruct | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SConstruct b/SConstruct index ebed8b4..0512d71 100644 --- a/SConstruct +++ b/SConstruct @@ -78,6 +78,8 @@ if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") +env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) + env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') From d0906e69e6a3cd17e68b43eb83b8e0b522f02fe8 Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 12:50:03 -0800 Subject: [PATCH 10/44] Fix for linux This actually is only for OS X. On linux ldconfig must be run after installing in order to update the library cache. Need to look into how to get scons to do this for the user. --- SConstruct | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index 0512d71..c46c090 100644 --- a/SConstruct +++ b/SConstruct @@ -37,6 +37,7 @@ env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attr if not env['PLATFORM'] == 'darwin': env.MergeFlags("-lrt") + env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) AddOption("--variant", dest="variant", @@ -78,8 +79,6 @@ if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin': env.Replace(CC="clang", CXX="clang++") -env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) - env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) #rootpath = env['ROOTPATH'] = os.path.abspath('.') From bb02aa18e76ed4236526572b3093ce38b64f2fbc Mon Sep 17 00:00:00 2001 From: Joe Rozner Date: Thu, 21 Nov 2013 22:09:25 -0800 Subject: [PATCH 11/44] Add warning message Specify warning for internal only headers that are now installed for use by extensions. --- src/backends/contextfree.h | 5 +++++ src/backends/regex.h | 5 +++++ src/internal.h | 5 +++++ src/parsers/parser_internal.h | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/src/backends/contextfree.h b/src/backends/contextfree.h index b387e55..ab04ab5 100644 --- a/src/backends/contextfree.h +++ b/src/backends/contextfree.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + // This is an internal header; it provides macros to make desugaring cleaner. #include #include "../internal.h" diff --git a/src/backends/regex.h b/src/backends/regex.h index a84904d..4ea85a8 100644 --- a/src/backends/regex.h +++ b/src/backends/regex.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + // Internal defs #ifndef HAMMER_BACKEND_REGEX__H #define HAMMER_BACKEND_REGEX__H diff --git a/src/internal.h b/src/internal.h index c402da5..89cb380 100644 --- a/src/internal.h +++ b/src/internal.h @@ -15,6 +15,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + #ifndef HAMMER_INTERNAL__H #define HAMMER_INTERNAL__H #include diff --git a/src/parsers/parser_internal.h b/src/parsers/parser_internal.h index aeb202b..ec97dd1 100644 --- a/src/parsers/parser_internal.h +++ b/src/parsers/parser_internal.h @@ -1,3 +1,8 @@ +/* + * NOTE: This is an internal header and installed for use by extensions. The + * API is not guaranteed stable. +*/ + #ifndef HAMMER_PARSE_INTERNAL__H #define HAMMER_PARSE_INTERNAL__H #include "../hammer.h" From 2cead893fc6dd31416cfd18f8755b71acaf477c4 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 22 Nov 2013 19:44:30 -0600 Subject: [PATCH 12/44] Allow in-place build via scons --in-place --- .gitignore | 1 + SConstruct | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 16fdda1..721dcf9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ Session.vim cscope.out build/ .sconsign.dblite +*.os *.pyc diff --git a/SConstruct b/SConstruct index c46c090..551f89e 100644 --- a/SConstruct +++ b/SConstruct @@ -53,7 +53,12 @@ AddOption("--coverage", action="store_true", help="Build with coverage instrumentation") -env['BUILDDIR'] = 'build/$VARIANT' +AddOption("--in-place", + dest="in_place", + default=False, + action="store_true", + help="Build in-place, rather than in the build/ tree") + dbg = env.Clone(VARIANT='debug') dbg.Append(CCFLAGS=['-g']) @@ -86,10 +91,16 @@ env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_")) Export('env') -env.SConscript(["src/SConscript"], variant_dir='build/$VARIANT/src') -env.SConscript(["examples/SConscript"], variant_dir='build/$VARIANT/examples') +if not GetOption("in_place"): + env['BUILD_BASE'] = 'build/$VARIANT' + env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src') + env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples') +else: + env['BUILD_BASE'] = '.' + env.SConscript(["src/SConscript"]) + env.SConscript(["examples/SConscript"]) -env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/$VARIANT/src $SOURCE') +env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE') env.Alias("install", "$libpath") env.Alias("install", "$incpath") From c6b4beefcb31c10926ea94a7c8ee5fdb9f2fa8ab Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 12:28:34 -0600 Subject: [PATCH 13/44] Make install_name darwin-only --- SConstruct | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index c46c090..b53947f 100644 --- a/SConstruct +++ b/SConstruct @@ -35,9 +35,10 @@ env.ScanReplace('libhammer.pc.in') env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes") -if not env['PLATFORM'] == 'darwin': +if env['PLATFORM'] == 'darwin': + env.Append(SHLINKFLAGS = ['-install_name', '$TARGET']) +else: env.MergeFlags("-lrt") - env.Append(SHLINKFLAGS = ['-install_name ' + '$TARGET']) AddOption("--variant", dest="variant", From e487d5435c89f93a1de0c65a604cfe7430002ba6 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 12:53:11 -0600 Subject: [PATCH 14/44] Added a void* user_data pointer to HAction --- examples/base64_sem1.c | 6 +++--- examples/base64_sem2.c | 2 +- examples/dns.c | 10 +++++----- examples/dns_common.c | 2 +- examples/dns_common.h | 2 +- examples/rr.c | 18 +++++++++--------- src/backends/llk.c | 4 ++-- src/backends/lr.c | 4 ++-- src/glue.c | 12 ++++++------ src/glue.h | 30 ++++++++++++++++++------------ src/hammer.h | 14 +++++++------- src/internal.h | 1 + src/parsers/action.c | 19 +++++++++++-------- src/parsers/bits.c | 16 ++++++---------- src/parsers/optional.c | 3 ++- src/parsers/sequence.c | 3 ++- src/parsers/token.c | 3 ++- src/t_parser.c | 7 ++++--- 18 files changed, 83 insertions(+), 73 deletions(-) diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c index 46e58da..4da171f 100644 --- a/examples/base64_sem1.c +++ b/examples/base64_sem1.c @@ -23,7 +23,7 @@ // They must be named act_. /// -HParsedToken *act_bsfdig(const HParseResult *p) +HParsedToken *act_bsfdig(const HParseResult *p, void* user_data) { HParsedToken *res = H_MAKE_UINT(0); @@ -54,7 +54,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0); #define act_document act_index0 // General-form action to turn a block of base64 digits into bytes. -HParsedToken *act_base64_n(int n, const HParseResult *p) +HParsedToken *act_base64_n(int n, const HParseResult *p, void* user_data) { HParsedToken *res = H_MAKE_SEQN(n); @@ -83,7 +83,7 @@ H_ACT_APPLY(act_base64_3, act_base64_n, 3); H_ACT_APPLY(act_base64_2, act_base64_n, 2); H_ACT_APPLY(act_base64_1, act_base64_n, 1); -HParsedToken *act_base64(const HParseResult *p) +HParsedToken *act_base64(const HParseResult *p, void* user_data) { assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used == 2); diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c index 84155da..b7a2263 100644 --- a/examples/base64_sem2.c +++ b/examples/base64_sem2.c @@ -49,7 +49,7 @@ uint8_t bsfdig_value(const HParsedToken *p) // helper: append a byte value to a sequence #define seq_append_byte(res, b) h_seq_snoc(res, H_MAKE_UINT(b)) -HParsedToken *act_base64(const HParseResult *p) +HParsedToken *act_base64(const HParseResult *p, void* user_data) { assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used == 2); diff --git a/examples/dns.c b/examples/dns.c index 3f730b9..5effb89 100644 --- a/examples/dns.c +++ b/examples/dns.c @@ -86,7 +86,7 @@ void set_rdata(struct dns_rr *rr, HCountedArray *rdata) { } } -HParsedToken* act_header(const HParseResult *p) { +HParsedToken* act_header(const HParseResult *p, void* user_data) { HParsedToken **fields = h_seq_elements(p->ast); dns_header_t header_ = { .id = H_CAST_UINT(fields[0]), @@ -109,7 +109,7 @@ HParsedToken* act_header(const HParseResult *p) { return H_MAKE(dns_header_t, header); } -HParsedToken* act_label(const HParseResult *p) { +HParsedToken* act_label(const HParseResult *p, void* user_data) { dns_label_t *r = H_ALLOC(dns_label_t); r->len = h_seq_len(p->ast); @@ -121,7 +121,7 @@ HParsedToken* act_label(const HParseResult *p) { return H_MAKE(dns_label_t, r); } -HParsedToken* act_rr(const HParseResult *p) { +HParsedToken* act_rr(const HParseResult *p, void* user_data) { dns_rr_t *rr = H_ALLOC(dns_rr_t); rr->name = *H_FIELD(dns_domain_t, 0); @@ -136,7 +136,7 @@ HParsedToken* act_rr(const HParseResult *p) { return H_MAKE(dns_rr_t, rr); } -HParsedToken* act_question(const HParseResult *p) { +HParsedToken* act_question(const HParseResult *p, void* user_data) { dns_question_t *q = H_ALLOC(dns_question_t); HParsedToken **fields = h_seq_elements(p->ast); @@ -153,7 +153,7 @@ HParsedToken* act_question(const HParseResult *p) { return H_MAKE(dns_question_t, q); } -HParsedToken* act_message(const HParseResult *p) { +HParsedToken* act_message(const HParseResult *p, void* user_data) { h_pprint(stdout, p->ast, 0, 2); dns_message_t *msg = H_ALLOC(dns_message_t); diff --git a/examples/dns_common.c b/examples/dns_common.c index 01dd8f0..3cdd04e 100644 --- a/examples/dns_common.c +++ b/examples/dns_common.c @@ -18,7 +18,7 @@ bool validate_label(HParseResult *p) { #define act_label h_act_flatten -HParsedToken* act_domain(const HParseResult *p) { +HParsedToken* act_domain(const HParseResult *p, void* user_data) { HParsedToken *ret = NULL; char *arr = NULL; diff --git a/examples/dns_common.h b/examples/dns_common.h index 8af014b..3e30770 100644 --- a/examples/dns_common.h +++ b/examples/dns_common.h @@ -7,6 +7,6 @@ HParser* init_domain(); HParser* init_character_string(); -HParsedToken* act_index0(const HParseResult *p); +HParsedToken* act_index0(const HParseResult *p, void* user_data); #endif diff --git a/examples/rr.c b/examples/rr.c index dd25063..4a7c4be 100644 --- a/examples/rr.c +++ b/examples/rr.c @@ -17,7 +17,7 @@ bool validate_null(HParseResult *p) { return (65536 > p->ast->seq->used); } -HParsedToken *act_null(const HParseResult *p) { +HParsedToken *act_null(const HParseResult *p, void* user_data) { dns_rr_null_t *null = H_ALLOC(dns_rr_null_t); size_t len = h_seq_len(p->ast); @@ -28,7 +28,7 @@ HParsedToken *act_null(const HParseResult *p) { return H_MAKE(dns_rr_null_t, null); } -HParsedToken *act_txt(const HParseResult *p) { +HParsedToken *act_txt(const HParseResult *p, void* user_data) { dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t); const HCountedArray *arr = H_CAST_SEQ(p->ast); @@ -47,7 +47,7 @@ HParsedToken *act_txt(const HParseResult *p) { return H_MAKE(dns_rr_txt_t, txt); } -HParsedToken* act_cstr(const HParseResult *p) { +HParsedToken* act_cstr(const HParseResult *p, void* user_data) { dns_cstr_t *cs = H_ALLOC(dns_cstr_t); const HCountedArray *arr = H_CAST_SEQ(p->ast); @@ -60,7 +60,7 @@ HParsedToken* act_cstr(const HParseResult *p) { return H_MAKE(dns_cstr_t, cs); } -HParsedToken* act_soa(const HParseResult *p) { +HParsedToken* act_soa(const HParseResult *p, void* user_data) { dns_rr_soa_t *soa = H_ALLOC(dns_rr_soa_t); soa->mname = *H_FIELD(dns_domain_t, 0); @@ -74,7 +74,7 @@ HParsedToken* act_soa(const HParseResult *p) { return H_MAKE(dns_rr_soa_t, soa); } -HParsedToken* act_wks(const HParseResult *p) { +HParsedToken* act_wks(const HParseResult *p, void* user_data) { dns_rr_wks_t *wks = H_ALLOC(dns_rr_wks_t); wks->address = H_FIELD_UINT(0); @@ -87,7 +87,7 @@ HParsedToken* act_wks(const HParseResult *p) { return H_MAKE(dns_rr_wks_t, wks); } -HParsedToken* act_hinfo(const HParseResult *p) { +HParsedToken* act_hinfo(const HParseResult *p, void* user_data) { dns_rr_hinfo_t *hinfo = H_ALLOC(dns_rr_hinfo_t); hinfo->cpu = *H_FIELD(dns_cstr_t, 0); @@ -96,7 +96,7 @@ HParsedToken* act_hinfo(const HParseResult *p) { return H_MAKE(dns_rr_hinfo_t, hinfo); } -HParsedToken* act_minfo(const HParseResult *p) { +HParsedToken* act_minfo(const HParseResult *p, void* user_data) { dns_rr_minfo_t *minfo = H_ALLOC(dns_rr_minfo_t); minfo->rmailbx = *H_FIELD(dns_domain_t, 0); @@ -105,7 +105,7 @@ HParsedToken* act_minfo(const HParseResult *p) { return H_MAKE(dns_rr_minfo_t, minfo); } -HParsedToken* act_mx(const HParseResult *p) { +HParsedToken* act_mx(const HParseResult *p, void* user_data) { dns_rr_mx_t *mx = H_ALLOC(dns_rr_mx_t); mx->preference = H_FIELD_UINT(0); @@ -184,7 +184,7 @@ HParser* init_rdata(uint16_t type) { for(uint16_t i = 0; ireshape) - tok = (HParsedToken *)x->reshape(make_result(arena, tok)); + tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data); // call validation and semantic action, if present if(x->pred && !x->pred(make_result(tarena, tok))) goto no_parse; // validation failed -> no parse if(x->action) - tok = (HParsedToken *)x->action(make_result(arena, tok)); + tok = (HParsedToken *)x->action(make_result(arena, tok), x->user_data); // append to result sequence h_carray_append(seq, tok); diff --git a/src/backends/lr.c b/src/backends/lr.c index d258e8a..3739ec9 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -307,13 +307,13 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) // perform token reshape if indicated if(symbol->reshape) - value = (HParsedToken *)symbol->reshape(make_result(arena, value)); + value = (HParsedToken *)symbol->reshape(make_result(arena, value), symbol->user_data); // call validation and semantic action, if present if(symbol->pred && !symbol->pred(make_result(tarena, value))) return false; // validation failed -> no parse; terminate if(symbol->action) - value = (HParsedToken *)symbol->action(make_result(arena, value)); + value = (HParsedToken *)symbol->action(make_result(arena, value), symbol->user_data); // this is LR, building a right-most derivation bottom-up, so no reduce can // follow a reduce. we can also assume no conflict follows for GLR if we diff --git a/src/glue.c b/src/glue.c index 5e3804d..c2d915a 100644 --- a/src/glue.c +++ b/src/glue.c @@ -5,7 +5,7 @@ #include "parsers/parser_internal.h" // Helper to build HAction's that pick one index out of a sequence. -HParsedToken *h_act_index(int i, const HParseResult *p) +HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data) { if(!p) return NULL; @@ -23,7 +23,7 @@ HParsedToken *h_act_index(int i, const HParseResult *p) return tok->seq->elements[i]; } -HParsedToken *h_act_first(const HParseResult *p) { +HParsedToken *h_act_first(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -31,7 +31,7 @@ HParsedToken *h_act_first(const HParseResult *p) { return p->ast->seq->elements[0]; } -HParsedToken *h_act_second(const HParseResult *p) { +HParsedToken *h_act_second(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -39,7 +39,7 @@ HParsedToken *h_act_second(const HParseResult *p) { return p->ast->seq->elements[1]; } -HParsedToken *h_act_last(const HParseResult *p) { +HParsedToken *h_act_last(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); assert(p->ast->seq->used > 0); @@ -59,7 +59,7 @@ static void act_flatten_(HCountedArray *seq, const HParsedToken *tok) { } } -HParsedToken *h_act_flatten(const HParseResult *p) { +HParsedToken *h_act_flatten(const HParseResult *p, void* user_data) { HCountedArray *seq = h_carray_new(p->arena); act_flatten_(seq, p->ast); @@ -72,7 +72,7 @@ HParsedToken *h_act_flatten(const HParseResult *p) { return res; } -HParsedToken *h_act_ignore(const HParseResult *p) { +HParsedToken *h_act_ignore(const HParseResult *p, void* user_data) { return NULL; } diff --git a/src/glue.h b/src/glue.h index 54b5985..7486e46 100644 --- a/src/glue.h +++ b/src/glue.h @@ -56,13 +56,19 @@ #define H_RULE(rule, def) HParser *rule = def -#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule) +#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) #define H_VRULE(rule, def) HParser *rule = \ - h_attr_bool(def, validate_ ## rule) + h_attr_bool(def, validate_ ## rule) #define H_VARULE(rule, def) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule) #define H_AVRULE(rule, def) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule) + h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, NULL) +#define H_ADRULE(rule, def, data) HParser *rule = \ + h_action(def, act_ ## rule, data) +#define H_VADRULE(rule, def, data) HParser *rule = \ + h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule) +#define H_AVDRULE(rule, def, data) HParser *rule = \ + h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, data) // @@ -88,18 +94,18 @@ // action such as h_act_index. // -HParsedToken *h_act_index(int i, const HParseResult *p); -HParsedToken *h_act_first(const HParseResult *p); -HParsedToken *h_act_second(const HParseResult *p); -HParsedToken *h_act_last(const HParseResult *p); -HParsedToken *h_act_flatten(const HParseResult *p); -HParsedToken *h_act_ignore(const HParseResult *p); +HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data); +HParsedToken *h_act_first(const HParseResult *p, void* user_data); +HParsedToken *h_act_second(const HParseResult *p, void* user_data); +HParsedToken *h_act_last(const HParseResult *p, void* user_data); +HParsedToken *h_act_flatten(const HParseResult *p, void* user_data); +HParsedToken *h_act_ignore(const HParseResult *p, void* user_data); // Define 'myaction' as a specialization of 'paction' by supplying the leading // parameters. #define H_ACT_APPLY(myaction, paction, ...) \ - HParsedToken *myaction(const HParseResult *p) { \ - return paction(__VA_ARGS__, p); \ + HParsedToken *myaction(const HParseResult *p, void* user_data) { \ + return paction(__VA_ARGS__, p, user_data); \ } diff --git a/src/hammer.h b/src/hammer.h index 0175142..bd3ae02 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -113,7 +113,7 @@ typedef struct HBitWriter_ HBitWriter; * say, structs) and stuff values for them into the void* in the * tagged union in HParsedToken. */ -typedef HParsedToken* (*HAction)(const HParseResult *p); +typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data); /** * Type of a boolean attribute-checking function, used in the @@ -349,7 +349,7 @@ HAMMER_FN_DECL(HParser*, h_middle, const HParser* p, const HParser* x, const HPa * * Result token type: any */ -HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a); +HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a, void* user_data); /** * Parse a single character in the given charset. @@ -621,11 +621,11 @@ void h_bit_writer_free(HBitWriter* w); // General-purpose actions for use with h_action // XXX to be consolidated with glue.h when merged upstream -HParsedToken *h_act_first(const HParseResult *p); -HParsedToken *h_act_second(const HParseResult *p); -HParsedToken *h_act_last(const HParseResult *p); -HParsedToken *h_act_flatten(const HParseResult *p); -HParsedToken *h_act_ignore(const HParseResult *p); +HParsedToken *h_act_first(const HParseResult *p, void* userdata); +HParsedToken *h_act_second(const HParseResult *p, void* userdata); +HParsedToken *h_act_last(const HParseResult *p, void* userdata); +HParsedToken *h_act_flatten(const HParseResult *p, void* userdata); +HParsedToken *h_act_ignore(const HParseResult *p, void* userdata); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); diff --git a/src/internal.h b/src/internal.h index 89cb380..3c639d0 100644 --- a/src/internal.h +++ b/src/internal.h @@ -365,6 +365,7 @@ struct HCFChoice_ { // to execute before action and pred are applied. HAction action; HPredicate pred; + void* user_data; }; struct HCFSequence_ { diff --git a/src/parsers/action.c b/src/parsers/action.c index 52c9bc1..e254a89 100644 --- a/src/parsers/action.c +++ b/src/parsers/action.c @@ -4,6 +4,7 @@ typedef struct { const HParser *p; HAction action; + void* user_data; } HParseAction; static HParseResult* parse_action(void *env, HParseState *state) { @@ -12,8 +13,8 @@ static HParseResult* parse_action(void *env, HParseState *state) { HParseResult *tmp = h_do_parse(a->p, state); //HParsedToken *tok = a->action(h_do_parse(a->p, state)); if(tmp) { - const HParsedToken *tok = a->action(tmp); - return make_result(state->arena, (HParsedToken*)tok); + const HParsedToken *tok = a->action(tmp, a->user_data); + return make_result(state->arena, (HParsedToken*)tok); } else return NULL; } else // either the parser's missing or the action's missing @@ -27,6 +28,7 @@ static void desugar_action(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_BEGIN_SEQ() { HCFS_DESUGAR(a->p); } HCFS_END_SEQ(); + HCFS_THIS_CHOICE->user_data = a->user_data; HCFS_THIS_CHOICE->action = a->action; HCFS_THIS_CHOICE->reshape = h_act_first; } HCFS_END_CHOICE(); @@ -44,7 +46,7 @@ static bool action_isValidCF(void *env) { static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) { HParseResult res; - HAction action = arg; + HParseAction *a = arg; assert(ctx->stack_count >= 1); if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) { assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK); @@ -56,7 +58,7 @@ static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) { } res.arena = arena; - HParsedToken *tok = action(&res); + HParsedToken *tok = a->action(&res, a->user_data); if (tok != NULL) ctx->stack[ctx->stack_count-1] = tok; else @@ -69,7 +71,7 @@ static bool action_ctrvm(HRVMProg *prog, void* env) { h_rvm_insert_insn(prog, RVM_PUSH, 0); if (!h_compile_regex(prog, a->p)) return false; - h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a->action)); + h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a)); return true; } @@ -81,13 +83,14 @@ static const HParserVtable action_vt = { .compile_to_rvm = action_ctrvm, }; -HParser* h_action(const HParser* p, const HAction a) { - return h_action__m(&system_allocator, p, a); +HParser* h_action(const HParser* p, const HAction a, void* user_data) { + return h_action__m(&system_allocator, p, a, user_data); } -HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a) { +HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a, void* user_data) { HParseAction *env = h_new(HParseAction, 1); env->p = p; env->action = a; + env->user_data = user_data; return h_new_parser(mm__, &action_vt, env); } diff --git a/src/parsers/bits.c b/src/parsers/bits.c index 93b4aef..716524c 100644 --- a/src/parsers/bits.c +++ b/src/parsers/bits.c @@ -17,7 +17,9 @@ static HParseResult* parse_bits(void* env, HParseState *state) { return make_result(state->arena, result); } -static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) { +static HParsedToken *reshape_bits(const HParseResult *p, void* signedp_p) { + // signedp == NULL iff unsigned + bool signedp = (signedp_p != NULL); // XXX works only for whole bytes // XXX assumes big-endian assert(p->ast); @@ -45,12 +47,6 @@ static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) { return ret; } -static HParsedToken *reshape_bits_unsigned(const HParseResult *p) { - return reshape_bits(p, false); -} -static HParsedToken *reshape_bits_signed(const HParseResult *p) { - return reshape_bits(p, true); -} static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) { struct bits_env *bits = (struct bits_env*)env; @@ -67,9 +63,9 @@ static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_ADD_CHARSET(match_all); } } HCFS_END_SEQ(); - HCFS_THIS_CHOICE->reshape = bits->signedp - ? reshape_bits_signed - : reshape_bits_unsigned; + HCFS_THIS_CHOICE->reshape = reshape_bits; + HCFS_THIS_CHOICE->user_data = bits->signedp ? HCFS_THIS_CHOICE : NULL; // HCFS_THIS_CHOICE is an arbitrary non-null pointer + } HCFS_END_CHOICE(); } diff --git a/src/parsers/optional.c b/src/parsers/optional.c index c4282a9..ff9fc15 100644 --- a/src/parsers/optional.c +++ b/src/parsers/optional.c @@ -22,7 +22,7 @@ static bool opt_isValidCF(void *env) { return p->vtable->isValidCF(p->env); } -static HParsedToken* reshape_optional(const HParseResult *p) { +static HParsedToken* reshape_optional(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -52,6 +52,7 @@ static void desugar_optional(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_BEGIN_SEQ() { } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_optional; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c index bdea8c0..42c0913 100644 --- a/src/parsers/sequence.c +++ b/src/parsers/sequence.c @@ -43,7 +43,7 @@ static bool sequence_isValidCF(void *env) { return true; } -static HParsedToken *reshape_sequence(const HParseResult *p) { +static HParsedToken *reshape_sequence(const HParseResult *p, void* user_data) { assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -72,6 +72,7 @@ static void desugar_sequence(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_DESUGAR(s->p_array[i]); } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_sequence; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/parsers/token.c b/src/parsers/token.c index 97886b7..0a43f8d 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -20,7 +20,7 @@ static HParseResult* parse_token(void *env, HParseState *state) { } -static HParsedToken *reshape_token(const HParseResult *p) { +static HParsedToken *reshape_token(const HParseResult *p, void* user_data) { // fetch sequence of uints from p assert(p->ast); assert(p->ast->token_type == TT_SEQUENCE); @@ -52,6 +52,7 @@ static void desugar_token(HAllocator *mm__, HCFStack *stk__, void *env) { HCFS_ADD_CHAR(tok->str[i]); } HCFS_END_SEQ(); HCFS_THIS_CHOICE->reshape = reshape_token; + HCFS_THIS_CHOICE->user_data = NULL; } HCFS_END_CHOICE(); } diff --git a/src/t_parser.c b/src/t_parser.c index bba0148..292d1c4 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -162,7 +162,7 @@ static void test_middle(gconstpointer backend) { #include -HParsedToken* upcase(const HParseResult *p) { +HParsedToken* upcase(const HParseResult *p, void* user_data) { switch(p->ast->token_type) { case TT_SEQUENCE: { @@ -202,7 +202,8 @@ static void test_action(gconstpointer backend) { h_ch('B'), NULL), NULL), - upcase); + upcase, + NULL); g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x41 u0x42)"); g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "AB", 2, "(u0x41 u0x42)"); @@ -433,7 +434,7 @@ static void test_ambiguous(gconstpointer backend) { HParser *p_ = h_ch('+'); HParser *E_ = h_indirect(); h_bind_indirect(E_, h_choice(h_sequence(E_, p_, E_, NULL), d_, NULL)); - HParser *expr_ = h_action(E_, h_act_flatten); + HParser *expr_ = h_action(E_, h_act_flatten, NULL); g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "(u0x64)"); g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)"); From 69e84bcbb047c0de92d2595742d3dfa337a940d3 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 13:01:55 -0600 Subject: [PATCH 15/44] Added a void* user_data pointer to HPredicate --- examples/dns.c | 4 ++-- examples/dns_common.c | 2 +- examples/rr.c | 2 +- src/backends/llk.c | 2 +- src/backends/lr.c | 2 +- src/glue.h | 12 +++++++----- src/hammer.h | 4 ++-- src/parsers/attr_bool.c | 17 ++++++++++------- src/t_parser.c | 5 +++-- 9 files changed, 28 insertions(+), 22 deletions(-) diff --git a/examples/dns.c b/examples/dns.c index 5effb89..63df3a4 100644 --- a/examples/dns.c +++ b/examples/dns.c @@ -15,7 +15,7 @@ // Validations /// -bool validate_hdzero(HParseResult *p) { +bool validate_hdzero(HParseResult *p, void* user_data) { if (TT_UINT != p->ast->token_type) return false; return (0 == p->ast->uint); @@ -25,7 +25,7 @@ bool validate_hdzero(HParseResult *p) { * Every DNS message should have QDCOUNT entries in the question * section, and ANCOUNT+NSCOUNT+ARCOUNT resource records. */ -bool validate_message(HParseResult *p) { +bool validate_message(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; diff --git a/examples/dns_common.c b/examples/dns_common.c index 3cdd04e..bf934d6 100644 --- a/examples/dns_common.c +++ b/examples/dns_common.c @@ -10,7 +10,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0) /** * A label can't be more than 63 characters. */ -bool validate_label(HParseResult *p) { +bool validate_label(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; return (64 > p->ast->seq->used); diff --git a/examples/rr.c b/examples/rr.c index 4a7c4be..c179922 100644 --- a/examples/rr.c +++ b/examples/rr.c @@ -11,7 +11,7 @@ // Validations and Semantic Actions /// -bool validate_null(HParseResult *p) { +bool validate_null(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; return (65536 > p->ast->seq->used); diff --git a/src/backends/llk.c b/src/backends/llk.c index 39ac07a..2bc39da 100644 --- a/src/backends/llk.c +++ b/src/backends/llk.c @@ -375,7 +375,7 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream* tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data); // call validation and semantic action, if present - if(x->pred && !x->pred(make_result(tarena, tok))) + if(x->pred && !x->pred(make_result(tarena, tok), x->user_data)) goto no_parse; // validation failed -> no parse if(x->action) tok = (HParsedToken *)x->action(make_result(arena, tok), x->user_data); diff --git a/src/backends/lr.c b/src/backends/lr.c index 3739ec9..e7f2377 100644 --- a/src/backends/lr.c +++ b/src/backends/lr.c @@ -310,7 +310,7 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action) value = (HParsedToken *)symbol->reshape(make_result(arena, value), symbol->user_data); // call validation and semantic action, if present - if(symbol->pred && !symbol->pred(make_result(tarena, value))) + if(symbol->pred && !symbol->pred(make_result(tarena, value), symbol->user_data)) return false; // validation failed -> no parse; terminate if(symbol->action) value = (HParsedToken *)symbol->action(make_result(arena, value), symbol->user_data); diff --git a/src/glue.h b/src/glue.h index 7486e46..74963b0 100644 --- a/src/glue.h +++ b/src/glue.h @@ -58,17 +58,19 @@ #define H_RULE(rule, def) HParser *rule = def #define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) #define H_VRULE(rule, def) HParser *rule = \ - h_attr_bool(def, validate_ ## rule) + h_attr_bool(def, validate_ ## rule, NULL) #define H_VARULE(rule, def) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule, NULL) #define H_AVRULE(rule, def) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, NULL) + h_action(h_attr_bool(def, validate_ ## rule, NULL), act_ ## rule, NULL) #define H_ADRULE(rule, def, data) HParser *rule = \ h_action(def, act_ ## rule, data) +#define H_VDRULE(rule, def, data) HParser *rule = \ + h_attr_bool(def, validate_ ## rule, data) #define H_VADRULE(rule, def, data) HParser *rule = \ - h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule) + h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule, data) #define H_AVDRULE(rule, def, data) HParser *rule = \ - h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule, data) + h_action(h_attr_bool(def, validate_ ## rule, data), act_ ## rule, data) // diff --git a/src/hammer.h b/src/hammer.h index bd3ae02..541e38d 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -120,7 +120,7 @@ typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data); * attr_bool() parser. It can be any (user-defined) function that takes * a HParseResult* and returns true or false. */ -typedef bool (*HPredicate)(HParseResult *p); +typedef bool (*HPredicate)(HParseResult *p, void* user_data); typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; @@ -515,7 +515,7 @@ HAMMER_FN_DECL(HParser*, h_length_value, const HParser* length, const HParser* v * * Result token type: p's result type if pred succeeded, NULL otherwise. */ -HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred); +HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred, void* user_data); /** * The 'and' parser asserts that a conditional syntax is satisfied, diff --git a/src/parsers/attr_bool.c b/src/parsers/attr_bool.c index fc980b2..e8359ab 100644 --- a/src/parsers/attr_bool.c +++ b/src/parsers/attr_bool.c @@ -4,13 +4,14 @@ typedef struct { const HParser *p; HPredicate pred; + void* user_data; } HAttrBool; static HParseResult* parse_attr_bool(void *env, HParseState *state) { HAttrBool *a = (HAttrBool*)env; HParseResult *res = h_do_parse(a->p, state); if (res && res->ast) { - if (a->pred(res)) + if (a->pred(res, a->user_data)) return res; else return NULL; @@ -42,12 +43,13 @@ static void desugar_ab(HAllocator *mm__, HCFStack *stk__, void *env) { } HCFS_END_SEQ(); HCFS_THIS_CHOICE->pred = a->pred; HCFS_THIS_CHOICE->reshape = h_act_first; + HCFS_THIS_CHOICE->user_data = a->user_data; } HCFS_END_CHOICE(); } static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) { HParseResult res; - HPredicate pred = arg; + HAttrBool *ab = arg; assert(ctx->stack_count >= 1); if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) { assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK); @@ -59,7 +61,7 @@ static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) { res.ast = NULL; } res.arena = arena; - return pred(&res); + return ab->pred(&res, ab->user_data); } static bool ab_ctrvm(HRVMProg *prog, void *env) { @@ -67,7 +69,7 @@ static bool ab_ctrvm(HRVMProg *prog, void *env) { h_rvm_insert_insn(prog, RVM_PUSH, 0); if (!h_compile_regex(prog, ab->p)) return false; - h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab->pred)); + h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab)); return true; } @@ -80,12 +82,13 @@ static const HParserVtable attr_bool_vt = { }; -HParser* h_attr_bool(const HParser* p, HPredicate pred) { - return h_attr_bool__m(&system_allocator, p, pred); +HParser* h_attr_bool(const HParser* p, HPredicate pred, void* user_data) { + return h_attr_bool__m(&system_allocator, p, pred, user_data); } -HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred) { +HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred, void* user_data) { HAttrBool *env = h_new(HAttrBool, 1); env->p = p; env->pred = pred; + env->user_data = user_data; return h_new_parser(mm__, &attr_bool_vt, env); } diff --git a/src/t_parser.c b/src/t_parser.c index 292d1c4..12edba9 100644 --- a/src/t_parser.c +++ b/src/t_parser.c @@ -365,7 +365,7 @@ static void test_epsilon_p(gconstpointer backend) { g_check_parse_match(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)"); } -bool validate_test_ab(HParseResult *p) { +bool validate_test_ab(HParseResult *p, void* user_data) { if (TT_SEQUENCE != p->ast->token_type) return false; if (TT_UINT != p->ast->seq->elements[0]->token_type) @@ -377,7 +377,8 @@ bool validate_test_ab(HParseResult *p) { static void test_attr_bool(gconstpointer backend) { const HParser *ab_ = h_attr_bool(h_many1(h_choice(h_ch('a'), h_ch('b'), NULL)), - validate_test_ab); + validate_test_ab, + NULL); g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)"); g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "bb", 2, "(u0x62 u0x62)"); From 890c6383a78f469429aeddf5c3f02af318cf103f Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 13:09:39 -0600 Subject: [PATCH 16/44] Added documentation for H_{A,V,VA,AV}DRULE --- src/glue.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/glue.h b/src/glue.h index 74963b0..1fe6ce4 100644 --- a/src/glue.h +++ b/src/glue.h @@ -53,7 +53,10 @@ // H_AVRULE is like H_VARULE but the action is attached outside the validation, // i.e. the validation receives the uninterpreted AST as input. // - +// H_ADRULE, H_VDRULE, H_AVDRULE, and H_VADRULE are the same as the +// equivalent non-D variants, except that they also allow you to uset +// the user_data pointer. In cases where both an attr_bool and an +// action are used, the same userdata pointer is given to both. #define H_RULE(rule, def) HParser *rule = def #define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL) From de6f6115a80db68d5d5b887f12d7b8cec1e19e23 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:00:50 -0400 Subject: [PATCH 17/44] We declared some functions that were never implemented. Implement them. --- src/hammer.c | 4 ++++ src/hammer.h | 3 ++- src/parsers/indirect.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hammer.c b/src/hammer.c index 7fc80db..2456bdc 100644 --- a/src/hammer.c +++ b/src/hammer.c @@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* return backends[parser->backend]->parse(mm__, parser, &input_stream); } +void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) { + h_parse_result_free(result); +} + void h_parse_result_free(HParseResult *result) { if(result == NULL) return; h_delete_arena(result->arena); diff --git a/src/hammer.h b/src/hammer.h index 541e38d..08819f5 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -126,6 +126,7 @@ typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; +// TODO: Make this internal typedef struct HParser_ { const HParserVtable *vtable; HParserBackend backend; @@ -586,7 +587,7 @@ char* h_write_result_unamb(const HParsedToken* tok); * Format token to the given output stream. Indent starting at * [indent] spaces, with [delta] spaces between levels. */ -HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta); +void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta); /** * Build parse tables for the given parser backend. See the diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c index 2217a20..c91eaab 100644 --- a/src/parsers/indirect.c +++ b/src/parsers/indirect.c @@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = { .compile_to_rvm = h_not_regular, }; +void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) { + h_bind_indirect(indirect, inner); +} + void h_bind_indirect(HParser* indirect, const HParser* inner) { assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser"); indirect->env = (void*)inner; From 28bee303f9d1885c69b0428548ca89e528d6e201 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 1 Nov 2013 18:01:44 -0400 Subject: [PATCH 18/44] Add CFFI python bindings --- src/bindings/python/hammer.py | 244 ++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py new file mode 100644 index 0000000..d3d1e3d --- /dev/null +++ b/src/bindings/python/hammer.py @@ -0,0 +1,244 @@ +from cffi import FFI + +ffi = FFI() + +## Types +ffi.cdef("typedef struct HAllocator_ HAllocator;") +ffi.cdef("typedef struct HArena_ HArena;") +ffi.cdef("typedef int bool;") +ffi.cdef("typedef struct HParseState_ HParseState;") +ffi.cdef(""" +typedef enum HParserBackend_ { + PB_MIN = 0, + PB_PACKRAT = 0, // PB_MIN is always the default. + PB_REGULAR, + PB_LLk, + PB_LALR, + PB_GLR +// TODO: support PB_MAX +} HParserBackend; +""") +ffi.cdef(""" +typedef enum HTokenType_ { + // Before you change the explicit values of these, think of the poor bindings ;_; + TT_NONE = 1, + TT_BYTES = 2, + TT_SINT = 4, + TT_UINT = 8, + TT_SEQUENCE = 16, + TT_RESERVED_1, // reserved for backend-specific internal use + TT_ERR = 32, + TT_USER = 64, + TT_MAX +} HTokenType; +""") +ffi.cdef(""" +typedef struct HCountedArray_ { + size_t capacity; + size_t used; + HArena * arena; + struct HParsedToken_ **elements; +} HCountedArray; +""") +ffi.cdef(""" +typedef struct HBytes_ { + const uint8_t *token; + size_t len; +} HBytes; +""") +ffi.cdef(""" +typedef struct HParsedToken_ { + HTokenType token_type; + union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; // a sequence of HParsedToken's + void *user; + }; + size_t index; + char bit_offset; +} HParsedToken; +""") +ffi.cdef(""" +typedef struct HParseResult_ { + const HParsedToken *ast; + long long bit_length; + HArena * arena; +} HParseResult; +""") + +ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +ffi.cdef(""" +typedef struct HCFChoice_ HCFChoice; +typedef struct HRVMProg_ HRVMProg; +typedef struct HParserVtable_ HParserVtable; +""") + +ffi.cdef("typedef struct HParser_ HParser;") +ffi.cdef(""" +typedef struct HParserTestcase_ { + unsigned char* input; + size_t length; + char* output_unambiguous; +} HParserTestcase; + +typedef struct HCaseResult_ { + bool success; + union { + const char* actual_results; // on failure, filled in with the results of h_write_result_unamb + size_t parse_time; // on success, filled in with time for a single parse, in nsec + }; +} HCaseResult; + +typedef struct HBackendResults_ { + HParserBackend backend; + bool compile_success; + size_t n_testcases; + size_t failed_testcases; // actually a count... + HCaseResult *cases; +} HBackendResults; + +typedef struct HBenchmarkResults_ { + size_t len; + HBackendResults *results; +} HBenchmarkResults; +""") + +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' +ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +ffi.cdef("HParser* h_ch(const uint8_t c);") +ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +ffi.cdef("HParser* h_bits(size_t len, bool sign);") +ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +ffi.cdef("HParser* h_int64(void);") +ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int32(void);") +ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int16(void);") +ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_int8(void);") +ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint64(void);") +ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint32(void);") +ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint16(void);") +ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +ffi.cdef("HParser* h_uint8(void);") +ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +ffi.cdef("HParser* h_whitespace(const HParser* p);") +ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +ffi.cdef("HParser* h_end_p(void);") +ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_nothing_p(void);") +ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_sequence(HParser* p, ...);") +ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_sequence__a(void* args);") +ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_choice(HParser* p, ...);") +ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +ffi.cdef("HParser* h_choice__a(void* args);") +ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +ffi.cdef("HParser* h_many(const HParser* p);") +ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_many1(const HParser* p);") +ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +ffi.cdef("HParser* h_optional(const HParser* p);") +ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_ignore(const HParser* p);") +ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +ffi.cdef("HParser* h_epsilon_p(void);") +ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +ffi.cdef("HParser* h_and(const HParser* p);") +ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_not(const HParser* p);") +ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +ffi.cdef("HParser* h_indirect(void);") +ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +ffi.cdef("void h_parse_result_free(HParseResult *result);") +ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +lib = ffi.verify("#include ", + libraries=['hammer']) + + +# Quick test +def fromCobj(cobj): + # TODO: Free the toplevel parser + tt = cobj.token_type + if cobj.token_type == lib.TT_BYTES: + return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == lib.TT_ERR: + # I have no idea what this is for + pass + elif cobj.token_type == lib.TT_NONE: + return None + elif cobj.token_type == lib.TT_SEQUENCE: + return [fromCobj(cobj.seq.elements[i]) + for i in range(cobj.seq.used)] + elif cobj.token_type == lib.TT_SINT: + return cobj.sint + elif cobj.token_type == lib.TT_UINT: + return cobj.uint + +def fromParseResult(cobj): + ret = fromCobj(cobj.ast) + lib.h_parse_result_free(cobj) + return ret + +def run_test(): + p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), + lib.h_ch(ord('2')), + lib.h_ch(ord('3')), + ffi.NULL), + lib.h_ch(ord(','))) + return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) + From 9d43244c2e50dd3c92df2ad8b46a885cf340c272 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Thu, 7 Nov 2013 22:34:27 -0500 Subject: [PATCH 19/44] Working python bindings --- src/bindings/desugar-header.pl | 22 ++ src/bindings/python/hammer.py | 462 +++++++++++++++++++++++---------- src/parsers/token.c | 4 +- 3 files changed, 353 insertions(+), 135 deletions(-) create mode 100644 src/bindings/desugar-header.pl diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl new file mode 100644 index 0000000..5bdd11e --- /dev/null +++ b/src/bindings/desugar-header.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + + +my $arg = qr/[^,]*/; + +while(<>) { + chomp; + if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) { + print "$1 $2(void);\n"; + print "$1 $2__m(HAllocator* mm__);\n"; + } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) { + print "$1 $2($3);\n"; + print "$1 $2__m(HAllocator* mm__, $3);\n"; + } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) { + print "$2 $3($4, ...);\n"; + print "$2 $3__m(HAllocator *mm__, $4, ...);\n"; + print "$2 $3__a(void* args);\n"; + print "$2 $3__ma(HAllocator* mm__, void* args);\n"; + } elsif (/^HAMMER_FN_DECL/) { + print "\e[1;31m!!!\e[0m " . $_ . "\n"; + } +} diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index d3d1e3d..e7e0822 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -1,13 +1,15 @@ from cffi import FFI +import threading +import sys -ffi = FFI() +_ffi = FFI() ## Types -ffi.cdef("typedef struct HAllocator_ HAllocator;") -ffi.cdef("typedef struct HArena_ HArena;") -ffi.cdef("typedef int bool;") -ffi.cdef("typedef struct HParseState_ HParseState;") -ffi.cdef(""" +_ffi.cdef("typedef struct HAllocator_ HAllocator;") +_ffi.cdef("typedef struct HArena_ HArena;") +_ffi.cdef("typedef int bool;") +_ffi.cdef("typedef struct HParseState_ HParseState;") +_ffi.cdef(""" typedef enum HParserBackend_ { PB_MIN = 0, PB_PACKRAT = 0, // PB_MIN is always the default. @@ -18,7 +20,7 @@ typedef enum HParserBackend_ { // TODO: support PB_MAX } HParserBackend; """) -ffi.cdef(""" +_ffi.cdef(""" typedef enum HTokenType_ { // Before you change the explicit values of these, think of the poor bindings ;_; TT_NONE = 1, @@ -32,7 +34,7 @@ typedef enum HTokenType_ { TT_MAX } HTokenType; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HCountedArray_ { size_t capacity; size_t used; @@ -40,13 +42,13 @@ typedef struct HCountedArray_ { struct HParsedToken_ **elements; } HCountedArray; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HBytes_ { const uint8_t *token; size_t len; } HBytes; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParsedToken_ { HTokenType token_type; union { @@ -62,7 +64,7 @@ typedef struct HParsedToken_ { char bit_offset; } HParsedToken; """) -ffi.cdef(""" +_ffi.cdef(""" typedef struct HParseResult_ { const HParsedToken *ast; long long bit_length; @@ -70,16 +72,16 @@ typedef struct HParseResult_ { } HParseResult; """) -ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -ffi.cdef(""" +_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") +_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") +_ffi.cdef(""" typedef struct HCFChoice_ HCFChoice; typedef struct HRVMProg_ HRVMProg; typedef struct HParserVtable_ HParserVtable; """) -ffi.cdef("typedef struct HParser_ HParser;") -ffi.cdef(""" +_ffi.cdef("typedef struct HParser_ HParser;") +_ffi.cdef(""" typedef struct HParserTestcase_ { unsigned char* input; size_t length; @@ -108,137 +110,329 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/ffi.cdef("&")/' -ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -ffi.cdef("HParser* h_ch(const uint8_t c);") -ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -ffi.cdef("HParser* h_bits(size_t len, bool sign);") -ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -ffi.cdef("HParser* h_int64(void);") -ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int32(void);") -ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int16(void);") -ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_int8(void);") -ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint64(void);") -ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint32(void);") -ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint16(void);") -ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -ffi.cdef("HParser* h_uint8(void);") -ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -ffi.cdef("HParser* h_whitespace(const HParser* p);") -ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -ffi.cdef("HParser* h_end_p(void);") -ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_nothing_p(void);") -ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_sequence(HParser* p, ...);") -ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_sequence__a(void* args);") -ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_choice(HParser* p, ...);") -ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -ffi.cdef("HParser* h_choice__a(void* args);") -ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -ffi.cdef("HParser* h_many(const HParser* p);") -ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_many1(const HParser* p);") -ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -ffi.cdef("HParser* h_optional(const HParser* p);") -ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_ignore(const HParser* p);") -ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -ffi.cdef("HParser* h_epsilon_p(void);") -ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -ffi.cdef("HParser* h_and(const HParser* p);") -ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_not(const HParser* p);") -ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -ffi.cdef("HParser* h_indirect(void);") -ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -ffi.cdef("void h_parse_result_free(HParseResult *result);") -ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") +## Arena functions +_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") +_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -lib = ffi.verify("#include ", +## The following section was generated by +## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' +_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") +_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") +_ffi.cdef("HParser* h_ch(const uint8_t c);") +_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") +_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") +_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") +_ffi.cdef("HParser* h_bits(size_t len, bool sign);") +_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") +_ffi.cdef("HParser* h_int64(void);") +_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int32(void);") +_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int16(void);") +_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_int8(void);") +_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint64(void);") +_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint32(void);") +_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint16(void);") +_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_uint8(void);") +_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_whitespace(const HParser* p);") +_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") +_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") +_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") +_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") +_ffi.cdef("HParser* h_end_p(void);") +_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_nothing_p(void);") +_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_sequence(HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_sequence__a(void* args);") +_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_choice(HParser* p, ...);") +_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") +_ffi.cdef("HParser* h_choice__a(void* args);") +_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") +_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") +_ffi.cdef("HParser* h_many(const HParser* p);") +_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_many1(const HParser* p);") +_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") +_ffi.cdef("HParser* h_optional(const HParser* p);") +_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_ignore(const HParser* p);") +_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") +_ffi.cdef("HParser* h_epsilon_p(void);") +_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") +_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") +_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") +_ffi.cdef("HParser* h_and(const HParser* p);") +_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_not(const HParser* p);") +_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") +_ffi.cdef("HParser* h_indirect(void);") +_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") +_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") +_ffi.cdef("void h_parse_result_free(HParseResult *result);") +_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") +_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") +_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") +_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") +_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") + +_lib = _ffi.verify("#include ", libraries=['hammer']) +_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# Quick test -def fromCobj(cobj): +class _DynamicScopeHolder(threading.local): + """A dynamically-scoped holder of python objects, which may or may not + otherwise appear in the object graph. Intended for use with CFFI """ + def __init__(self): + self._ctxstack = [] + def __enter__(self): + self._ctxstack.append([]) + def __exit__(self, exc_type, exc_value, traceback): + self._ctxstack.pop() + return False + def stash(self, *objs): + if len(self._ctxstack) < 1: + raise Exception("Not in any dynamic scope") + for obj in objs: + self._ctxstack[-1].append(obj) +def _fromHParsedToken(cobj): # TODO: Free the toplevel parser tt = cobj.token_type - if cobj.token_type == lib.TT_BYTES: - return ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == lib.TT_ERR: + + if cobj.token_type == _lib.TT_BYTES: + return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] + elif cobj.token_type == _lib.TT_ERR: # I have no idea what this is for pass - elif cobj.token_type == lib.TT_NONE: + elif cobj.token_type == _lib.TT_NONE: return None - elif cobj.token_type == lib.TT_SEQUENCE: - return [fromCobj(cobj.seq.elements[i]) + elif cobj.token_type == _lib.TT_SEQUENCE: + return [_fromHParsedToken(cobj.seq.elements[i]) for i in range(cobj.seq.used)] - elif cobj.token_type == lib.TT_SINT: + elif cobj.token_type == _lib.TT_SINT: return cobj.sint - elif cobj.token_type == lib.TT_UINT: + elif cobj.token_type == _lib.TT_UINT: return cobj.uint + elif cobj.token_type == _lib.TT_PYTHON: + return _ffi.from_handle(cobj.user) -def fromParseResult(cobj): - ret = fromCobj(cobj.ast) - lib.h_parse_result_free(cobj) +_parser_result_holder = _DynamicScopeHolder() +def _toHParsedToken(arena, pyobj): + if pyobj is None: + return _ffi.NULL + cobj = _ffi.new_handle(pyobj) + _parser_result_holder.stash(cobj) + + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt.token_type = _lib.TT_PYTHON + hpt.user = cobj + hpt.bit_offset = 127; + hpt.index = 0; + return hpt + +def _fromParseResult(cobj): + ret = _fromHParsedToken(cobj.ast) + _lib.h_parse_result_free(cobj) return ret -def run_test(): - p_test = lib.h_sepBy1(lib.h_choice(lib.h_ch(ord('1')), - lib.h_ch(ord('2')), - lib.h_ch(ord('3')), - ffi.NULL), - lib.h_ch(ord(','))) - return fromParseResult(lib.h_parse(p_test, "1,2,3", 5)) +def _to_haction(fn): + """Turn a function that transforms a parsed value into an HAction""" + def action(parse_result): + res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) + if res != _ffi.NULL and parse_result.ast != _ffi.NULL: + res.index = parse_result.ast.index + res.bit_offset = parse_result.ast.bit_offset + return res + return _ffi.callback("HParsedToken*(HParseResult*)", action) +def _to_hpredicate(fn): + """Turn a function that transforms a parsed value into an HAction""" + def predicate(parse_result): + res = fn(_fromParseResult(parse_result)) + # TODO: Handle exceptions; parse should fail. + if type(res) != bool: + raise TypeError("Predicates should return a bool") + return res + return _ffi.callback("bool(HParseResult*)", action) + +class Parser(object): + # TODO: Map these to individually garbage-collected blocks of + # memory. Perhaps with an arena allocator with block size of 1? + # There has to be something more efficient than that, though. + + # TODO: How do we handle encodings? By default, we're using UTF-8 + def __init__(self, internal, deps): + """Create a new parser from an FFI object. Not for user code""" + self._parser = internal + self._deps = deps + + def parse(self, string): + with _parser_result_holder: + pres = _lib.h_parse(self._parser, string, len(string)) + if pres: + return _fromParseResult(pres) + else: + return None + +class IndirectParser(Parser): + def bind(self, inner): + _lib.h_bind_indirect(self._parser, inner._parser) + self._deps = (inner,) + +class BitsParser(Parser): + pass + +def token(token): + # TODO: Does not clone argument. + if isinstance(token, unicode): + token = token.encode("utf-8") + return Parser(_lib.h_token(token, len(token)), ()) + +def ch(char): + return token(char) + +def ch_range(chr1, chr2): + if not isinstance(chr1, str) or not isinstance(chr2, str): + raise TypeError("ch_range can't handle unicode") + return Parser(_lib.h_ch_range(chr1, chr2), ()) + +def int_range(parser, i1, i2): + if type(parser) != BitsParser: + raise TypeError("int_range is only valid when used with a bits parser") + return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + +def bits(length, signedp): + return BitsParser(_lib.h_bits(length, signedp), ()) + +def int64(): return bits(64, True) +def int32(): return bits(32, True) +def int16(): return bits(16, True) +def int8 (): return bits(8, True) +def uint64(): return bits(64, False) +def uint32(): return bits(32, False) +def uint16(): return bits(16, False) +def uint8 (): return bits(8, False) + +def whitespace(p): + return Parser(_lib.h_whitespace(p._parser), (p,)) +def left(p1, p2): + return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) +def right(p1, p2): + return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) +def middle(p1, p2, p3): + return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) +def action(parser, action): + caction = _to_haction(action) + return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) +def in_(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_in(charset, len(charset)), ()) +def not_in(charset): + if typeof(charset) is not str: + # TODO/Python3: change str to bytes + raise TypeError("in_ can't deal with unicode") + return Parser(_lib.h_not_in(charset, len(charset)), ()) +def end_p(): + return Parser(_lib.h_end_p(), ()) +def nothing_p(): + return Parser(_lib.h_nothing_p(), ()) +def sequence(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_sequence(*plist), (plist,)) +def choice(*parsers): + plist = [p._parser for p in parsers] + plist.append(_ffi.NULL) + return Parser(_lib.h_choice(*plist), (plist,)) +def butnot(p1, p2): + return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) +def difference(p1, p2): + return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) +def xor(p1, p2): + return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) +def many(p1): + return Parser(_lib.h_many(p1._parser), (p1,)) +def many1(p1): + return Parser(_lib.h_many1(p1._parser), (p1,)) +def repeat_n(p1, n): + return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) +def optional(p1): + return Parser(_lib.h_optional(p1._parser), (p1,)) +def ignore(p1): + return Parser(_lib.h_ignore(p1._parser), (p1,)) +def sepBy(p, sep): + return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) +def sepBy1(p, sep): + return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) +def epsilon_p(): + return Parser(_lib.h_epsilon_p(), ()) +def length_value(p_len, p_value): + return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) +def attr_bool(parser, predicate): + cpredicate = _to_hpredicate(predicate) + return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) +def and_(parser): + return Parser(_lib.h_and(parser._parser), (parser,)) +def not_(parser): + return Parser(_lib.h_not(parser._parser), (parser,)) +def indirect(): + return IndirectParser(_lib.h_indirect(), ()) +def bind_indirect(indirect, inner): + indirect.bind(inner) + +def parse(parser): + return parser.parse() + +# Unfortunately, "in", "and", and "not" are keywords. This makes them +# show up in the module namespace for the use of automated tools. Do +# not attempt to use them by hand; only use the mangled forms (with +# the '_') +sys.modules[__name__].__dict__["in"] = in_ +sys.modules[__name__].__dict__["and"] = and_ +sys.modules[__name__].__dict__["not"] = not_ + +def run_test(): + p_test = sepBy1(choice(ch('1'), + ch('2'), + ch('3')), + ch(',')) + return p_test.parse("1,2,3") diff --git a/src/parsers/token.c b/src/parsers/token.c index 0a43f8d..d36ec54 100644 --- a/src/parsers/token.c +++ b/src/parsers/token.c @@ -80,6 +80,8 @@ HParser* h_token(const uint8_t *str, const size_t len) { } HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { HToken *t = h_new(HToken, 1); - t->str = (uint8_t*)str, t->len = len; + uint8_t *str_cpy = h_new(uint8_t, len); + memcpy(str_cpy, str, len); + t->str = str_cpy, t->len = len; return h_new_parser(mm__, &token_vt, t); } From a31f2d31a4f3b5ebae5df16aa254c5a12689d9b9 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 8 Nov 2013 17:20:00 -0500 Subject: [PATCH 20/44] Added auto-construction, tidied h_ch a bit --- src/bindings/python/hammer.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index e7e0822..a25dd93 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -4,7 +4,8 @@ import sys _ffi = FFI() -## Types +# {{{ Types + _ffi.cdef("typedef struct HAllocator_ HAllocator;") _ffi.cdef("typedef struct HArena_ HArena;") _ffi.cdef("typedef int bool;") @@ -110,10 +111,12 @@ typedef struct HBenchmarkResults_ { } HBenchmarkResults; """) -## Arena functions +# }}} +# {{{ Arena functions _ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") _ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") - +# }}} +# {{{ cdefs ## The following section was generated by ## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' _ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") @@ -216,7 +219,7 @@ _lib = _ffi.verify("#include ", libraries=['hammer']) _lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 - +# }}} class _DynamicScopeHolder(threading.local): """A dynamically-scoped holder of python objects, which may or may not otherwise appear in the object graph. Intended for use with CFFI """ @@ -263,8 +266,8 @@ def _toHParsedToken(arena, pyobj): hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj - hpt.bit_offset = 127; - hpt.index = 0; + hpt.bit_offset = chr(127) + hpt.index = 0 return hpt def _fromParseResult(cobj): @@ -311,6 +314,11 @@ class Parser(object): else: return None + def __mul__(self, count): + return repeat_n(self, count) + + + class IndirectParser(Parser): def bind(self, inner): _lib.h_bind_indirect(self._parser, inner._parser) @@ -326,12 +334,20 @@ def token(token): return Parser(_lib.h_token(token, len(token)), ()) def ch(char): - return token(char) + """Returns either a token or an int, depending on the type of the + argument""" + if isinstance(char, int): + return Parser(_lib.h_ch(char), ()) + else: + return token(char) def ch_range(chr1, chr2): if not isinstance(chr1, str) or not isinstance(chr2, str): raise TypeError("ch_range can't handle unicode") - return Parser(_lib.h_ch_range(chr1, chr2), ()) + def my_action(pr): + # print "In action: ", pr + return pr + return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) def int_range(parser, i1, i2): if type(parser) != BitsParser: @@ -436,3 +452,36 @@ def run_test(): ch('3')), ch(',')) return p_test.parse("1,2,3") + +# {{{ Automatic parser construction... python specific + +# TODO: Implement Parsable metaclass, which requires the existence of +# a "parse" method. + +# This is expected to be extended by user code. As a general rule, +# only provide auto-parsers for your own types. +AUTO_PARSERS = { + str: token, + unicode: token, +} + +def _auto_seq(lst): + return sequence(*(auto_1(p, default_method=_auto_choice) + for p in lst)) + +def _auto_choice(lst): + return choice(*(auto_1(p, default_method=_auto_seq) + for p in lst)) + +def auto_1(arg, default_method=_auto_choice): + if isinstance(arg, Parser): + return arg + elif type(arg) in AUTO_PARSERS: + return AUTO_PARSERS[type(arg)](arg) + else: + return default_method(arg) + +def auto(*args): + return auto_1(args, default_method=_auto_choice) + +# }}} From 1841c9d77ecc006046c164293f3ba67c17cad161 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Tue, 12 Nov 2013 19:07:32 -0600 Subject: [PATCH 21/44] Added a few tests --- src/bindings/python/hammer_tests.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bindings/python/hammer_tests.py diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py new file mode 100644 index 0000000..b040141 --- /dev/null +++ b/src/bindings/python/hammer_tests.py @@ -0,0 +1,29 @@ +import unittest +import hammer as h + +class TestTokenParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.token("95\xa2") + def test_success(self): + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + def test_partial_fails(self): + self.assertEqual(self.parser.parse("95"), None) + +class TestChParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") + def test_success(self): + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + def test_failure(self): + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) + +class TestChRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = + From 75453d8b2f88336131e31130a89fc25c327cb421 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Thu, 14 Nov 2013 15:50:58 +0100 Subject: [PATCH 22/44] Finished writing unit tests. Fixed a few small bugs in hammer.py. Further issues: * "in_" and "not_in" should coerce their results to strings (i.e., chr(result)) * TestLeftrec: success case 2 fails * TestChRange: success case segfaults * TestWhitespaceEnd: success case segfaults * TestAction: success case segfaults with "corrupted double-linked list" * TestButNotRange: segfaults, probably because of whatever's wrong with ch_range * TestXor: segfaults; failure case craps out with "malloc(): smallbin double linked list corrupted" --- src/bindings/python/hammer.py | 15 +- src/bindings/python/hammer_tests.py | 479 +++++++++++++++++++++++++++- 2 files changed, 486 insertions(+), 8 deletions(-) diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py index a25dd93..36b78c8 100644 --- a/src/bindings/python/hammer.py +++ b/src/bindings/python/hammer.py @@ -263,7 +263,7 @@ def _toHParsedToken(arena, pyobj): cobj = _ffi.new_handle(pyobj) _parser_result_holder.stash(cobj) - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(_ffi.sizeof(parseResult.arena, "HParsedToken"))) + hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) hpt.token_type = _lib.TT_PYTHON hpt.user = cobj hpt.bit_offset = chr(127) @@ -293,7 +293,7 @@ def _to_hpredicate(fn): if type(res) != bool: raise TypeError("Predicates should return a bool") return res - return _ffi.callback("bool(HParseResult*)", action) + return _ffi.callback("bool(HParseResult*)", predicate) class Parser(object): # TODO: Map these to individually garbage-collected blocks of @@ -352,7 +352,7 @@ def ch_range(chr1, chr2): def int_range(parser, i1, i2): if type(parser) != BitsParser: raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (_parser,)) + return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) def bits(length, signedp): return BitsParser(_lib.h_bits(length, signedp), ()) @@ -373,17 +373,18 @@ def left(p1, p2): def right(p1, p2): return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3.parser), (p1, p2, p3)) + return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) def action(parser, action): caction = _to_haction(action) return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) + def in_(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_in(charset, len(charset)), ()) def not_in(charset): - if typeof(charset) is not str: + if not isinstance(charset, str): # TODO/Python3: change str to bytes raise TypeError("in_ can't deal with unicode") return Parser(_lib.h_not_in(charset, len(charset)), ()) @@ -402,7 +403,7 @@ def choice(*parsers): def butnot(p1, p2): return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) def difference(p1, p2): - return Parser(_lib.h_difference(p1, _parser, p2._parser), (p1, p2)) + return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) def xor(p1, p2): return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) def many(p1): diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index b040141..a56d669 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -25,5 +25,482 @@ class TestChParser(unittest.TestCase): class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = + cls.parser = h.ch_range("a", "c") +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestInt64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int64() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + +class TestInt32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int32() + def test_success(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestInt16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int16() + def test_success(self): + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) + +class TestInt8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int8() + def test_success(self): + self.assertEqual(self.parser.parse("\x88"), -0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestUint64(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint64() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + +class TestUint32(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint32() + def test_success(self): + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + def test_failure(self): + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + +class TestUint16(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint16() + def test_success(self): + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + def test_failure(self): + self.assertEqual(self.parser.parse("\x02"), None) + +class TestUint8(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.uint8() + def test_success(self): + self.assertEqual(self.parser.parse("\x78"), 0x78) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) +class TestIntRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.int_range(h.uint8(), 3, 10) + def test_success(self): + self.assertEqual(self.parser.parse("\x05"), 5) + def test_failure(self): + self.assertEqual(self.parser.parse("\x0b"), None) + +class TestWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("_a"), None) + +class TestWhitespaceEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.whitespace(h.end_p()) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse(""), "") +# self.assertEqual(self.parser.parse(" "), "") + def test_failure(self): + self.assertEqual(self.parser.parse(" x"), None) + +class TestLeft(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.left(h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse("a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) + +class TestRight(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.right(h.ch(" "), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse(" a"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) + +class TestMiddle(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + def test_success(self): + self.assertEqual(self.parser.parse(" a "), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) + +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) +### fails with "corrupted double-linked list" +# def test_success(self): +# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) +# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + def test_failure(self): + self.assertEqual(self.parser.parse("XX"), None) + +class TestIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.in_("abc") + def test_success(self): + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("d"), None) + +class TestNotIn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.not_in("abc") + def test_success(self): + self.assertEqual(self.parser.parse("d"), "d") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestEndP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.end_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aa"), None) + +class TestNothingP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.nothing_p() + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +class TestSequence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) + +class TestSequenceWhitespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a c"), None) + +class TestChoice(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.choice(h.ch("a"), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") + def test_failure(self): + self.assertEqual(self.parser.parse("c"), None) + +class TestButNot(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.butnot(h.ch("a"), h.token("ab")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") + def test_failure(self): + self.assertEqual(self.parser.parse("ab"), None) + +### fails with malloc() memory corruption +#class TestButNotRange(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) +# def test_success(self): +# self.assertEqual(self.parser.parse("4"), "4") +### this segfaults +# def test_failure(self): +# self.assertEqual(self.parser.parse("6"), None) + +class TestDifference(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.difference(h.token("ab"), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), "ab") + def test_failure(self): + self.assertEqual(self.parser.parse("a"), None) + +#class TestXor(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) +### this segfaults +# def test_success(self): +# self.assertEqual(self.parser.parse("0"), "0") +# self.assertEqual(self.parser.parse("9"), "9") +### fails with "malloc(): smallbin double linked list corrupted" +# def test_failure(self): +# self.assertEqual(self.parser.parse("5"), None) +# self.assertEqual(self.parser.parse("a"), None) + +class TestMany(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse(""), []) + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + pass + +class TestMany1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("b"), ["b"]) + self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) + +class TestRepeatN(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + def test_success(self): + self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) + +class TestOptional(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + def test_success(self): + self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) + self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) + self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + def test_failure(self): + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) + +class TestIgnore(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + def test_success(self): + self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + def test_failure(self): + self.assertEqual(self.parser.parse("ac"), None) + +class TestSepBy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(self.parser.parse(""), []) + def test_failure(self): + pass + +class TestSepBy1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + def test_success(self): + self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) + self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) + self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) + self.assertEqual(self.parser.parse("3"), ["3"]) + def test_failure(self): + self.assertEqual(self.parser.parse(""), None) + +class TestEpsilonP1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + def test_failure(self): + pass + +class TestEpsilonP2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +class TestEpsilonP3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + def test_failure(self): + pass + +# this has a double-free problem +#class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("ab"), None) + +class TestAnd1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + def test_success(self): + self.assertEqual(self.parser.parse("0"), ["0"]) + def test_failure(self): + pass + +class TestAnd2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + def test_success(self): + pass + def test_failure(self): + self.assertEqual(self.parser.parse("0"), None) + +class TestAnd3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + def test_success(self): + self.assertEqual(self.parser.parse("12"), ["1"]) + def test_failure(self): + pass + +class TestNot1(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + def test_failure(self): + self.assertEqual(self.parser.parse("a++b"), None) + +class TestNot2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + def test_success(self): + self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) + self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + def test_failure(self): + pass + +class TestLeftrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + def test_success(self): + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), ["a", "a"]) + self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + def test_failure(self): + pass + +class TestRightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.indirect() + a = h.ch("a") + h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + def test_success(self): + self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) + self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + def test_failure(self): + pass + +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.indirect() +# d = h.ch("d") +# p = h.ch("+") +# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(self.parser.parse("d"), ["d"]) +# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(self.parser.parse("d+"), None) + From 49f22e2afffe70c93f5cc8ecc86b5a8039f5dac5 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 07:56:47 +0100 Subject: [PATCH 23/44] Import combinator and allocator APIs into SWIG. Next is glue. Need to fix unions in HParsedToken and HCaseResult. --- src/allocator.h | 4 ++++ src/bindings/swig/hammer.i | 4 ++++ src/hammer.h | 20 +++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/bindings/swig/hammer.i diff --git a/src/allocator.h b/src/allocator.h index 2dfc14e..a02d97c 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -29,7 +29,11 @@ typedef struct HAllocator_ { typedef struct HArena_ HArena ; // hidden implementation HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default... +#ifndef SWIG void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) )); +#else +void* h_arena_malloc(HArena *arena, size_t count); +#endif void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers. void h_delete_arena(HArena *arena); diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i new file mode 100644 index 0000000..95870e2 --- /dev/null +++ b/src/bindings/swig/hammer.i @@ -0,0 +1,4 @@ +%module hammer + +%import "hammer/allocator.h" +%import "hammer/hammer.h" diff --git a/src/hammer.h b/src/hammer.h index 08819f5..02e4085 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -29,7 +29,9 @@ #define BIT_LITTLE_ENDIAN 0x0 #define BYTE_LITTLE_ENDIAN 0x0 +#ifndef HAMMER_INTERNAL__NO_STDARG_H typedef int bool; +#endif // HAMMER_INTERNAL__NO_STDARG_H typedef struct HParseState_ HParseState; @@ -177,7 +179,7 @@ typedef struct HBenchmarkResults_ { rtype_t name(__VA_ARGS__) attr; \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr -#ifndef HAMMER_INTERNAL__NO_STDARG_H +#ifndef SWIG #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ rtype_t name(__VA_ARGS__, ...); \ rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ @@ -195,17 +197,17 @@ typedef struct HBenchmarkResults_ { rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #else -#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...); \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) // Note: this drops the attributes on the floor for the __v versions -#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \ - rtype_t name(__VA_ARGS__, ...) attr; \ - rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \ - rtype_t name##__a(void *args[]); \ +#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \ + rtype_t name(params, ...); \ + rtype_t name##__m(HAllocator* mm__, params, ...); \ + rtype_t name##__a(void *args[]); \ rtype_t name##__ma(HAllocator *mm__, void *args[]) #endif // HAMMER_INTERNAL__NO_STDARG_H // }}} From 8f427b5a6d4d9b8b36552c51f20ff1cebd3a88d2 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:12:29 +0100 Subject: [PATCH 24/44] fixed nested unions --- src/bindings/swig/hammer.i | 1 + src/hammer.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 95870e2..81274c3 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -2,3 +2,4 @@ %import "hammer/allocator.h" %import "hammer/hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index 02e4085..c8a2bf3 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -70,8 +70,21 @@ typedef struct HBytes_ { size_t len; } HBytes; +#ifdef SWIG +typedef union { + HBytes bytes; + int64_t sint; + uint64_t uint; + double dbl; + float flt; + HCountedArray *seq; + void *user; +} HTokenData; +#endif + typedef struct HParsedToken_ { HTokenType token_type; +#ifndef SWIG union { HBytes bytes; int64_t sint; @@ -81,6 +94,9 @@ typedef struct HParsedToken_ { HCountedArray *seq; // a sequence of HParsedToken's void *user; }; +#else + HTokenData token_data; +#endif size_t index; char bit_offset; } HParsedToken; @@ -144,12 +160,23 @@ typedef struct HParserTestcase_ { char* output_unambiguous; } HParserTestcase; +#ifdef SWIG +typedef union { + const char* actual_results; + size_t parse_time; +} HResultTiming; +#endif + typedef struct HCaseResult_ { bool success; +#ifndef SWIG union { const char* actual_results; // on failure, filled in with the results of h_write_result_unamb size_t parse_time; // on success, filled in with time for a single parse, in nsec }; +#else + HResultTiming timestamp; +#endif } HCaseResult; typedef struct HBackendResults_ { From 0e4253d0efc2f13b43e35f28e51e68def7a4b763 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 08:15:01 +0100 Subject: [PATCH 25/44] invoke swig with relative path instead, -I../../ --- src/bindings/swig/hammer.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81274c3..357018b 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,5 @@ %module hammer -%import "hammer/allocator.h" -%import "hammer/hammer.h" +%import "allocator.h" +%import "hammer.h" From 73eabc7bf9e5eaccaa3175a16c331d7c697c01c8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sat, 16 Nov 2013 20:24:05 +0100 Subject: [PATCH 26/44] SWIG bindings build, compile, and appear to work for python. Will port over tests next. --- src/bindings/swig/hammer.i | 10 ++++++++-- src/hammer.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 357018b..ac8c47f 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,11 @@ %module hammer -%import "allocator.h" -%import "hammer.h" +%{ +#include "allocator.h" +#include "hammer.h" +#include "internal.h" +%} +%include "allocator.h" +%include "hammer.h" + diff --git a/src/hammer.h b/src/hammer.h index c8a2bf3..2aded7a 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -660,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p, void* userdata); // {{{ Benchmark functions HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases); void h_benchmark_report(FILE* stream, HBenchmarkResults* results); -void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); +//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results); // }}} // {{{ Token type registry From ada7bf89ee61d12b2f624cb6e82168f78d647379 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 15:55:38 -0600 Subject: [PATCH 27/44] There's a typemap problem between const uint8_t* and target-language strings, but I can create parsers in python and php. --- src/bindings/swig/hammer.i | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ac8c47f..f784f31 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,9 @@ %module hammer +%include "typemaps.i" +%include "stdint.i" + + // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ #include "allocator.h" #include "hammer.h" @@ -8,4 +12,5 @@ %include "allocator.h" %include "hammer.h" +%apply const char* { const uint8_t* } From 77d48e2b7c939e9066e8a899cce47aaeec0c7ed1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:26:51 -0600 Subject: [PATCH 28/44] SWIG python bindings, has same typemap problem as PHP --- src/SConscript | 2 +- src/bindings/python/SConscript | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/bindings/python/SConscript diff --git a/src/SConscript b/src/SConscript index 1d9ca76..0cb41af 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -bindings = [] +bindings = ['python'] dist_headers = [ "hammer.h", diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript new file mode 100644 index 0000000..29d1ad9 --- /dev/null +++ b/src/bindings/python/SConscript @@ -0,0 +1,14 @@ +# -*- python -*- +Import('env') + +pythonenv = env.Clone() + +pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) +pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) + +pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) + +swig = ['hammer.i'] + +libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file From dc9124724af7ffe4b837e6777c96080fe2d0b4d7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:31:10 -0600 Subject: [PATCH 29/44] fix travis config here too --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 246a3d6..f23f01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c compiler: - gcc - clang +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq swig python-dev script: - scons notifications: From 27f94dbe610e7c30573c741a3a1be09b360e95a7 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:41:56 -0600 Subject: [PATCH 30/44] helps to link against the library... --- src/bindings/python/SConscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 29d1ad9..2b1c4cc 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -5,6 +5,8 @@ pythonenv = env.Clone() pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) From 4bd2fc9e72f4149b9398d827c15ed3d4e7eb2a33 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Sun, 17 Nov 2013 20:56:03 -0600 Subject: [PATCH 31/44] sync with php-bindings SConscript --- src/bindings/python/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 2b1c4cc..a08e55f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,7 +1,7 @@ # -*- python -*- Import('env') -pythonenv = env.Clone() +pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) @@ -13,4 +13,4 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) \ No newline at end of file +libhammer_python = pythonenv.SharedLibrary('hammer', swig) From eba8ecc6c39f38efaebd76aba206cc98f5936164 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 17:19:46 -0600 Subject: [PATCH 32/44] typemap to fix conversion from python strings to uint8_t* --- src/bindings/swig/hammer.i | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f784f31..bef82b9 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,7 +1,16 @@ %module hammer -%include "typemaps.i" %include "stdint.i" +%include "typemaps.i" +%apply char [ANY] { uint8_t [ANY] }; + +#if defined(SWIGPYTHON) +%typemap(in) uint8_t* { + $1 = (uint8_t*)PyString_AsString($input); + } +#else + #warning no "in" typemap defined +#endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. %{ @@ -12,5 +21,5 @@ %include "allocator.h" %include "hammer.h" -%apply const char* { const uint8_t* } + From 073d6d8a275ad4826b0f6a301f63b45920ca6efb Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:14:44 -0600 Subject: [PATCH 33/44] python tests baked into scons; h_ch needs the first char of its input string as its input --- src/bindings/python/SConscript | 5 +- src/bindings/python/hammer_tests.py | 444 ++++++++++++++-------------- src/bindings/swig/hammer.i | 5 +- 3 files changed, 226 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index a08e55f..318103e 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -13,4 +13,7 @@ pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig) +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') + +pytestenv = pythonenv.Clone() +pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a56d669..3f0596c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,503 +4,495 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(h.h_parse(self.parser, "95", 2), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_int = h.h_ch(0xa2) + cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("b"), "b") + cls.parser = h.h_ch_range("a", "c") + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int64() + cls.parser = h.h_int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int32() + cls.parser = h.h_int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int16() + cls.parser = h.h_int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int8() + cls.parser = h.h_int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint64() + cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint32() + cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint16() + cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.uint8() + cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.int_range(h.uint8(), 3, 10) + cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(h.h_parse(self.parser, "_a", 2), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.end_p()) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse(""), "") -# self.assertEqual(self.parser.parse(" "), "") + cls.parser = h.h_whitespace(h.h_end_p()) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(h.h_parse(self.parser, " x", 3), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, "ba", 2), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, " ", 1), None) + self.assertEqual(h.h_parse(self.parser, " a", 2), None) + self.assertEqual(h.h_parse(self.parser, "a ", 2), None) + self.assertEqual(h.h_parse(self.parser, " b ", 3), None) + self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) + self.assertEqual(h.h_parse(self.parser, " ab", 3), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) -### fails with "corrupted double-linked list" -# def test_success(self): -# self.assertEqual(self.parser.parse("ab"), ["A", "B"]) -# self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(h.h_parse(self.parser, "d", 1), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.nothing_p() + cls.parser = h.h_nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "b", 1), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) - self.assertEqual(self.parser.parse("a b"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(h.h_parse(self.parser, "a c", 4), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(h.h_parse(self.parser, "c", 1), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) -### fails with malloc() memory corruption -#class TestButNotRange(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) -# def test_success(self): -# self.assertEqual(self.parser.parse("4"), "4") -### this segfaults -# def test_failure(self): -# self.assertEqual(self.parser.parse("6"), None) +class TestButNotRange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "6", 1), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) -#class TestXor(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) -### this segfaults -# def test_success(self): -# self.assertEqual(self.parser.parse("0"), "0") -# self.assertEqual(self.parser.parse("9"), "9") -### fails with "malloc(): smallbin double linked list corrupted" -# def test_failure(self): -# self.assertEqual(self.parser.parse("5"), None) -# self.assertEqual(self.parser.parse("a"), None) +class TestXor(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "5", 1), None) + self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), []) - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("b"), ["b"]) - self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) + self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(h.h_parse(self.parser, "adef", 4), None) + self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"]) - self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"]) - self.assertEqual(self.parser.parse("ad"), ["a", None, "d"]) + self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) + self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) + self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "aed", 3), None) + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ["a", "c"]) + self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(h.h_parse(self.parser, "ac", 2), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) - self.assertEqual(self.parser.parse(""), []) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"]) - self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"]) - self.assertEqual(self.parser.parse("1,3"), ["1", "3"]) - self.assertEqual(self.parser.parse("3"), ["3"]) + self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) + self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) + self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) + self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(h.h_parse(self.parser, "", 0), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) def test_failure(self): pass -# this has a double-free problem -#class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# self.assertEqual(self.parser.parse("bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ["0"]) + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ["1"]) + self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"]) - self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass class TestLeftrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), ["a", "a"]) - self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) def test_failure(self): pass class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.indirect() - a = h.ch("a") - h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p())) + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ["a"]) - self.assertEqual(self.parser.parse("aa"), ["a", ["a"]]) - self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]]) + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) + self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) def test_failure(self): pass -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.indirect() -# d = h.ch("d") -# p = h.ch("+") -# h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(self.parser.parse("d"), ["d"]) -# self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(self.parser.parse("d+"), None) +class TestAmbiguous(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + d = h.h_ch("d") + p = h.h_ch("+") + h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) + # this is supposed to be flattened + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) + self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) + self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index bef82b9..ad61e49 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -8,8 +8,11 @@ %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } +%typemap(out) uint8_t* { + $result = PyString_FromString((char*)$1); + } #else - #warning no "in" typemap defined + #warning no uint8_t* typemaps defined #endif // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind. From f685f9ea4ed8ad14178ad7f48b5237171c94162b Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Mon, 18 Nov 2013 21:50:28 -0600 Subject: [PATCH 34/44] there's the uint8_t problem (mostly) sorted --- src/bindings/python/hammer_tests.py | 10 +++++----- src/bindings/swig/hammer.i | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 3f0596c..d1c18db 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes.token, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95", 2), None) @@ -226,7 +226,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["a", "b"]) + self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -255,7 +255,7 @@ class TestChoice(unittest.TestCase): class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") @@ -439,7 +439,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) def test_failure(self): @@ -448,7 +448,7 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++")), h.h_ch("b")) + cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index ad61e49..412743a 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -11,6 +11,20 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) uint8_t { + if (PyInt_Check($input)) { + $1 = PyInt_AsLong($input); + } + else if (!PyString_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Expecting a string"); + return NULL; + } else { + $1 = *(uint8_t*)PyString_AsString($input); + } + } +%typemap(out) HBytes* { + $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + } #else #warning no uint8_t* typemaps defined #endif @@ -24,5 +38,3 @@ %include "allocator.h" %include "hammer.h" - - From 3228a86b90682115decf12be64338be3582283b3 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 00:18:25 -0600 Subject: [PATCH 35/44] h_sequence, h_choice and HCountedSequence are all having weird issues, but tests are in a workable format now --- src/bindings/python/hammer_tests.py | 157 ++++++++++++++-------------- src/bindings/swig/hammer.i | 19 +++- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index d1c18db..ee797d9 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.bytes, "\xa2") + self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a", 2), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, " ", 1), None) @@ -190,7 +190,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d", 1), None) @@ -208,8 +208,9 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): + ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -226,7 +227,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) self.assertEqual(h.h_parse(self.parser, "b", 1), None) @@ -236,9 +237,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 3).ast.token_data.seq, ["a", "b"]) - self.assertEqual(h.h_parse(self.parser, "a b", 4).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c", 4), None) @@ -247,8 +248,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, "a") - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.bytes, "b") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c", 1), None) @@ -257,8 +258,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.bytes, "a") + self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -267,7 +268,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.bytes, "4") + self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6", 1), None) @@ -285,8 +286,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.bytes, "0") - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.bytes, "9") + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5", 1), None) self.assertEqual(h.h_parse(self.parser, "a", 1), None) @@ -297,9 +298,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass @@ -308,9 +309,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.seq, ["b"]) - self.assertEqual(h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq, ["a", "a", "b", "b", "a", "b", "a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) @@ -330,9 +331,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd", 3).ast.token_data.seq, ["a", "b", "d"]) - self.assertEqual(h.h_parse(self.parser, "acd", 3).ast.token_data.seq, ["a", "c", "d"]) - self.assertEqual(h.h_parse(self.parser, "ad", 2).ast.token_data.seq, ["a", None, "d"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -343,7 +344,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc", 3).ast.token_data.seq, ["a", "c"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac", 2), None) @@ -352,10 +353,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) def test_failure(self): pass @@ -365,10 +366,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq, ["1", "2", "3"]) - self.assertEqual(h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq, ["1", "3", "2"]) - self.assertEqual(h.h_parse(self.parser, "1,3", 3).ast.token_data.seq, ["1", "3"]) - self.assertEqual(h.h_parse(self.parser, "3", 1).ast.token_data.seq, ["3"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) @@ -377,7 +378,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass @@ -386,7 +387,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -395,7 +396,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass @@ -414,6 +415,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) def test_success(self): + ### failing: [] != ["0"]. Token type is sequence. self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) def test_failure(self): pass @@ -425,14 +427,15 @@ class TestAnd2(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + ### failing: [] is not None, parse should have failed + self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12", 2).ast.token_data.seq, ["1"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass @@ -441,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", "+", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) @@ -450,23 +453,24 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b", 3).ast.token_data.seq, ["a", ["+"], "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) def test_failure(self): pass -class TestLeftrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) - def test_failure(self): - pass +### this is commented out for packrat in C ... +#class TestLeftrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# def test_failure(self): +# pass class TestRightrec(unittest.TestCase): @classmethod @@ -475,24 +479,25 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.seq, ["a"]) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", ["a"]]) - self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", ["a", ["a"]]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) def test_failure(self): pass -class TestAmbiguous(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - d = h.h_ch("d") - p = h.h_ch("+") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) - # this is supposed to be flattened - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) - self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) - self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +### this is just for GLR +#class TestAmbiguous(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# d = h.h_ch("d") +# p = h.h_ch("+") +# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # this is supposed to be flattened +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 412743a..f97985d 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,11 @@ %module hammer %include "stdint.i" -%include "typemaps.i" -%apply char [ANY] { uint8_t [ANY] }; + //%include "typemaps.i" + //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) +%ignore HCountedArray_; %typemap(in) uint8_t* { $1 = (uint8_t*)PyString_AsString($input); } @@ -22,9 +23,23 @@ $1 = *(uint8_t*)PyString_AsString($input); } } +/* +%typemap(out) uint8_t { + $result = PyString_FromString(&$1); + } +*/ %typemap(out) HBytes* { $result = PyString_FromStringAndSize((char*)$1->token, $1->len); } +%typemap(out) struct HCountedArray_* { + int i; + $result = PyList_New($1->used); + for (i=0; i<$1->used; i++) { + HParsedToken *t = $1->elements[i]; + PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0); + PyList_SetItem($result, i, o); + } + } #else #warning no uint8_t* typemaps defined #endif From 96760f1dafe792105cf18cc3dae1ef03b21fae39 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 02:41:45 -0600 Subject: [PATCH 36/44] kindasorta working. all tests (except h_action and h_attr_bool, need typemaps) have passed at one time or another, but some segfault at not-quite-random; h_and, h_epsilon_p, h_end_p, h_ignore, h_not, maybe h_choice seem culprity. --- src/bindings/python/SConscript | 4 +- src/bindings/python/hammer_tests.py | 179 ++++++++++++++-------------- src/bindings/swig/hammer.i | 26 +++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 318103e..718f8b9 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -16,4 +16,6 @@ swig = ['hammer.i'] libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests $SOURCE") +pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") + +Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee797d9..c776520 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -175,15 +175,15 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) self.assertEqual(h.h_parse(self.parser, " ab", 3), None) -class TestAction(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence(h.h_choice(h.h_ch("a"), h.h_ch("A")), h.h_choice(h.h_ch("b"), h.h_ch("B"))), lambda x: [y.upper() for y in x]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +# class TestAction(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) +# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) class TestIn(unittest.TestCase): @classmethod @@ -199,18 +199,17 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc", 3) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.bytes, "d") # segfaulting when looking at bytes! + self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a", 1), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_end_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - ### failing: parses a single 'a', dunno why self.assertEqual(h.h_parse(self.parser, "aa", 2), None) class TestNothingP(unittest.TestCase): @@ -225,7 +224,7 @@ class TestNothingP(unittest.TestCase): class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -235,7 +234,7 @@ class TestSequence(unittest.TestCase): class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_whitespace(h.h_ch("b"))) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) @@ -246,7 +245,7 @@ class TestSequenceWhitespace(unittest.TestCase): class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice(h.h_ch("a"), h.h_ch("b")) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) @@ -295,7 +294,7 @@ class TestXor(unittest.TestCase): class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) @@ -307,7 +306,7 @@ class TestMany(unittest.TestCase): class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) @@ -319,9 +318,9 @@ class TestMany1(unittest.TestCase): class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice(h.h_ch("a"), h.h_ch("b")), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef", 5).ast.token_data.seq, ["a", "b"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef", 4), None) self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) @@ -329,11 +328,12 @@ class TestRepeatN(unittest.TestCase): class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_optional(h.h_choice(h.h_ch("b"), h.h_ch("c"))), h.h_ch("d")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq if x is not None], [ord(y)["a", "d"]]) + ### FIXME check this out in repl, what does tree look like + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed", 3), None) self.assertEqual(h.h_parse(self.parser, "ab", 2), None) @@ -342,7 +342,7 @@ class TestOptional(unittest.TestCase): class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): @@ -351,7 +351,7 @@ class TestIgnore(unittest.TestCase): class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -364,7 +364,7 @@ class TestSepBy(unittest.TestCase): class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice(h.h_ch("1"), h.h_ch("2"), h.h_ch("3")), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) @@ -373,10 +373,11 @@ class TestSepBy1(unittest.TestCase): def test_failure(self): self.assertEqual(h.h_parse(self.parser, "", 0), None) +### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): @@ -385,7 +386,7 @@ class TestEpsilonP1(unittest.TestCase): class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_epsilon_p(), h.h_ch("a")) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): @@ -394,46 +395,44 @@ class TestEpsilonP2(unittest.TestCase): class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_epsilon_p()) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass -class TestAttrBool(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice(h.h_ch("a"), h.h_ch("b"))), lambda x: x[0] == x[1]) - def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) - self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) - def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# class TestAttrBool(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# def test_success(self): +# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# def test_failure(self): +# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("0")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) def test_success(self): - ### failing: [] != ["0"]. Token type is sequence. - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, ["0"]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_and(h.h_ch("0")), h.h_ch("1")) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) def test_success(self): pass def test_failure(self): - ### failing: [] is not None, parse should have failed - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.seq, None) + self.assertEqual(h.h_parse(self.parser, "0", 1), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("1"), h.h_and(h.h_ch("2"))) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): @@ -442,7 +441,7 @@ class TestAnd3(unittest.TestCase): class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_ch("+"), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): @@ -451,53 +450,59 @@ class TestNot1(unittest.TestCase): class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence(h.h_ch("a"), h.h_choice(h.h_sequence(h.h_ch("+"), h.h_not(h.h_ch("+"))), h.h_token("++", 2)), h.h_ch("b")) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], ["a", ["+"], "b"]) - self.assertEqual(h.h_parse(self.parser, "a++b", 4).ast.token_data.seq, ["a", "++", "b"]) + tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree[1] = tree[1].token_data.seq[0] + self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) + tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree[0] = chr(tree[0].token_data.uint) + tree[1] = tree[1].token_data.bytes + tree[2] = chr(tree[2].token_data.uint) + self.assertEqual(tree, ["a", "++", "b"]) def test_failure(self): pass -### this is commented out for packrat in C ... -#class TestLeftrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) -# def test_failure(self): -# pass +# ### this is commented out for packrat in C ... +# #class TestLeftrec(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # a = h.h_ch("a") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # def test_failure(self): +# # pass -class TestRightrec(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(a, cls.parser), h.h_epsilon_p())) - def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) - def test_failure(self): - pass +# class TestRightrec(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls.parser = h.h_indirect() +# a = h.h_ch("a") +# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) +# def test_success(self): +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) +# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) +# def test_failure(self): +# pass -### this is just for GLR -#class TestAmbiguous(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# d = h.h_ch("d") -# p = h.h_ch("+") -# h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) -# # this is supposed to be flattened -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# ### this is just for GLR +# #class TestAmbiguous(unittest.TestCase): +# # @classmethod +# # def setUpClass(cls): +# # cls.parser = h.h_indirect() +# # d = h.h_ch("d") +# # p = h.h_ch("+") +# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # # this is supposed to be flattened +# # def test_success(self): +# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # def test_failure(self): +# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index f97985d..81a9dd1 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,5 @@ %module hammer +%nodefaultctor; %include "stdint.i" //%include "typemaps.i" @@ -7,11 +8,31 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; %typemap(in) uint8_t* { + Py_INCREF($input); $1 = (uint8_t*)PyString_AsString($input); } %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } +%typemap(in) void*[] { + if (PyList_Check($input)) { + Py_INCREF($input); + int size = PyList_Size($input); + int i = 0; + int res = 0; + $1 = (void**)malloc(size*sizeof(HParser*)); + for (i=0; itoken, $1->len); } From 0aac5f4622d52b9df8f01769a6a6d81a63baceb1 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 03:45:49 -0600 Subject: [PATCH 37/44] travis needs python-nose to run nosetests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f23f01c..0e406ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang before_install: - sudo apt-get update -qq - - sudo apt-get install -qq swig python-dev + - sudo apt-get install -qq swig python-dev python-nose script: - scons notifications: From 12bb5cac36c5225d09bdd029147ea46eb1337133 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 17:26:01 -0600 Subject: [PATCH 38/44] more verbose tests; should run w/o hammer installed system-wide --- src/bindings/python/SConscript | 12 +++++++----- src/bindings/python/hammer_tests.py | 30 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 718f8b9..30d387c 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,21 +1,23 @@ # -*- python -*- -Import('env') +import os.path +Import('env libhammer_shared') pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -pythonenv.Append(LIBS = ['hammer']) -pythonenv.Append(LIBPATH = ['../../']) +#pythonenv.Append(LIBS = ['hammer']) +#pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() -pytestenv.Command(None, 'hammer_tests.py', "nosetests -v $SOURCE") +pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) +pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index c776520..9d0c487 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,18 +477,24 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -# class TestRightrec(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_indirect() -# a = h.h_ch("a") -# h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p()])) -# def test_success(self): -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aa", 2).ast.token_data.seq], ["a", ["a"]]) -# self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aaa", 3).ast.token_data.seq], ["a", ["a", ["a"]]]) -# def test_failure(self): -# pass +class TestARightrec(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_indirect() + a = h.h_ch("a") + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + def test_success(self): + tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + self.assertEqual(tree[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + def test_failure(self): + pass # ### this is just for GLR # #class TestAmbiguous(unittest.TestCase): From 8b6d6084c5ed15630d19a059e037525193835ef8 Mon Sep 17 00:00:00 2001 From: "Meredith L. Patterson" Date: Tue, 19 Nov 2013 19:00:58 -0600 Subject: [PATCH 39/44] fixed the segfault! hand-initialized HParser needed PB_MIN set. --- src/bindings/python/SConscript | 6 +++--- src/bindings/python/hammer_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 30d387c..0bde353 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -6,15 +6,15 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) pythonenv.Append(CPPPATH = ['../../', '/usr/include/python2.7']) pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error']) -#pythonenv.Append(LIBS = ['hammer']) -#pythonenv.Append(LIBPATH = ['../../']) +pythonenv.Append(LIBS = ['hammer']) +pythonenv.Append(LIBPATH = ['../../']) pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python']) pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) swig = ['hammer.i'] -libhammer_python = pythonenv.SharedLibrary('hammer', libhammer_shared + swig, SHLIBPREFIX='_') +libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 9d0c487..41cd13f 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -477,7 +477,7 @@ class TestNot2(unittest.TestCase): # # def test_failure(self): # # pass -class TestARightrec(unittest.TestCase): +class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_indirect() @@ -486,13 +486,13 @@ class TestARightrec(unittest.TestCase): def test_success(self): tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].uint, ord("a")) + self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) def test_failure(self): pass From 954b6e32d3757fd8d1cf7bffc76ea4018ebcbb4c Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Fri, 22 Nov 2013 19:42:02 -0600 Subject: [PATCH 40/44] Changed typemap for strings to also parse lengths --- src/bindings/python/hammer_tests.py | 341 ++++++++++++++-------------- src/bindings/swig/hammer.i | 25 +- 2 files changed, 195 insertions(+), 171 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 41cd13f..a5967e6 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,11 +4,11 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2", 3) + cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2", 3).ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95", 2), None) + self.assertEqual(h.h_parse(self.parser, "95"), None) class TestChParser(unittest.TestCase): @classmethod @@ -16,201 +16,202 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2", 1).ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2", 1).ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3", 1), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3", 1), None) + self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) + self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00", 8).ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00", 4).ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00", 3), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00", 2).ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe", 1), None) - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\xfe"), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88", 1).ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00", 8).ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00", 7), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00", 4).ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00", 3), None) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00", 2).ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78", 1).ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05", 1).ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b", 1), None) + self.assertEqual(h.h_parse(self.parser, "\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a", 3).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a", 2), None) + self.assertEqual(h.h_parse(self.parser, "_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ", 2).ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x", 3), None) + self.assertEqual(h.h_parse(self.parser, " x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, "ba", 2), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, "ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ", 3).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, " ", 1), None) - self.assertEqual(h.h_parse(self.parser, " a", 2), None) - self.assertEqual(h.h_parse(self.parser, "a ", 2), None) - self.assertEqual(h.h_parse(self.parser, " b ", 3), None) - self.assertEqual(h.h_parse(self.parser, "ba ", 3), None) - self.assertEqual(h.h_parse(self.parser, " ab", 3), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, " "), None) + self.assertEqual(h.h_parse(self.parser, " a"), None) + self.assertEqual(h.h_parse(self.parser, "a "), None) + self.assertEqual(h.h_parse(self.parser, " b "), None) + self.assertEqual(h.h_parse(self.parser, "ba "), None) + self.assertEqual(h.h_parse(self.parser, " ab"), None) -# class TestAction(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A"), None]), h.h_choice__a([h.h_ch("b"), h.h_ch("B"), None]), None]), lambda x: [y.upper() for y in x]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.seq, ["A", "B"]) -# self.assertEqual(h.h_parse(self.parser, "AB", 2).ast.token_data.seq, ["A", "B"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "XX", 2), None) +@unittest.skip("Action not implemented yet") +class TestAction(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc", 3) + cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d", 1), None) + self.assertEqual(h.h_parse(self.parser, "d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc", 3) + cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa", 2), None) + self.assertEqual(h.h_parse(self.parser, "aa"), None) class TestNothingP(unittest.TestCase): @classmethod @@ -219,243 +220,243 @@ class TestNothingP(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) - self.assertEqual(h.h_parse(self.parser, "b", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(h.h_parse(self.parser, "b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b")), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 3).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b", 4).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c", 4), None) + self.assertEqual(h.h_parse(self.parser, "a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]) + cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b", 1).ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c", 1), None) + self.assertEqual(h.h_parse(self.parser, "c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab", 2)) + cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4", 1).ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6", 1), None) + self.assertEqual(h.h_parse(self.parser, "6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab", 2), h.h_ch("a")) + cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab", 2).ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0", 1).ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9", 1).ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) + self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5", 1), None) - self.assertEqual(h.h_parse(self.parser, "a", 1), None) + self.assertEqual(h.h_parse(self.parser, "5"), None) + self.assertEqual(h.h_parse(self.parser, "a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])) + cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b", 1).ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba", 7).ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef", 11), None) + self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None]), 2) + cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef", 5).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef", 4), None) - self.assertEqual(h.h_parse(self.parser, "dabdef", 5), None) + self.assertEqual(h.h_parse(self.parser, "adef"), None) + self.assertEqual(h.h_parse(self.parser, "dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c"), None])), h.h_ch("d"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd", 3).ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd", 3).ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad", 2).ast.token_data.seq], [ord(y)["a", None, "d"]]) + #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed", 3), None) - self.assertEqual(h.h_parse(self.parser, "ab", 2), None) - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "aed"), None) + self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc", 3).ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac", 2), None) + self.assertEqual(h.h_parse(self.parser, "ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "", 0).ast.token_data.seq, []) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3"), None]), h.h_ch(",")) + cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3", 5).ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2", 5).ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3", 3).ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3", 1).ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "", 0), None) + self.assertEqual(h.h_parse(self.parser, ""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab", 2).ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a"), None]) + cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a", 1).ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) def test_failure(self): pass # class TestAttrBool(unittest.TestCase): # @classmethod # def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b"), None])), lambda x: x[0] == x[1]) +# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb", 2).ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) # def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab", 2), None) +# self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0", 1).ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1"), None]) + cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0", 1), None) + self.assertEqual(h.h_parse(self.parser, "0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2")), None]) + cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12", 2).ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b", 3).ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b", 4), None) + self.assertEqual(h.h_parse(self.parser, "a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+")), None]), h.h_token("++", 2), None]), h.h_ch("b"), None]) + cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "a+b").ast.token_data.seq tree[1] = tree[1].token_data.seq[0] self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b", 4).ast.token_data.seq + tree = h.h_parse(self.parser, "a++b").ast.token_data.seq tree[0] = chr(tree[0].token_data.uint) tree[1] = tree[1].token_data.bytes tree[2] = chr(tree[2].token_data.uint) @@ -471,9 +472,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a", 1).ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa", 2).ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa", 3).ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") +# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -482,14 +483,14 @@ class TestRightrec(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_indirect() a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser, None]), h.h_epsilon_p(), None])) + h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a", 1).ast.token_data.seq + tree = h.h_parse(self.parser, "a").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa", 2).ast.token_data.seq + tree = h.h_parse(self.parser, "aa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa", 3).ast.token_data.seq + tree = h.h_parse(self.parser, "aaa").ast.token_data.seq self.assertEqual(tree[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) @@ -506,9 +507,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d", 1).ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d", 3).ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d", 5).ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+", 2), None) +# # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 81a9dd1..4fb30c5 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -14,13 +14,35 @@ %typemap(out) uint8_t* { $result = PyString_FromString((char*)$1); } + +%typemap(newfree) HParseResult* { + h_parse_result_free($1); + } + +%newobject h_parse +%delobject h_parse_result_free + + /* +%typemap(in) (uint8_t* str, size_t len) { + if (PyString_Check($input) || + PyUnicode_Check($input)) { + PyString_AsStringAndSize($input, (char**)&$1, &$2); + } else { + PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); + } + } +*/ +%apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} +%apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} +%apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); int size = PyList_Size($input); int i = 0; int res = 0; - $1 = (void**)malloc(size*sizeof(HParser*)); + $1 = (void**)malloc((size+1)*sizeof(HParser*)); for (i=0; i Date: Sat, 23 Nov 2013 16:40:57 -0600 Subject: [PATCH 41/44] Got rid of accessors in Python; made h_action work --- src/bindings/python/hammer.py | 488 ---------------------------- src/bindings/python/hammer_tests.py | 181 +++++------ src/bindings/swig/hammer.i | 134 ++++++-- 3 files changed, 194 insertions(+), 609 deletions(-) delete mode 100644 src/bindings/python/hammer.py diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py deleted file mode 100644 index 36b78c8..0000000 --- a/src/bindings/python/hammer.py +++ /dev/null @@ -1,488 +0,0 @@ -from cffi import FFI -import threading -import sys - -_ffi = FFI() - -# {{{ Types - -_ffi.cdef("typedef struct HAllocator_ HAllocator;") -_ffi.cdef("typedef struct HArena_ HArena;") -_ffi.cdef("typedef int bool;") -_ffi.cdef("typedef struct HParseState_ HParseState;") -_ffi.cdef(""" -typedef enum HParserBackend_ { - PB_MIN = 0, - PB_PACKRAT = 0, // PB_MIN is always the default. - PB_REGULAR, - PB_LLk, - PB_LALR, - PB_GLR -// TODO: support PB_MAX -} HParserBackend; -""") -_ffi.cdef(""" -typedef enum HTokenType_ { - // Before you change the explicit values of these, think of the poor bindings ;_; - TT_NONE = 1, - TT_BYTES = 2, - TT_SINT = 4, - TT_UINT = 8, - TT_SEQUENCE = 16, - TT_RESERVED_1, // reserved for backend-specific internal use - TT_ERR = 32, - TT_USER = 64, - TT_MAX -} HTokenType; -""") -_ffi.cdef(""" -typedef struct HCountedArray_ { - size_t capacity; - size_t used; - HArena * arena; - struct HParsedToken_ **elements; -} HCountedArray; -""") -_ffi.cdef(""" -typedef struct HBytes_ { - const uint8_t *token; - size_t len; -} HBytes; -""") -_ffi.cdef(""" -typedef struct HParsedToken_ { - HTokenType token_type; - union { - HBytes bytes; - int64_t sint; - uint64_t uint; - double dbl; - float flt; - HCountedArray *seq; // a sequence of HParsedToken's - void *user; - }; - size_t index; - char bit_offset; -} HParsedToken; -""") -_ffi.cdef(""" -typedef struct HParseResult_ { - const HParsedToken *ast; - long long bit_length; - HArena * arena; -} HParseResult; -""") - -_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""") -_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""") -_ffi.cdef(""" -typedef struct HCFChoice_ HCFChoice; -typedef struct HRVMProg_ HRVMProg; -typedef struct HParserVtable_ HParserVtable; -""") - -_ffi.cdef("typedef struct HParser_ HParser;") -_ffi.cdef(""" -typedef struct HParserTestcase_ { - unsigned char* input; - size_t length; - char* output_unambiguous; -} HParserTestcase; - -typedef struct HCaseResult_ { - bool success; - union { - const char* actual_results; // on failure, filled in with the results of h_write_result_unamb - size_t parse_time; // on success, filled in with time for a single parse, in nsec - }; -} HCaseResult; - -typedef struct HBackendResults_ { - HParserBackend backend; - bool compile_success; - size_t n_testcases; - size_t failed_testcases; // actually a count... - HCaseResult *cases; -} HBackendResults; - -typedef struct HBenchmarkResults_ { - size_t len; - HBackendResults *results; -} HBenchmarkResults; -""") - -# }}} -# {{{ Arena functions -_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);") -_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);") -# }}} -# {{{ cdefs -## The following section was generated by -## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/' -_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);") -_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);") -_ffi.cdef("HParser* h_ch(const uint8_t c);") -_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);") -_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);") -_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);") -_ffi.cdef("HParser* h_bits(size_t len, bool sign);") -_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);") -_ffi.cdef("HParser* h_int64(void);") -_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int32(void);") -_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int16(void);") -_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_int8(void);") -_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint64(void);") -_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint32(void);") -_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint16(void);") -_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_uint8(void);") -_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_whitespace(const HParser* p);") -_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);") -_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);") -_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);") -_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);") -_ffi.cdef("HParser* h_end_p(void);") -_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_nothing_p(void);") -_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_sequence(HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_sequence__a(void* args);") -_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_choice(HParser* p, ...);") -_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);") -_ffi.cdef("HParser* h_choice__a(void* args);") -_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);") -_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);") -_ffi.cdef("HParser* h_many(const HParser* p);") -_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_many1(const HParser* p);") -_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);") -_ffi.cdef("HParser* h_optional(const HParser* p);") -_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_ignore(const HParser* p);") -_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);") -_ffi.cdef("HParser* h_epsilon_p(void);") -_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);") -_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);") -_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);") -_ffi.cdef("HParser* h_and(const HParser* p);") -_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_not(const HParser* p);") -_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);") -_ffi.cdef("HParser* h_indirect(void);") -_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);") -_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);") -_ffi.cdef("void h_parse_result_free(HParseResult *result);") -_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);") -_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);") -_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);") -_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);") -_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);") - -_lib = _ffi.verify("#include ", - libraries=['hammer']) - -_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45 -# }}} -class _DynamicScopeHolder(threading.local): - """A dynamically-scoped holder of python objects, which may or may not - otherwise appear in the object graph. Intended for use with CFFI """ - def __init__(self): - self._ctxstack = [] - def __enter__(self): - self._ctxstack.append([]) - def __exit__(self, exc_type, exc_value, traceback): - self._ctxstack.pop() - return False - def stash(self, *objs): - if len(self._ctxstack) < 1: - raise Exception("Not in any dynamic scope") - for obj in objs: - self._ctxstack[-1].append(obj) -def _fromHParsedToken(cobj): - # TODO: Free the toplevel parser - tt = cobj.token_type - - if cobj.token_type == _lib.TT_BYTES: - return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:] - elif cobj.token_type == _lib.TT_ERR: - # I have no idea what this is for - pass - elif cobj.token_type == _lib.TT_NONE: - return None - elif cobj.token_type == _lib.TT_SEQUENCE: - return [_fromHParsedToken(cobj.seq.elements[i]) - for i in range(cobj.seq.used)] - elif cobj.token_type == _lib.TT_SINT: - return cobj.sint - elif cobj.token_type == _lib.TT_UINT: - return cobj.uint - elif cobj.token_type == _lib.TT_PYTHON: - return _ffi.from_handle(cobj.user) - -_parser_result_holder = _DynamicScopeHolder() -def _toHParsedToken(arena, pyobj): - if pyobj is None: - return _ffi.NULL - cobj = _ffi.new_handle(pyobj) - _parser_result_holder.stash(cobj) - - hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken"))) - hpt.token_type = _lib.TT_PYTHON - hpt.user = cobj - hpt.bit_offset = chr(127) - hpt.index = 0 - return hpt - -def _fromParseResult(cobj): - ret = _fromHParsedToken(cobj.ast) - _lib.h_parse_result_free(cobj) - return ret - -def _to_haction(fn): - """Turn a function that transforms a parsed value into an HAction""" - def action(parse_result): - res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result))) - if res != _ffi.NULL and parse_result.ast != _ffi.NULL: - res.index = parse_result.ast.index - res.bit_offset = parse_result.ast.bit_offset - return res - return _ffi.callback("HParsedToken*(HParseResult*)", action) - -def _to_hpredicate(fn): - """Turn a function that transforms a parsed value into an HAction""" - def predicate(parse_result): - res = fn(_fromParseResult(parse_result)) - # TODO: Handle exceptions; parse should fail. - if type(res) != bool: - raise TypeError("Predicates should return a bool") - return res - return _ffi.callback("bool(HParseResult*)", predicate) - -class Parser(object): - # TODO: Map these to individually garbage-collected blocks of - # memory. Perhaps with an arena allocator with block size of 1? - # There has to be something more efficient than that, though. - - # TODO: How do we handle encodings? By default, we're using UTF-8 - def __init__(self, internal, deps): - """Create a new parser from an FFI object. Not for user code""" - self._parser = internal - self._deps = deps - - def parse(self, string): - with _parser_result_holder: - pres = _lib.h_parse(self._parser, string, len(string)) - if pres: - return _fromParseResult(pres) - else: - return None - - def __mul__(self, count): - return repeat_n(self, count) - - - -class IndirectParser(Parser): - def bind(self, inner): - _lib.h_bind_indirect(self._parser, inner._parser) - self._deps = (inner,) - -class BitsParser(Parser): - pass - -def token(token): - # TODO: Does not clone argument. - if isinstance(token, unicode): - token = token.encode("utf-8") - return Parser(_lib.h_token(token, len(token)), ()) - -def ch(char): - """Returns either a token or an int, depending on the type of the - argument""" - if isinstance(char, int): - return Parser(_lib.h_ch(char), ()) - else: - return token(char) - -def ch_range(chr1, chr2): - if not isinstance(chr1, str) or not isinstance(chr2, str): - raise TypeError("ch_range can't handle unicode") - def my_action(pr): - # print "In action: ", pr - return pr - return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action) - -def int_range(parser, i1, i2): - if type(parser) != BitsParser: - raise TypeError("int_range is only valid when used with a bits parser") - return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,)) - -def bits(length, signedp): - return BitsParser(_lib.h_bits(length, signedp), ()) - -def int64(): return bits(64, True) -def int32(): return bits(32, True) -def int16(): return bits(16, True) -def int8 (): return bits(8, True) -def uint64(): return bits(64, False) -def uint32(): return bits(32, False) -def uint16(): return bits(16, False) -def uint8 (): return bits(8, False) - -def whitespace(p): - return Parser(_lib.h_whitespace(p._parser), (p,)) -def left(p1, p2): - return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2)) -def right(p1, p2): - return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2)) -def middle(p1, p2, p3): - return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3)) -def action(parser, action): - caction = _to_haction(action) - return Parser(_lib.h_action(parser._parser, caction), (parser, caction)) - -def in_(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_in(charset, len(charset)), ()) -def not_in(charset): - if not isinstance(charset, str): - # TODO/Python3: change str to bytes - raise TypeError("in_ can't deal with unicode") - return Parser(_lib.h_not_in(charset, len(charset)), ()) -def end_p(): - return Parser(_lib.h_end_p(), ()) -def nothing_p(): - return Parser(_lib.h_nothing_p(), ()) -def sequence(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_sequence(*plist), (plist,)) -def choice(*parsers): - plist = [p._parser for p in parsers] - plist.append(_ffi.NULL) - return Parser(_lib.h_choice(*plist), (plist,)) -def butnot(p1, p2): - return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2)) -def difference(p1, p2): - return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2)) -def xor(p1, p2): - return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2)) -def many(p1): - return Parser(_lib.h_many(p1._parser), (p1,)) -def many1(p1): - return Parser(_lib.h_many1(p1._parser), (p1,)) -def repeat_n(p1, n): - return Parser(_lib.h_repeat_n(p1._parser, n), (p1,)) -def optional(p1): - return Parser(_lib.h_optional(p1._parser), (p1,)) -def ignore(p1): - return Parser(_lib.h_ignore(p1._parser), (p1,)) -def sepBy(p, sep): - return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep)) -def sepBy1(p, sep): - return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep)) -def epsilon_p(): - return Parser(_lib.h_epsilon_p(), ()) -def length_value(p_len, p_value): - return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value)) -def attr_bool(parser, predicate): - cpredicate = _to_hpredicate(predicate) - return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate)) -def and_(parser): - return Parser(_lib.h_and(parser._parser), (parser,)) -def not_(parser): - return Parser(_lib.h_not(parser._parser), (parser,)) -def indirect(): - return IndirectParser(_lib.h_indirect(), ()) -def bind_indirect(indirect, inner): - indirect.bind(inner) - -def parse(parser): - return parser.parse() - -# Unfortunately, "in", "and", and "not" are keywords. This makes them -# show up in the module namespace for the use of automated tools. Do -# not attempt to use them by hand; only use the mangled forms (with -# the '_') -sys.modules[__name__].__dict__["in"] = in_ -sys.modules[__name__].__dict__["and"] = and_ -sys.modules[__name__].__dict__["not"] = not_ - -def run_test(): - p_test = sepBy1(choice(ch('1'), - ch('2'), - ch('3')), - ch(',')) - return p_test.parse("1,2,3") - -# {{{ Automatic parser construction... python specific - -# TODO: Implement Parsable metaclass, which requires the existence of -# a "parse" method. - -# This is expected to be extended by user code. As a general rule, -# only provide auto-parsers for your own types. -AUTO_PARSERS = { - str: token, - unicode: token, -} - -def _auto_seq(lst): - return sequence(*(auto_1(p, default_method=_auto_choice) - for p in lst)) - -def _auto_choice(lst): - return choice(*(auto_1(p, default_method=_auto_seq) - for p in lst)) - -def auto_1(arg, default_method=_auto_choice): - if isinstance(arg, Parser): - return arg - elif type(arg) in AUTO_PARSERS: - return AUTO_PARSERS[type(arg)](arg) - else: - return default_method(arg) - -def auto(*args): - return auto_1(args, default_method=_auto_choice) - -# }}} diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index a5967e6..82f4084 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,7 +6,7 @@ class TestTokenParser(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2").ast.token_data.bytes, "95\xa2") + self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") def test_partial_fails(self): self.assertEqual(h.h_parse(self.parser, "95"), None) @@ -16,8 +16,8 @@ class TestChParser(unittest.TestCase): cls.parser_int = h.h_ch(0xa2) cls.parser_chr = h.h_ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2").ast.token_data.uint, 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2").ast.token_data.uint, ord("\xa2")) + self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -36,7 +36,7 @@ class TestInt64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00").ast.token_data.sint, -0x200000000) + self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) @@ -45,8 +45,8 @@ class TestInt32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00").ast.token_data.sint, -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.sint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -56,8 +56,8 @@ class TestInt16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00").ast.token_data.sint, -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.sint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\xfe"), None) self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -67,7 +67,7 @@ class TestInt8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88").ast.token_data.sint, -0x78) + self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -76,7 +76,7 @@ class TestUint64(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00").ast.token_data.uint, 0x200000000) + self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) @@ -85,7 +85,7 @@ class TestUint32(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00").ast.token_data.uint, 0x20000) + self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) @@ -94,7 +94,7 @@ class TestUint16(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00").ast.token_data.uint, 0x200) + self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x02"), None) @@ -103,7 +103,7 @@ class TestUint8(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78").ast.token_data.uint, 0x78) + self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -112,7 +112,7 @@ class TestIntRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_int_range(h.h_uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05").ast.token_data.uint, 5) + self.assertEqual(h.h_parse(self.parser, "\x05"), 5) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "\x0b"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -133,8 +133,8 @@ class TestWhitespaceEnd(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast, None) # empty string - self.assertEqual(h.h_parse(self.parser, " ").ast, None) # empty string + self.assertEqual(h.h_parse(self.parser, ""), None) # empty string + self.assertEqual(h.h_parse(self.parser, " "), None) # empty string def test_failure(self): self.assertEqual(h.h_parse(self.parser, " x"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a ").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,14 +175,16 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -@unittest.skip("Action not implemented yet") +#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), lambda x: [y.upper() for y in x]) + cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), + h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + lambda x: [chr(y).upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.seq, ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB").ast.token_data.seq, ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) + self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "XX"), None) @@ -191,7 +193,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -200,7 +202,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.uint, ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -209,7 +211,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -227,7 +229,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -237,9 +239,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a b").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -248,8 +250,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "b").ast.token_data.uint, ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -258,8 +260,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.uint, ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -268,7 +270,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4").ast.token_data.uint, ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -277,7 +279,7 @@ class TestDifference(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab").ast.token_data.bytes, "ab") + self.assertEqual(h.h_parse(self.parser, "ab"), "ab") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -286,8 +288,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0").ast.token_data.uint, ord("0")) - self.assertEqual(h.h_parse(self.parser, "9").ast.token_data.uint, ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) + self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -297,10 +299,10 @@ class TestMany(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) def test_failure(self): pass @@ -309,9 +311,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "b").ast.token_data.seq], [ord(y) for y in ["b"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "aabbaba").ast.token_data.seq], [ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -321,7 +323,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abdef").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -331,10 +333,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abd").ast.token_data.seq], [ord(y) for y in ["a", "b", "d"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "acd").ast.token_data.seq], [ord(y) for y in ["a", "c", "d"]]) - ### FIXME check this out in repl, what does tree look like - #self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ad").ast.token_data.seq], [ord(y)["a", None, "d"]]) + self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) + self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -345,7 +346,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "abc").ast.token_data.seq], [ord(y) for y in ["a", "c"]]) + self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -354,11 +355,11 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) - self.assertEqual(h.h_parse(self.parser, "").ast.token_data.seq, []) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -367,10 +368,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,2,3").ast.token_data.seq], [ord(y) for y in ["1", "2", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3,2").ast.token_data.seq], [ord(y) for y in ["1", "3", "2"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "1,3").ast.token_data.seq], [ord(y) for y in ["1", "3"]]) - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "3").ast.token_data.seq], [ord(y) for y in ["3"]]) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) + self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) + self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -380,7 +381,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "ab").ast.token_data.seq], [ord(y) for y in ["a", "b"]]) + self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) def test_failure(self): pass @@ -389,7 +390,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -398,7 +399,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a").ast.token_data.seq], [ord(y) for y in ["a"]]) + self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) def test_failure(self): pass @@ -407,8 +408,8 @@ class TestEpsilonP3(unittest.TestCase): # def setUpClass(cls): # cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) # def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb").ast.token_data.seq, ["b", "b"]) +# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) # def test_failure(self): # self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -417,7 +418,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "0").ast.token_data.seq], [ord(y) for y in ["0"]]) + self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) def test_failure(self): pass @@ -435,7 +436,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "12").ast.token_data.seq], [ord(y) for y in ["1"]]) + self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) def test_failure(self): pass @@ -444,7 +445,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual([x.token_data.uint for x in h.h_parse(self.parser, "a+b").ast.token_data.seq], [ord(y) for y in ["a", "+", "b"]]) + self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -453,14 +454,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - tree = h.h_parse(self.parser, "a+b").ast.token_data.seq - tree[1] = tree[1].token_data.seq[0] - self.assertEqual([x.token_data.uint for x in tree], [ord(y) for y in ["a", "+", "b"]]) - tree = h.h_parse(self.parser, "a++b").ast.token_data.seq - tree[0] = chr(tree[0].token_data.uint) - tree[1] = tree[1].token_data.bytes - tree[2] = chr(tree[2].token_data.uint) - self.assertEqual(tree, ["a", "++", "b"]) + self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) def test_failure(self): pass @@ -472,9 +467,9 @@ class TestNot2(unittest.TestCase): # # a = h.h_ch("a") # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a").ast.token_data.bytes, "a") -# # self.assertEqual(h.h_parse(self.parser, "aa").ast.token_data.seq, ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa").ast.token_data.seq, ["a", "a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "a"), "a") +# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) +# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass @@ -485,15 +480,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - tree = h.h_parse(self.parser, "a").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - tree = h.h_parse(self.parser, "aaa").ast.token_data.seq - self.assertEqual(tree[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[0].token_data.uint, ord("a")) - self.assertEqual(tree[1].token_data.seq[1].token_data.seq[0].token_data.uint, ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) + self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) + self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) def test_failure(self): pass @@ -507,9 +496,9 @@ class TestRightrec(unittest.TestCase): # # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d").ast.token_data.seq, ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d").ast.token_data.seq, ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d").ast.token_data.seq, ["d", "+", "d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) +# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): # # self.assertEqual(h.h_parse(self.parser, "d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 4fb30c5..23bcc15 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,5 +1,6 @@ %module hammer %nodefaultctor; +//%nodefaultdtor; %include "stdint.i" //%include "typemaps.i" @@ -7,31 +8,6 @@ #if defined(SWIGPYTHON) %ignore HCountedArray_; -%typemap(in) uint8_t* { - Py_INCREF($input); - $1 = (uint8_t*)PyString_AsString($input); - } -%typemap(out) uint8_t* { - $result = PyString_FromString((char*)$1); - } - -%typemap(newfree) HParseResult* { - h_parse_result_free($1); - } - -%newobject h_parse -%delobject h_parse_result_free - - /* -%typemap(in) (uint8_t* str, size_t len) { - if (PyString_Check($input) || - PyUnicode_Check($input)) { - PyString_AsStringAndSize($input, (char**)&$1, &$2); - } else { - PyErr_SetString(PyExc_TypeError, "Argument must be a str or unicode"); - } - } -*/ %apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} @@ -79,6 +55,43 @@ PyList_SetItem($result, i, o); } } +%typemap(out) struct HParseResult_* { + if ($1 == NULL) { + // TODO: raise parse failure + Py_INCREF(Py_None); + $result = Py_None; + } else { + $result = hpt_to_python($1->ast); + } + } + +%inline %{ + static int h_tt_python; + %} +%init %{ + h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); + %} + +/* +%typemap(in) (HPredicate* pred, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_predicate; + } +*/ +%typemap(in) (const HAction a, void* user_data) { + Py_INCREF($input); + $2 = $input; + $1 = call_action; + } + +%inline { + struct HParsedToken_; + struct HParseResult_; + static PyObject* hpt_to_python(const struct HParsedToken_ *token); + + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); + } #else #warning no uint8_t* typemaps defined #endif @@ -92,3 +105,74 @@ %include "allocator.h" %include "hammer.h" + +%extend HArena_ { + ~HArena_() { + h_delete_arena($self); + } + }; +%extend HParseResult_ { + ~HParseResult_() { + h_parse_result_free($self); + } +}; + +%newobject h_parse; +%delobject h_parse_result_free; +%newobject h_new_arena; +%delobject h_delete_arena; + +#ifdef SWIGPYTHON +%inline { + static PyObject* hpt_to_python(const HParsedToken *token) { + // Caller holds a reference to returned object + PyObject *ret; + if (token == NULL) { + Py_RETURN_NONE; + } + switch (token->token_type) { + case TT_NONE: + Py_RETURN_NONE; + break; + case TT_BYTES: + return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); + case TT_SINT: + // TODO: return PyINT if appropriate + return PyLong_FromLong(token->token_data.sint); + case TT_UINT: + // TODO: return PyINT if appropriate + return PyLong_FromUnsignedLong(token->token_data.uint); + case TT_SEQUENCE: + ret = PyTuple_New(token->token_data.seq->used); + for (int i = 0; i < token->token_data.seq->used; i++) { + PyTuple_SET_ITEM(ret, i, hpt_to_python(token->token_data.seq->elements[i])); + } + return ret; + default: + if (token->token_type == h_tt_python) { + ret = (PyObject*)token->token_data.user; + Py_INCREF(ret); + return ret; + } else { + return SWIG_NewPointerObj((void*)token, SWIGTYPE_p_HParsedToken_, 0 | 0); + // TODO: support registry + } + + } + } + static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + if (ret == NULL) { + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + HParsedToken *tok = h_make(p->arena, h_tt_python, ret); + return tok; + } + + } +#endif From 294bca3fce98ead2a3383ab578e39c9dfde0d124 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sat, 23 Nov 2013 19:53:58 -0600 Subject: [PATCH 42/44] Cleaned up python bindings interface a bit; now returns strings rather than character codes if appropriate --- src/bindings/python/SConscript | 2 +- src/bindings/python/hammer_tests.py | 109 ++++++++++++++-------------- src/bindings/swig/hammer.i | 77 ++++++++++++++++++-- 3 files changed, 126 insertions(+), 62 deletions(-) diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index 0bde353..fd03a3f 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -18,6 +18,6 @@ libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_') pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) -pytestenv.Command(None, ['hammer_tests.py', libhammer_python], "nosetests -vv $SOURCE") +pytestenv.Command(None, ['hammer_tests.py'] + libhammer_python, "nosetests -vv $SOURCE") Clean('.', ['hammer.pyc', 'hammer_tests.py', 'hammer_tests.pyc']) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 82f4084..7a7d5f1 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -17,7 +17,7 @@ class TestChParser(unittest.TestCase): cls.parser_chr = h.h_ch("\xa2") def test_success(self): self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), ord("\xa2")) # TODO: interface change + self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") def test_failure(self): self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) @@ -27,7 +27,7 @@ class TestChRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -121,10 +121,10 @@ class TestWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_whitespace(h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "\ta"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(h.h_parse(self.parser, "\ta"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "_a"), None) @@ -143,7 +143,7 @@ class TestLeft(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -154,7 +154,7 @@ class TestRight(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -165,7 +165,7 @@ class TestMiddle(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), ord("a")) + self.assertEqual(h.h_parse(self.parser, " a "), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, " "), None) @@ -175,13 +175,12 @@ class TestMiddle(unittest.TestCase): self.assertEqual(h.h_parse(self.parser, "ba "), None) self.assertEqual(h.h_parse(self.parser, " ab"), None) -#@unittest.skip("Action not implemented yet") class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), - lambda x: [chr(y).upper() for y in x]) + lambda x: [y.upper() for y in x]) def test_success(self): self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) @@ -193,7 +192,7 @@ class TestIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "d"), None) @@ -202,7 +201,7 @@ class TestNotIn(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), ord("d")) + self.assertEqual(h.h_parse(self.parser, "d"), "d") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -211,7 +210,7 @@ class TestEndP(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aa"), None) @@ -229,7 +228,7 @@ class TestSequence(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord, "ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a"), None) self.assertEqual(h.h_parse(self.parser, "b"), None) @@ -239,9 +238,9 @@ class TestSequenceWhitespace(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) - self.assertEqual(h.h_parse(self.parser, "a b"), tuple(map(ord,"ab"))) + self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a c"), None) @@ -250,8 +249,8 @@ class TestChoice(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "b"), ord("b")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "b"), "b") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "c"), None) @@ -260,8 +259,8 @@ class TestButNot(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ord("a")) - self.assertEqual(h.h_parse(self.parser, "aa"), ord("a")) + self.assertEqual(h.h_parse(self.parser, "a"), "a") + self.assertEqual(h.h_parse(self.parser, "aa"), "a") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -270,7 +269,7 @@ class TestButNotRange(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), ord("4")) + self.assertEqual(h.h_parse(self.parser, "4"), "4") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "6"), None) @@ -288,8 +287,8 @@ class TestXor(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ord("0")) - self.assertEqual(h.h_parse(self.parser, "9"), ord("9")) + self.assertEqual(h.h_parse(self.parser, "0"), "0") + self.assertEqual(h.h_parse(self.parser, "9"), "9") def test_failure(self): self.assertEqual(h.h_parse(self.parser, "5"), None) self.assertEqual(h.h_parse(self.parser, "a"), None) @@ -300,9 +299,9 @@ class TestMany(unittest.TestCase): cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), tuple(map(ord, "a"))) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(map(ord, "b"))) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(map(ord, "aabbaba"))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass @@ -311,9 +310,9 @@ class TestMany1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) - self.assertEqual(h.h_parse(self.parser, "b"), tuple(ord(y) for y in ["b"])) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), tuple(ord(y) for y in ["a", "a", "b", "b", "a", "b", "a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) + self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) @@ -323,7 +322,7 @@ class TestRepeatN(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), (ord('a'), ord('b'))) + self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "adef"), None) self.assertEqual(h.h_parse(self.parser, "dabdef"), None) @@ -333,9 +332,9 @@ class TestOptional(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), (ord('a'),ord('b'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "acd"), (ord('a'),ord('c'),ord('d'))) - self.assertEqual(h.h_parse(self.parser, "ad"), (ord('a'),None,ord('d'))) + self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) + self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) + self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "aed"), None) self.assertEqual(h.h_parse(self.parser, "ab"), None) @@ -346,7 +345,7 @@ class TestIgnore(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), tuple(map(ord, "ac"))) + self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "ac"), None) @@ -355,10 +354,10 @@ class TestSepBy(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) self.assertEqual(h.h_parse(self.parser, ""), ()) def test_failure(self): pass @@ -368,10 +367,10 @@ class TestSepBy1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), tuple(map(ord, "123"))) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), tuple(map(ord, "132"))) - self.assertEqual(h.h_parse(self.parser, "1,3"), tuple(map(ord, "13"))) - self.assertEqual(h.h_parse(self.parser, "3"), (ord('3'),)) + self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) + self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) + self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) + self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) def test_failure(self): self.assertEqual(h.h_parse(self.parser, ""), None) @@ -381,7 +380,7 @@ class TestEpsilonP1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), tuple(ord(y) for y in ["a", "b"])) + self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) def test_failure(self): pass @@ -390,7 +389,7 @@ class TestEpsilonP2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -399,7 +398,7 @@ class TestEpsilonP3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), tuple(ord(y) for y in ["a"])) + self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) def test_failure(self): pass @@ -418,7 +417,7 @@ class TestAnd1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), (0x30,)) + self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) def test_failure(self): pass @@ -436,7 +435,7 @@ class TestAnd3(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), (0x31,)) + self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) def test_failure(self): pass @@ -445,7 +444,7 @@ class TestNot1(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), tuple(ord(y) for y in ["a", "+", "b"])) + self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) def test_failure(self): self.assertEqual(h.h_parse(self.parser, "a++b"), None) @@ -454,8 +453,8 @@ class TestNot2(unittest.TestCase): def setUpClass(cls): cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), (ord('a'), (ord('+'),), ord('b'))) - self.assertEqual(h.h_parse(self.parser, "a++b"), (ord('a'), "++", ord('b'))) + self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) + self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -480,9 +479,9 @@ class TestRightrec(unittest.TestCase): a = h.h_ch("a") h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), (ord('a'),)) - self.assertEqual(h.h_parse(self.parser, "aa"), (ord('a'), (ord('a'),))) - self.assertEqual(h.h_parse(self.parser, "aaa"), (ord('a'), (ord('a'), (ord('a'),)))) + self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) + self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) + self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 23bcc15..a2a8940 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,10 +1,8 @@ %module hammer + %nodefaultctor; -//%nodefaultdtor; %include "stdint.i" - //%include "typemaps.i" - //%apply char [ANY] { uint8_t [ANY] }; #if defined(SWIGPYTHON) %ignore HCountedArray_; @@ -12,6 +10,67 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)} %apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)} %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} + + +%rename(_h_ch) h_ch; +%pythoncode %{ + def h_ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return h_token(ch) + else: + return _h_ch(ch) +%} + +%rename(_h_ch_range) h_ch_range; +%pythoncode %{ + def h_ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return h_action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +%} + +%rename(_h_in) h_in; +%rename(_h_not_in) h_not_in; +%pythoncode %{ + def h_in(charset): + return h_action(_h_in(charset), chr) + def h_not_in(charset): + return h_action(_h_not_in(charset), chr) + %} + +%inline { + static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; + + static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + _helper_ParseError = parse_error; + _helper_Placeholder = placeholder; + } + } + +%pythoncode %{ + class Placeholder(object): + """The python equivalent of TT_NONE""" + def __str__(self): + return "Placeholder" + def __repr__(self): + return "Placeholder" + def __eq__(self, other): + return type(self) == type(other) + class ParseError(Exception): + """The parse failed; the message may have more information""" + pass + + _hammer._register_helpers(ParseError, + Placeholder) + %} + %typemap(in) void*[] { if (PyList_Check($input)) { Py_INCREF($input); @@ -64,7 +123,9 @@ $result = hpt_to_python($1->ast); } } - +%typemap(newfree) struct HParseResult_* { + h_parse_result_free($input); + } %inline %{ static int h_tt_python; %} @@ -72,6 +133,8 @@ h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python"); %} + + /* %typemap(in) (HPredicate* pred, void* user_data) { Py_INCREF($input); @@ -132,7 +195,7 @@ } switch (token->token_type) { case TT_NONE: - Py_RETURN_NONE; + return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL); break; case TT_BYTES: return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); @@ -173,6 +236,8 @@ HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } - + } + + #endif From 43357bbcda19ff22b509ecbd134b0b14bbe1b0ef Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Sun, 24 Nov 2013 18:43:14 -0600 Subject: [PATCH 43/44] AttrBool working --- src/bindings/python/hammer_tests.py | 18 ++++++++-------- src/bindings/swig/hammer.i | 32 ++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 7a7d5f1..587fbd5 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -402,15 +402,15 @@ class TestEpsilonP3(unittest.TestCase): def test_failure(self): pass -# class TestAttrBool(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) -# def test_success(self): -# self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# self.assertEqual(h.h_parse(self.parser, "bb"), ["b", "b"]) -# def test_failure(self): -# self.assertEqual(h.h_parse(self.parser, "ab"), None) +class TestAttrBool(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + def test_success(self): + self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) + self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + def test_failure(self): + self.assertEqual(h.h_parse(self.parser, "ab"), None) class TestAnd1(unittest.TestCase): @classmethod diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index a2a8940..20661c8 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -135,26 +135,28 @@ -/* -%typemap(in) (HPredicate* pred, void* user_data) { + +%typemap(in) (HPredicate pred, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_predicate; } -*/ + %typemap(in) (const HAction a, void* user_data) { Py_INCREF($input); $2 = $input; $1 = call_action; } -%inline { +%inline %{ + struct HParsedToken_; struct HParseResult_; static PyObject* hpt_to_python(const struct HParsedToken_ *token); static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data); - } + static int call_predicate(const struct HParseResult_ *p, void* user_data); + %} #else #warning no uint8_t* typemaps defined #endif @@ -164,6 +166,7 @@ #include "allocator.h" #include "hammer.h" #include "internal.h" +#include "glue.h" %} %include "allocator.h" %include "hammer.h" @@ -233,11 +236,30 @@ assert(ret != NULL); } // TODO: add reference to ret to parse-local data + // For now, just hold onto reference HParsedToken *tok = h_make(p->arena, h_tt_python, ret); return tok; } + static int call_predicate(const struct HParseResult_ *p, void* user_data) { + PyObject *callable = user_data; + PyObject *ret = PyObject_CallFunctionObjArgs(callable, + hpt_to_python(p->ast), + NULL); + int rret = 0; + if (ret == NULL) { + // TODO: throw exception + PyErr_Print(); + assert(ret != NULL); + } + // TODO: add reference to ret to parse-local data + rret = PyObject_IsTrue(ret); + Py_DECREF(ret); + return rret; + } + } + #endif From cdc7ea83816289231ef14d655c0c5f4138be8fb0 Mon Sep 17 00:00:00 2001 From: Dan Hirsch Date: Mon, 25 Nov 2013 02:49:02 -0600 Subject: [PATCH 44/44] Finished Python bindings --- src/bindings/python/hammer_tests.py | 398 ++++++++++++++-------------- src/bindings/swig/hammer.i | 145 +++++++--- 2 files changed, 315 insertions(+), 228 deletions(-) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 587fbd5..9014d5c 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -4,457 +4,462 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_token("95\xa2") + cls.parser = h.token("95\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "95\xa2"), "95\xa2") + self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") def test_partial_fails(self): - self.assertEqual(h.h_parse(self.parser, "95"), None) + self.assertEqual(self.parser.parse("95"), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser_int = h.h_ch(0xa2) - cls.parser_chr = h.h_ch("\xa2") + cls.parser_int = h.ch(0xa2) + cls.parser_chr = h.ch("\xa2") def test_success(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa2"), 0xa2) - self.assertEqual(h.h_parse(self.parser_chr, "\xa2"), "\xa2") + self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") def test_failure(self): - self.assertEqual(h.h_parse(self.parser_int, "\xa3"), None) - self.assertEqual(h.h_parse(self.parser_chr, "\xa3"), None) + self.assertEqual(self.parser_int.parse("\xa3"), None) + self.assertEqual(self.parser_chr.parse("\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_ch_range("a", "c") + cls.parser = h.ch_range("a", "c") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int64() + cls.parser = h.int64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int32() + cls.parser = h.int32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xff\xfe\x00"), None) - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int16() + cls.parser = h.int16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\xfe\x00"), -0x200) - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\xfe"), None) - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\xfe"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int8() + cls.parser = h.int8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x88"), -0x78) + self.assertEqual(self.parser.parse("\x88"), -0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint64() + cls.parser = h.uint64() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint32() + cls.parser = h.uint32() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x00\x02\x00"), None) + self.assertEqual(self.parser.parse("\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint16() + cls.parser = h.uint16() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x02\x00"), 0x200) + self.assertEqual(self.parser.parse("\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x02"), None) + self.assertEqual(self.parser.parse("\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_uint8() + cls.parser = h.uint8() def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x78"), 0x78) + self.assertEqual(self.parser.parse("\x78"), 0x78) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_int_range(h.h_uint8(), 3, 10) + cls.parser = h.int_range(h.uint8(), 3, 10) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "\x05"), 5) + self.assertEqual(self.parser.parse("\x05"), 5) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "\x0b"), None) + self.assertEqual(self.parser.parse("\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_ch("a")) + cls.parser = h.whitespace(h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, " a"), "a") - self.assertEqual(h.h_parse(self.parser, "\ta"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse("\ta"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "_a"), None) + self.assertEqual(self.parser.parse("_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_whitespace(h.h_end_p()) + cls.parser = h.whitespace(h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), None) # empty string - self.assertEqual(h.h_parse(self.parser, " "), None) # empty string + self.assertEqual(self.parser.parse(""), None) # empty string + self.assertEqual(self.parser.parse(" "), None) # empty string def test_failure(self): - self.assertEqual(h.h_parse(self.parser, " x"), None) + self.assertEqual(self.parser.parse(" x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_left(h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.left(h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a "), "a") + self.assertEqual(self.parser.parse("a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_right(h.h_ch(" "), h.h_ch("a")) + cls.parser = h.right(h.ch(" "), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a"), "a") + self.assertEqual(self.parser.parse(" a"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, "ba"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse("ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_middle(h.h_ch(" "), h.h_ch("a"), h.h_ch(" ")) + cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, " a "), "a") + self.assertEqual(self.parser.parse(" a "), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, " "), None) - self.assertEqual(h.h_parse(self.parser, " a"), None) - self.assertEqual(h.h_parse(self.parser, "a "), None) - self.assertEqual(h.h_parse(self.parser, " b "), None) - self.assertEqual(h.h_parse(self.parser, "ba "), None) - self.assertEqual(h.h_parse(self.parser, " ab"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(" "), None) + self.assertEqual(self.parser.parse(" a"), None) + self.assertEqual(self.parser.parse("a "), None) + self.assertEqual(self.parser.parse(" b "), None) + self.assertEqual(self.parser.parse("ba "), None) + self.assertEqual(self.parser.parse(" ab"), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_action(h.h_sequence__a([h.h_choice__a([h.h_ch("a"), h.h_ch("A")]), - h.h_choice__a([h.h_ch("b"), h.h_ch("B")])]), + cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), + h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ["A", "B"]) - self.assertEqual(h.h_parse(self.parser, "AB"), ["A", "B"]) + self.assertEqual(self.parser.parse("ab"), ["A", "B"]) + self.assertEqual(self.parser.parse("AB"), ["A", "B"]) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "XX"), None) + self.assertEqual(self.parser.parse("XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_in("abc") + cls.parser = h.in_("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "d"), None) + self.assertEqual(self.parser.parse("d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_not_in("abc") + cls.parser = h.not_in("abc") def test_success(self): - self.assertEqual(h.h_parse(self.parser, "d"), "d") + self.assertEqual(self.parser.parse("d"), "d") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_end_p()]) + cls.parser = h.sequence(h.ch("a"), h.end_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aa"), None) + self.assertEqual(self.parser.parse("aa"), None) class TestNothingP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_nothing_p() + cls.parser = h.nothing_p() def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) - self.assertEqual(h.h_parse(self.parser, "b"), None) + self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse("b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_whitespace(h.h_ch("b"))]) + cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) - self.assertEqual(h.h_parse(self.parser, "a b"), ('a','b')) + self.assertEqual(self.parser.parse("ab"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) + self.assertEqual(self.parser.parse("a b"), ('a','b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a c"), None) + self.assertEqual(self.parser.parse("a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_choice__a([h.h_ch("a"), h.h_ch("b")]) + cls.parser = h.choice(h.ch("a"), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "b"), "b") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("b"), "b") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "c"), None) + self.assertEqual(self.parser.parse("c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch("a"), h.h_token("ab")) + cls.parser = h.butnot(h.ch("a"), h.token("ab")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), "a") - self.assertEqual(h.h_parse(self.parser, "aa"), "a") + self.assertEqual(self.parser.parse("a"), "a") + self.assertEqual(self.parser.parse("aa"), "a") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_butnot(h.h_ch_range("0", "9"), h.h_ch("6")) + cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "4"), "4") + self.assertEqual(self.parser.parse("4"), "4") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "6"), None) + self.assertEqual(self.parser.parse("6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_difference(h.h_token("ab"), h.h_ch("a")) + cls.parser = h.difference(h.token("ab"), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), "ab") + self.assertEqual(self.parser.parse("ab"), "ab") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_xor(h.h_ch_range("0", "6"), h.h_ch_range("5", "9")) + cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), "0") - self.assertEqual(h.h_parse(self.parser, "9"), "9") + self.assertEqual(self.parser.parse("0"), "0") + self.assertEqual(self.parser.parse("9"), "9") def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "5"), None) - self.assertEqual(h.h_parse(self.parser, "a"), None) + self.assertEqual(self.parser.parse("5"), None) + self.assertEqual(self.parser.parse("a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, ""), ()) - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "b"), ('b',)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ('a','a','b','b','a','b','a')) + self.assertEqual(self.parser.parse(""), ()) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("b"), ('b',)) + self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a')) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])) + cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) - self.assertEqual(h.h_parse(self.parser, "b"), ("b",)) - self.assertEqual(h.h_parse(self.parser, "aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) + self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse("b"), ("b",)) + self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) - self.assertEqual(h.h_parse(self.parser, "daabbabadef"), None) + self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse("daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_repeat_n(h.h_choice__a([h.h_ch("a"), h.h_ch("b")]), 2) + cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abdef"), ('a', 'b')) + self.assertEqual(self.parser.parse("abdef"), ('a', 'b')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "adef"), None) - self.assertEqual(h.h_parse(self.parser, "dabdef"), None) + self.assertEqual(self.parser.parse("adef"), None) + self.assertEqual(self.parser.parse("dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_optional(h.h_choice__a([h.h_ch("b"), h.h_ch("c")])), h.h_ch("d")]) + cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abd"), ('a','b','d')) - self.assertEqual(h.h_parse(self.parser, "acd"), ('a','c','d')) - self.assertEqual(h.h_parse(self.parser, "ad"), ('a',h.Placeholder(), 'd')) + self.assertEqual(self.parser.parse("abd"), ('a','b','d')) + self.assertEqual(self.parser.parse("acd"), ('a','c','d')) + self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd')) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "aed"), None) - self.assertEqual(h.h_parse(self.parser, "ab"), None) - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("aed"), None) + self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_ignore(h.h_ch("b")), h.h_ch("c")]) + cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "abc"), ("a","c")) + self.assertEqual(self.parser.parse("abc"), ("a","c")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ac"), None) + self.assertEqual(self.parser.parse("ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) - self.assertEqual(h.h_parse(self.parser, ""), ()) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) + self.assertEqual(self.parser.parse(""), ()) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sepBy1(h.h_choice__a([h.h_ch("1"), h.h_ch("2"), h.h_ch("3")]), h.h_ch(",")) + cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "1,2,3"), ('1','2','3')) - self.assertEqual(h.h_parse(self.parser, "1,3,2"), ('1','3','2')) - self.assertEqual(h.h_parse(self.parser, "1,3"), ('1','3')) - self.assertEqual(h.h_parse(self.parser, "3"), ('3',)) + self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) + self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) + self.assertEqual(self.parser.parse("1,3"), ('1','3')) + self.assertEqual(self.parser.parse("3"), ('3',)) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, ""), None) + self.assertEqual(self.parser.parse(""), None) ### segfaults class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p(), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "ab"), ("a", "b")) + self.assertEqual(self.parser.parse("ab"), ("a", "b")) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_epsilon_p(), h.h_ch("a")]) + cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_epsilon_p()]) + cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ("a",)) + self.assertEqual(self.parser.parse("a"), ("a",)) def test_failure(self): pass class TestAttrBool(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_attr_bool(h.h_many1(h.h_choice__a([h.h_ch("a"), h.h_ch("b")])), lambda x: x[0] == x[1]) + cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), + lambda x: x[0] == x[1]) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "aa"), ("a", "a")) - self.assertEqual(h.h_parse(self.parser, "bb"), ("b", "b")) + self.assertEqual(self.parser.parse("aa"), ("a", "a")) + self.assertEqual(self.parser.parse("bb"), ("b", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "ab"), None) + self.assertEqual(self.parser.parse("ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("0")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "0"), ("0",)) + self.assertEqual(self.parser.parse("0"), ("0",)) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_and(h.h_ch("0")), h.h_ch("1")]) + cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) def test_success(self): pass def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "0"), None) + self.assertEqual(self.parser.parse("0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("1"), h.h_and(h.h_ch("2"))]) + cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "12"), ('1',)) + self.assertEqual(self.parser.parse("12"), ('1',)) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_ch("+"), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), + h.choice(h.ch("+"), h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ("a", "+", "b")) + self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b")) def test_failure(self): - self.assertEqual(h.h_parse(self.parser, "a++b"), None) + self.assertEqual(self.parser.parse("a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_sequence__a([h.h_ch("a"), h.h_choice__a([h.h_sequence__a([h.h_ch("+"), h.h_not(h.h_ch("+"))]), h.h_token("++")]), h.h_ch("b")]) + cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), + h.token("++")), + h.ch("b")) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a+b"), ('a', ('+',), 'b')) - self.assertEqual(h.h_parse(self.parser, "a++b"), ('a', "++", 'b')) + self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b')) + self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b')) def test_failure(self): pass @@ -462,26 +467,29 @@ class TestNot2(unittest.TestCase): # #class TestLeftrec(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # a = h.h_ch("a") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, a), a)) +# # cls.parser = h.indirect() +# # a = h.ch("a") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "a"), "a") -# # self.assertEqual(h.h_parse(self.parser, "aa"), ["a", "a"]) -# # self.assertEqual(h.h_parse(self.parser, "aaa"), ["a", "a", "a"]) +# # self.assertEqual(self.parser.parse("a"), "a") +# # self.assertEqual(self.parser.parse("aa"), ["a", "a"]) +# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) # # def test_failure(self): # # pass + class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.h_indirect() - a = h.h_ch("a") - h.h_bind_indirect(cls.parser, h.h_choice__a([h.h_sequence__a([a, cls.parser]), h.h_epsilon_p()])) + #raise unittest.SkipTest("Bind doesn't work right now") + cls.parser = h.indirect() + a = h.ch("a") + cls.parser.bind(h.choice(h.sequence(a, cls.parser), + h.epsilon_p())) def test_success(self): - self.assertEqual(h.h_parse(self.parser, "a"), ('a',)) - self.assertEqual(h.h_parse(self.parser, "aa"), ('a', ('a',))) - self.assertEqual(h.h_parse(self.parser, "aaa"), ('a', ('a', ('a',)))) + self.assertEqual(self.parser.parse("a"), ('a',)) + self.assertEqual(self.parser.parse("aa"), ('a', ('a',))) + self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',)))) def test_failure(self): pass @@ -489,15 +497,15 @@ class TestRightrec(unittest.TestCase): # #class TestAmbiguous(unittest.TestCase): # # @classmethod # # def setUpClass(cls): -# # cls.parser = h.h_indirect() -# # d = h.h_ch("d") -# # p = h.h_ch("+") -# # h.h_bind_indirect(cls.parser, h.h_choice(h.h_sequence(cls.parser, p, cls.parser), d)) +# # cls.parser = h.indirect() +# # d = h.ch("d") +# # p = h.ch("+") +# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(h.h_parse(self.parser, "d"), ["d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d"), ["d", "+", "d"]) -# # self.assertEqual(h.h_parse(self.parser, "d+d+d"), ["d", "+", "d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d"), ["d"]) +# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) +# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) # # def test_failure(self): -# # self.assertEqual(h.h_parse(self.parser, "d+"), None) +# # self.assertEqual(self.parser.parse("d+"), None) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 20661c8..e251904 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -12,43 +12,13 @@ %apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)} -%rename(_h_ch) h_ch; -%pythoncode %{ - def h_ch(ch): - if isinstance(ch, str) or isinstance(ch, unicode): - return h_token(ch) - else: - return _h_ch(ch) -%} - -%rename(_h_ch_range) h_ch_range; -%pythoncode %{ - def h_ch_range(c1, c2): - dostr = isinstance(c1, str) - dostr2 = isinstance(c2, str) - if isinstance(c1, unicode) or isinstance(c2, unicode): - raise TypeError("ch_range only works on bytes") - if dostr != dostr2: - raise TypeError("Both arguments to ch_range must be the same type") - if dostr: - return h_action(_h_ch_range(c1, c2), chr) - else: - return _h_ch_range(c1, c2) -%} - -%rename(_h_in) h_in; -%rename(_h_not_in) h_not_in; -%pythoncode %{ - def h_in(charset): - return h_action(_h_in(charset), chr) - def h_not_in(charset): - return h_action(_h_not_in(charset), chr) - %} +%rename("_%s") ""; +// %rename(_h_ch) h_ch; %inline { static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL; - static void _register_helpers(PyObject* parse_error, PyObject *placeholder) { + static void register_helpers(PyObject* parse_error, PyObject *placeholder) { _helper_ParseError = parse_error; _helper_Placeholder = placeholder; } @@ -260,6 +230,115 @@ } +%rename("%s") ""; +%extend HParser_ { + HParseResult* parse(const uint8_t* input, size_t length) { + return h_parse($self, input, length); + } + bool compile(HParserBackend backend) { + return h_compile($self, backend, NULL) == 0; + } + PyObject* __dir__() { + PyObject* ret = PyList_New(2); + PyList_SET_ITEM(ret, 0, PyString_FromString("parse")); + PyList_SET_ITEM(ret, 1, PyString_FromString("compile")); + return ret; + } +} + +%pythoncode %{ + +def action(p, act): + return _h_action(p, act) +def attr_bool(p, pred): + return _h_attr_bool(p, pred) + +def ch(ch): + if isinstance(ch, str) or isinstance(ch, unicode): + return token(ch) + else: + return _h_ch(ch) + +def ch_range(c1, c2): + dostr = isinstance(c1, str) + dostr2 = isinstance(c2, str) + if isinstance(c1, unicode) or isinstance(c2, unicode): + raise TypeError("ch_range only works on bytes") + if dostr != dostr2: + raise TypeError("Both arguments to ch_range must be the same type") + if dostr: + return action(_h_ch_range(c1, c2), chr) + else: + return _h_ch_range(c1, c2) +def epsilon_p(): return _h_epsilon_p() +def end_p(): + return _h_end_p() +def in_(charset): + return action(_h_in(charset), chr) +def not_in(charset): + return action(_h_not_in(charset), chr) +def not_(p): return _h_not(p) +def int_range(p, i1, i2): + return _h_int_range(p, i1, i2) +def token(string): + return _h_token(string) +def whitespace(p): + return _h_whitespace(p) +def xor(p1, p2): + return _h_xor(p1, p2) +def butnot(p1, p2): + return _h_butnot(p1, p2) +def and_(p1): + return _h_and(p1) +def difference(p1, p2): + return _h_difference(p1, p2) + +def sepBy(p, sep): return _h_sepBy(p, sep) +def sepBy1(p, sep): return _h_sepBy1(p, sep) +def many(p): return _h_many(p) +def many1(p): return _h_many1(p) +def repeat_n(p, n): return _h_repeat_n(p, n) +def choice(*args): return _h_choice__a(list(args)) +def sequence(*args): return _h_sequence__a(list(args)) + +def optional(p): return _h_optional(p) +def nothing_p(): return _h_nothing_p() +def ignore(p): return _h_ignore(p) + +def left(p1, p2): return _h_left(p1, p2) +def middle(p1, p2, p3): return _h_middle(p1, p2, p3) +def right(p1, p2): return _h_right(p1, p2) + + +class HIndirectParser(_HParser_): + def __init__(self): + # Shoves the guts of an _HParser_ into a HIndirectParser. + tret = _h_indirect() + self.__dict__.clear() + self.__dict__.update(tret.__dict__) + + def __dir__(self): + return super(HIndirectParser, self).__dir__() + ['bind'] + def bind(self, parser): + _h_bind_indirect(self, parser) + +def indirect(): + return HIndirectParser() + +def bind_indirect(indirect, new_parser): + indirect.bind(new_parser) + +def uint8(): return _h_uint8() +def uint16(): return _h_uint16() +def uint32(): return _h_uint32() +def uint64(): return _h_uint64() +def int8(): return _h_int8() +def int16(): return _h_int16() +def int32(): return _h_int32() +def int64(): return _h_int64() + + +%} #endif