dotnet and php bindings have bitrotted, but packrat is faster now

.travis.yml

@@ -1,3 +1,5 @@
+sudo: required
+dist: trusty
 language: c
 compiler:
   - gcc
@@ -8,35 +10,43 @@ matrix:
   include:
     - compiler: gcc
       language: ruby
-      rvm: ruby-1.9.3-p484
+      rvm: ruby-1.9.3-p551
       env: BINDINGS=ruby
     - compiler: clang
       language: ruby
-      rvm: ruby-1.9.3-p484
+      rvm: ruby-1.9.3-p551
       env: BINDINGS=ruby CC=clang
     - compiler: gcc
       language: ruby
-      rvm: ruby-2.0.0-p353
+      rvm: ruby-2.0.0-p647
       env: BINDINGS=ruby
     - compiler: clang
       language: ruby
-      rvm: ruby-2.0.0-p353
+      rvm: ruby-2.0.0-p647
       env: BINDINGS=ruby CC=clang
     - compiler: gcc
       language: ruby
-      rvm: ruby-2.1.0
+      rvm: ruby-2.1.7
       env: BINDINGS=ruby
     - compiler: clang
       language: ruby
-      rvm: ruby-2.1.0
+      rvm: ruby-2.1.7
+      env: BINDINGS=ruby CC=clang
+    - compiler: gcc
+      language: ruby
+      rvm: ruby-2.2.3
+      env: BINDINGS=ruby
+    - compiler: clang
+      language: ruby
+      rvm: ruby-2.2.3
       env: BINDINGS=ruby CC=clang
     - compiler: gcc
       language: python
-      python: "2.7"
+      python: "2.7.10"
       env: BINDINGS=python
     - compiler: clang
       language: python
-      python: "2.7"
+      python: "2.7.10"
       env: BINDINGS=python CC=clang
     - compiler: gcc
       language: perl
@@ -87,17 +97,16 @@ matrix:
     - compiler: gcc
       language: cpp
       env: BINDINGS=cpp
-    - compiler: gcc
+    - compiler: clang
       language: cpp
       env: BINDINGS=cpp CC=clang
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -y lcov
   - gem install coveralls-lcov
-  - if [ "$BINDINGS" != "none" ]; then sudo apt-get install -qq swig; fi
+  - if [ "$BINDINGS" != "none" ]; then sudo sh -c 'echo "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list'; sudo apt-get update -qq; sudo apt-get install -yqq swig3.0/trusty-backports; fi
-  - if [ "$BINDINGS" == "perl" ]; then sudo add-apt-repository ppa:dns/irc -y; sudo apt-get update -qq; sudo apt-get install -qq swig=2.0.8-1irc1~12.04; fi
+  - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -yqq python-dev; fi
-  - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq python-dev; fi
+  - if [ "$BINDINGS" == "dotnet" ]; then sudo add-apt-repository ppa:directhex/monoxide -y; sudo apt-get update -qq; sudo apt-get install -yqq mono-devel mono-mcs nunit nunit-console; mozroots --import --sync; fi
-  - if [ "$BINDINGS" == "dotnet" ]; then sudo add-apt-repository ppa:directhex/monoxide -y; sudo apt-get update -qq; sudo apt-get install -y -qq mono-devel mono-mcs nunit nunit-console; fi
 install: true
 before_script:
   - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi

SConstruct

@@ -9,8 +9,8 @@ if platform.system() == 'Windows':
     default_install_dir = 'build' # no obvious place for installation on Windows
 
 vars = Variables(None, ARGUMENTS)
-vars.Add(PathVariable('DESTDIR', 'Root directory to install in (useful for packaging scripts)', None, PathVariable.PathIsDirCreate))
+vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate))
-vars.Add(PathVariable('prefix', 'Where to install in the FHS', default_install_dir, PathVariable.PathAccept))
+vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept))
 vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby']))
 
 tools = ['default', 'scanreplace']

examples/base64.c

@@ -11,31 +11,76 @@
 #include <inttypes.h>
 #include "../src/hammer.h"
 
+#define DEBUG
+
 const HParser* document = NULL;
 
 void init_parser(void)
 {
     // CORE
-    HParser *digit = h_ch_range(0x30, 0x39);
+    const HParser *digit = h_ch_range(0x30, 0x39);
-    HParser *alpha = h_choice(h_ch_range(0x41, 0x5a), h_ch_range(0x61, 0x7a), NULL);
+    const HParser *alpha = h_choice(h_ch_range(0x41, 0x5a), h_ch_range(0x61, 0x7a), NULL);
 
     // AUX.
-    HParser *plus = h_ch('+');
+    const HParser *plus = h_ch('+');
-    HParser *slash = h_ch('/');
+    const HParser *slash = h_ch('/');
-    HParser *equals = h_ch('=');
+    const HParser *equals = h_ch('=');
 
-    HParser *bsfdig = h_choice(alpha, digit, plus, slash, NULL);
+    const HParser *bsfdig = h_choice(alpha, digit, plus, slash, NULL);
-    HParser *bsfdig_4bit = h_in((uint8_t *)"AEIMQUYcgkosw048", 16);
+    const HParser *bsfdig_4bit = h_choice(
-    HParser *bsfdig_2bit = h_in((uint8_t *)"AQgw", 4);
+        h_ch('A'), h_ch('E'), h_ch('I'), h_ch('M'), h_ch('Q'), h_ch('U'),
-    HParser *base64_3 = h_repeat_n(bsfdig, 4);
+        h_ch('Y'), h_ch('c'), h_ch('g'), h_ch('k'), h_ch('o'), h_ch('s'),
-    HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL);
+        h_ch('w'), h_ch('0'), h_ch('4'), h_ch('8'), NULL);
-    HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL);
+    const HParser *bsfdig_2bit = h_choice(h_ch('A'), h_ch('Q'), h_ch('g'), h_ch('w'), NULL);
-    HParser *base64 = h_sequence(h_many(base64_3),
-                                       h_optional(h_choice(base64_2,
-                                                           base64_1, NULL)),
-                                       NULL);
 
-    document = h_sequence(h_whitespace(base64), h_whitespace(h_end_p()), NULL);
+    const HParser *base64_quad = h_sequence(bsfdig, bsfdig, bsfdig, bsfdig, NULL);
+    const HParser *base64_quads = h_many(base64_quad);
+
+    const HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, h_end_p(), NULL);
+    const HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, h_end_p(), NULL);
+    const HParser *base64_ending = h_choice(h_end_p(), base64_2, base64_1, NULL);
+    const HParser *base64 = h_sequence(base64_quads, base64_ending, NULL);
+        // why does this parse "A=="?!
+        // why does this parse "aaA=" but not "aA=="?!
+
+    document = base64;
+}
+
+
+#include <string.h>
+#include <assert.h>
+#define TRUE (1)
+#define FALSE (0)
+
+void assert_parse(int expected, char *data) {
+    const HParseResult *result;
+
+    size_t datasize = strlen(data);
+    result = h_parse(document, (void*)data, datasize);
+    if((result != NULL) != expected) {
+        fprintf(stderr, "Test failed: %s\n", data);
+    }
+#ifdef DEBUG
+    else {
+        fprintf(stderr, "Test succeeded: %s\n", data);
+        fprintf(stderr, "parsed=%lld bytes\n", result->bit_length/8);
+        h_pprint(stdout, result->ast, 0, 0);
+    }
+#endif
+}
+
+void test() {
+    assert_parse(TRUE, "");
+    assert_parse(TRUE, "YQ==");
+    assert_parse(TRUE, "YXU=");
+    assert_parse(TRUE, "YXVy");
+    assert_parse(TRUE, "QVVSIFNBUkFG");
+    assert_parse(TRUE, "QVVSIEhFUlUgU0FSQUY=");
+    assert_parse(FALSE, "A");
+    assert_parse(FALSE, "A=");
+    assert_parse(FALSE, "A==");
+    assert_parse(FALSE, "AAA==");
+    assert_parse(FALSE, "aa==");
 }
 
 
@@ -49,6 +94,8 @@ int main(int argc, char **argv)
 
     init_parser();
 
+    test();
+
     inputsize = fread(input, 1, sizeof(input), stdin);
     fprintf(stderr, "inputsize=%zu\ninput=", inputsize);
     fwrite(input, 1, inputsize, stderr);

src/backends/packrat.c

@@ -188,9 +188,10 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
   // check to see if there is already a result for this object...
   if (!m) {
     // It doesn't exist, so create a dummy result to cache
-    HLeftRec *base = a_new(HLeftRec, 1);
+    HLeftRec *base = NULL;
     // But only cache it now if there's some chance it could grow; primitive parsers can't
     if (parser->vtable->higher) {
+      base = a_new(HLeftRec, 1);
       base->seed = NULL; base->rule = parser; base->head = NULL;
       h_slist_push(state->lr_stack, base);
       // cache it
@@ -207,7 +208,7 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
       cached->input_stream = state->input_stream;
     }
     // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
-    if (NULL == base->head) {
+    if (!base || NULL == base->head) {
       h_hashtable_put(state->cache, key, cached_result(state, tmp_res));
       return tmp_res;
     } else {

src/bindings/dotnet/SConscript

@@ -27,7 +27,7 @@ csfiles = os.path.join(thisdir, "*.cs")
 # target to stand in for.
 hammer_wrap = AlwaysBuild(dotnetenv.Command(['hammer_wrap.c'], swig,
                                             ["rm %s/*.cs || true" % (thisdir,),
-                                             "swig $SWIGFLAGS $SOURCE"]))
+                                             "swig3.0 $SWIGFLAGS $SOURCE"]))
 libhammer_dotnet = dotnetenv.SharedLibrary(['hammer_dotnet'], hammer_wrap)
 hammer_dll = AlwaysBuild(dotnetenv.Command(['hammer.dll'], Glob('ext/*.cs'),
                                            '$CSC -t:library -unsafe -out:$TARGET %s/*.cs $SOURCE' %(thisdir,)))

src/bindings/perl/SConscript

@@ -20,7 +20,7 @@ if 'PERL5LIB' in os.environ:
 
 swig = ['hammer.i']
 
-hammer_wrap = perlenv.Command(['hammer_wrap.c', 'hammer.pm'], swig, "swig $SWIGFLAGS $SOURCE")
+hammer_wrap = perlenv.Command(['hammer_wrap.c', 'hammer.pm'], swig, "swig3.0 $SWIGFLAGS $SOURCE")
 makefile = perlenv.Command(['Makefile'], ['Makefile.PL'], "perl $SOURCE CC=" + perlenv['ENV']['CC'])
 
 targetdir = os.path.dirname(str(hammer_wrap[0].path))

src/bindings/php/SConscript

@@ -11,7 +11,7 @@ phpenv.Append(LIBS = ['hammer'])
 phpenv.Append(LIBPATH = ['../../']) 
 
 swig = ['hammer.i']
-bindings_src = phpenv.Command(['hammer.php', 'hammer_wrap.c', 'php_hammer.h'], swig, 'swig -php -DHAMMER_INTERNAL__NO_STDARG_H -Isrc/ $SOURCE')
+bindings_src = phpenv.Command(['hammer.php', 'hammer_wrap.c', 'php_hammer.h'], swig, 'swig3.0 -php -DHAMMER_INTERNAL__NO_STDARG_H -Isrc/ $SOURCE')
 libhammer_php = phpenv.SharedLibrary('hammer', ['hammer_wrap.c'])
 Default(swig, bindings_src, libhammer_php)
 

src/bindings/python/SConscript

@@ -7,7 +7,7 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 swig = pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 setup = ['setup.py']
 pydir = os.path.join(env['BUILD_BASE'], 'src/bindings/python')
-libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --inplace')
+libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --swig=swig3.0 --inplace')
 Default(libhammer_python)
 
 pytestenv = pythonenv.Clone()