Merge remote-tracking branch 'upstream/master' into sloballoc

.travis.yml

@@ -92,12 +92,12 @@ matrix:
       env: BINDINGS=cpp CC=clang
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install lcov
+  - sudo apt-get install -y lcov
   - gem install coveralls-lcov
   - if [ "$BINDINGS" != "none" ]; then sudo apt-get install -qq swig; fi
   - if [ "$BINDINGS" == "perl" ]; then sudo add-apt-repository ppa:dns/irc -y; sudo apt-get update -qq; sudo apt-get install -qq swig=2.0.8-1irc1~12.04; fi
   - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq python-dev; fi
-  - if [ "$BINDINGS" == "dotnet" ]; then sudo add-apt-repository ppa:directhex/monoxide -y; sudo apt-get update -qq; sudo apt-get install -qq mono-devel mono-mcs nunit nunit-console; mozroots --import --sync; fi
+  - if [ "$BINDINGS" == "dotnet" ]; then sudo add-apt-repository ppa:directhex/monoxide -y; sudo apt-get update -qq; sudo apt-get install -y -qq mono-devel mono-mcs nunit nunit-console; fi
 install: true
 before_script:
   - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi

README.md

@@ -29,19 +29,19 @@ Features
 Installing
 ==========
 ### Prerequisites
-* SCons
+* [SCons](http://scons.org/)
 
 ### Optional Dependencies
 * pkg-config (for `scons test`)
 * glib-2.0 (>= 2.29) (for `scons test`)
 * glib-2.0-dev (for `scons test`)
-* swig (for Python/Perl/PHP bindings; Perl requires >= 2.0.8)
+* [swig](http://swig.org/) (for Python/Perl/PHP bindings; Perl requires >= 2.0.8)
 * python2.7-dev (for Python bindings)
 * a JDK (for Java bindings)
 * a working [phpenv](https://github.com/CHH/phpenv) configuration (for PHP bindings)
-* Ruby >= 1.9.3 and bundler, for the Ruby bindings
+* [Ruby](https://www.ruby-lang.org/) >= 1.9.3 and bundler, for the Ruby bindings
 * mono-devel and mono-mcs (>= 3.0.6) (for .NET bindings)
-* nunit (for testing .NET bindings)
+* [nunit](http://www.nunit.org/) (for testing .NET bindings)
 
 To build, type `scons`. To run the built-in test suite, type `scons test`. For a debug build, add `--variant=debug`.
 
@@ -60,13 +60,16 @@ Just `#include <hammer/hammer.h>` (also `#include <hammer/glue.h>` if you plan t
 
 If you've installed Hammer system-wide, you can use `pkg-config` in the usual way.
 
-For documentation, see the [user guide](https://github.com/UpstandingHackers/hammer/wiki/User-guide).
+To learn about hammer check
+* the [user guide](https://github.com/UpstandingHackers/hammer/wiki/User-guide)
+* [Hammer Primer](https://github.com/sergeybratus/HammerPrimer) (outdated in terms of code, but good to get the general thinking)
+* [Try Hammer](https://github.com/sboesen/TryHammer)
 
 Examples
 ========
 The `examples/` directory contains some simple examples, currently including:
-* base64
+* [base64](https://en.wikipedia.org/wiki/Base64)
-* DNS
+* [DNS](https://en.wikipedia.org/wiki/Domain_Name_System)
 
 Known Issues
 ============

SConstruct

@@ -4,10 +4,13 @@ import os.path
 import platform
 import sys
 
+default_install_dir='/usr/local'
+if platform.system() == 'Windows':
+    default_install_dir = 'build' # no obvious place for installation on Windows
 
 vars = Variables(None, ARGUMENTS)
-vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate))
+vars.Add(PathVariable('DESTDIR', 'Root directory to install in (useful for packaging scripts)', None, PathVariable.PathIsDirCreate))
-vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept))
+vars.Add(PathVariable('prefix', 'Where to install in the FHS', default_install_dir, PathVariable.PathAccept))
 vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby']))
 
 tools = ['default', 'scanreplace']
@@ -17,6 +20,9 @@ if 'dotnet' in ARGUMENTS.get('bindings', []):
 envvars = {'PATH' : os.environ['PATH']}
 if 'PKG_CONFIG_PATH' in os.environ:
     envvars['PKG_CONFIG_PATH'] = os.environ['PKG_CONFIG_PATH']
+if platform.system() == 'Windows':
+    # from the scons FAQ (keywords: LNK1104 TEMPFILE), needed by link.exe
+    envvars['TMP'] = os.environ['TMP']
 
 env = Environment(ENV = envvars,
                   variables = vars,
@@ -29,7 +35,7 @@ if not 'bindings' in env:
 def calcInstallPath(*elements):
     path = os.path.abspath(os.path.join(*map(env.subst, elements)))
     if 'DESTDIR' in env:
-        path = os.path.join(env['DESTDIR'], os.path.relpath(path, start="/"))
+        path = os.path.join(env['DESTDIR'], os.path.relpath(path, start='/'))
     return path
 
 rel_prefix = not os.path.isabs(env['prefix'])
@@ -37,102 +43,131 @@ env['prefix'] = os.path.abspath(env['prefix'])
 if 'DESTDIR' in env:
     env['DESTDIR'] = os.path.abspath(env['DESTDIR'])
     if rel_prefix:
-        print >>sys.stderr, "--!!-- You used a relative prefix with a DESTDIR. This is probably not what you"
+        print >>sys.stderr, '--!!-- You used a relative prefix with a DESTDIR. This is probably not what you'
-        print >>sys.stderr, "--!!-- you want; files will be installed in"
+        print >>sys.stderr, '--!!-- you want; files will be installed in'
-        print >>sys.stderr, "--!!--    %s" % (calcInstallPath("$prefix"),)
+        print >>sys.stderr, '--!!--    %s' % (calcInstallPath('$prefix'),)
 
 
-env['libpath'] = calcInstallPath("$prefix", "lib")
+env['libpath'] = calcInstallPath('$prefix', 'lib')
-env['incpath'] = calcInstallPath("$prefix", "include", "hammer")
+env['incpath'] = calcInstallPath('$prefix', 'include', 'hammer')
-env['parsersincpath'] = calcInstallPath("$prefix", "include", "hammer", "parsers")
+env['parsersincpath'] = calcInstallPath('$prefix', 'include', 'hammer', 'parsers')
-env['backendsincpath'] = calcInstallPath("$prefix", "include", "hammer", "backends")
+env['backendsincpath'] = calcInstallPath('$prefix', 'include', 'hammer', 'backends')
-env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig")
+env['pkgconfigpath'] = calcInstallPath('$prefix', 'lib', 'pkgconfig')
 env.ScanReplace('libhammer.pc.in')
 
-env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable")
+AddOption('--variant',
+          dest='variant',
+          nargs=1, type='choice',
+          choices=['debug', 'opt'],
+          default='opt',
+          action='store',
+          help='Build variant (debug or opt)')
 
+AddOption('--coverage',
+          dest='coverage',
+          default=False,
+          action='store_true',
+          help='Build with coverage instrumentation')
+
+AddOption('--in-place',
+          dest='in_place',
+          default=False,
+          action='store_true',
+          help='Build in-place, rather than in the build/<variant> tree')
+
+AddOption('--tests',
+          dest='with_tests',
+          default=env['PLATFORM'] != 'win32',
+          action='store_true',
+          help='Build tests')
+
+env['CC'] = os.getenv('CC') or env['CC']
+env['CXX'] = os.getenv('CXX') or env['CXX']
+
+if os.getenv('CC') == 'clang' or env['PLATFORM'] == 'darwin':
+    env.Replace(CC='clang',
+                CXX='clang++')
+
+# Language standard and warnings
+if env['CC'] == 'cl':
+    env.MergeFlags('-W3 -WX')
+    env.Append(
+        CPPDEFINES=[
+            '_CRT_SECURE_NO_WARNINGS' # allow uses of sprintf
+        ],
+        CFLAGS=[
+            '-wd4018', # 'expression' : signed/unsigned mismatch
+            '-wd4244', # 'argument' : conversion from 'type1' to 'type2', possible loss of data
+            '-wd4267', # 'var' : conversion from 'size_t' to 'type', possible loss of data
+        ]
+    )
+else:
+    env.MergeFlags('-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable')
+
+# Linker options
 if env['PLATFORM'] == 'darwin':
-    env.Append(SHLINKFLAGS = '-install_name ' + env["libpath"] + '/${TARGET.file}')
+    env.Append(SHLINKFLAGS = '-install_name ' + env['libpath'] + '/${TARGET.file}')
-elif os.uname()[0] == "OpenBSD":
+elif platform.system() == 'OpenBSD':
+    pass
+elif env['PLATFORM'] == 'win32':
+    # no extra lib needed
     pass
 else:
-    env.MergeFlags("-lrt")
+    env.MergeFlags('-lrt')
 
-AddOption("--variant",
+if GetOption('coverage'):
-          dest="variant",
+    env.Append(CFLAGS=['--coverage'],
-          nargs=1, type="choice",
+               CXXFLAGS=['--coverage'],
-          choices=["debug", "opt"],
+               LDFLAGS=['--coverage'])
-          default="opt",
+    if env['CC'] == 'gcc':
-          action="store",
-          help="Build variant (debug or opt)")
-
-AddOption("--coverage",
-          dest="coverage",
-          default=False,
-          action="store_true",
-          help="Build with coverage instrumentation")
-
-AddOption("--in-place",
-          dest="in_place",
-          default=False,
-          action="store_true",
-          help="Build in-place, rather than in the build/<variant> tree")
-
-
-dbg = env.Clone(VARIANT='debug')
-dbg.Append(CCFLAGS=['-g'])
-
-opt = env.Clone(VARIANT='opt')
-opt.Append(CCFLAGS=["-O3"])
-
-if GetOption("variant") == 'debug':
-    env = dbg
-else:
-    env = opt
-
-env["CC"] = os.getenv("CC") or env["CC"]
-env["CXX"] = os.getenv("CXX") or env["CXX"]
-
-if GetOption("coverage"):
-    env.Append(CFLAGS=["--coverage"],
-               CXXFLAGS=["--coverage"],
-               LDFLAGS=["--coverage"])
-    if env["CC"] == "gcc":
         env.Append(LIBS=['gcov'])
     else:
         env.ParseConfig('llvm-config --ldflags')
 
-if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin':
+dbg = env.Clone(VARIANT='debug')
-    env.Replace(CC="clang",
+if env['CC'] == 'cl':
-                CXX="clang++")
+    dbg.Append(CCFLAGS=['/Z7'])
+else:
+    dbg.Append(CCFLAGS=['-g'])
 
-env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_"))
+opt = env.Clone(VARIANT='opt')
+if env['CC'] == 'cl':
+    opt.Append(CCFLAGS=['/O2'])
+else:
+    opt.Append(CCFLAGS=['-O3'])
+
+if GetOption('variant') == 'debug':
+    env = dbg
+else:
+    env = opt
+
+env['ENV'].update(x for x in os.environ.items() if x[0].startswith('CCC_'))
 
 #rootpath = env['ROOTPATH'] = os.path.abspath('.')
-#env.Append(CPPPATH=os.path.join('#', "hammer"))
+#env.Append(CPPPATH=os.path.join('#', 'hammer'))
 
 testruns = []
 
-targets = ["$libpath",
+targets = ['$libpath',
-           "$incpath",
+           '$incpath',
-           "$parsersincpath",
+           '$parsersincpath',
-           "$backendsincpath",
+           '$backendsincpath',
-           "$pkgconfigpath"]
+           '$pkgconfigpath']
 
 Export('env')
 Export('testruns')
 Export('targets')
 
-if not GetOption("in_place"):
+if not GetOption('in_place'):
     env['BUILD_BASE'] = 'build/$VARIANT'
-    lib = env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src')
+    lib = env.SConscript(['src/SConscript'], variant_dir='$BUILD_BASE/src')
-    env.Alias("examples", env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples'))
+    env.Alias('examples', env.SConscript(['examples/SConscript'], variant_dir='$BUILD_BASE/examples'))
 else:
     env['BUILD_BASE'] = '.'
-    lib = env.SConscript(["src/SConscript"])
+    lib = env.SConscript(['src/SConscript'])
-    env.Alias(env.SConscript(["examples/SConscript"]))
+    env.Alias(env.SConscript(['examples/SConscript']))
 
 for testrun in testruns:
-    env.Alias("test", testrun)
+    env.Alias('test', testrun)
 
-env.Alias("install", targets)
+env.Alias('install', targets)

appveyor.yml

@@ -1,9 +1,16 @@
 platform:
 - x86
 - x64
+environment:
+    PYTHON: "C:\\Python27"
 version: 1.0.{build}
 os: Visual Studio 2015
+install:
+  - SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - easy_install scons
 build_script:
+- scons --version
+- scons install
 - '@echo off'
 - setlocal
 - ps: >-
@@ -14,6 +21,5 @@ build_script:
     }
 - call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" %VCVARS_PLATFORM%
 - call tools\windows\build.bat
-# FIXME(windows) TODO(uucidl): reactivate examples
+- call tools\windows\build_examples.bat
-# - call tools\windows\build_examples.bat
 - exit /b 0

examples/grammar.c

@@ -22,11 +22,12 @@
 const char *nonterminal_name(const HCFGrammar *g, const HCFChoice *nt) {
   // if user_data exists and is printable:
   if(nt->user_data != NULL && *(char*)(nt->user_data) > ' ' && *(char*)(nt->user_data) < 127) {
-    if(*(char*)(nt->user_data) != '0') {
+    char* user_str = (char*)(nt->user_data);
+    if(*user_str != '\0') {
       // user_data is a non-empty string
-      return nt->user_data;
+      return user_str;
     } else {
-      return nt->user_data+1;
+      return user_str+1;
     }
   }
   

src/SConscript

@@ -1,23 +1,24 @@
 # -*- python -*-
 import os.path
+
 Import('env testruns')
 
 dist_headers = [
-    "hammer.h",
+    'hammer.h',
-    "allocator.h",
+    'allocator.h',
-    "compiler_specifics.h",
+    'compiler_specifics.h',
-    "glue.h",
+    'glue.h',
-    "internal.h",
+    'internal.h',
-    "platform.h"
+    'platform.h'
 ]
 
 parsers_headers = [
-    "parsers/parser_internal.h"
+    'parsers/parser_internal.h'
 ]
 
 backends_headers = [
-    "backends/regex.h",
+    'backends/regex.h',
-    "backends/contextfree.h"
+    'backends/contextfree.h'
 ]
 
 parsers = ['parsers/%s.c'%s for s in
@@ -63,12 +64,19 @@ misc_hammer_parts = [
     'desugar.c',
     'glue.c',
     'hammer.c',
-    'platform_bsdlike.c',
     'pprint.c',
     'registry.c',
     'system_allocator.c',
     'sloballoc.c']
 
+if env['PLATFORM'] == 'win32':
+    misc_hammer_parts += [
+        'platform_win32.c',
+        'tsearch.c',
+    ]
+else:
+    misc_hammer_parts += ['platform_bsdlike.c']
+
 ctests = ['t_benchmark.c',
           't_bitreader.c',
           't_bitwriter.c',
@@ -78,26 +86,40 @@ ctests = ['t_benchmark.c',
           't_mm.c',
           't_regression.c']
 
+
+static_library_name = 'hammer'
+build_shared_library=True
+if env['PLATFORM'] == 'win32':
+    # FIXME(windows): symbols in hammer are not exported yet, a shared lib would be useless
+    build_shared_library=False
+    # prevent collision between .lib from dll and .lib for static lib
+    static_library_name = 'hammer_s'
+
 libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts)
-libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts)
+libhammer_static = env.StaticLibrary(static_library_name, parsers + backends + misc_hammer_parts)
+if build_shared_library:
     Default(libhammer_shared, libhammer_static)
+    env.Install('$libpath', [libhammer_static, libhammer_shared])
+else:
+    Default(libhammer_static)
+    env.Install('$libpath', [libhammer_static])
 
-env.Install("$libpath", [libhammer_static, libhammer_shared])
+env.Install('$incpath', dist_headers)
-env.Install("$incpath", dist_headers)
+env.Install('$parsersincpath', parsers_headers)
-env.Install("$parsersincpath", parsers_headers)
+env.Install('$backendsincpath', backends_headers)
-env.Install("$backendsincpath", backends_headers)
+env.Install('$pkgconfigpath', '../../../libhammer.pc')
-env.Install("$pkgconfigpath", "../../../libhammer.pc")
 
+if GetOption('with_tests'):
     testenv = env.Clone()
     testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
     testenv.Append(LIBS=['hammer'])
     testenv.Prepend(LIBPATH=['.'])
-ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS="--coverage" if testenv.GetOption("coverage") else None)
+    ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS='--coverage' if testenv.GetOption('coverage') else None)
-ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path]))
+    ctest = Alias('testc', [ctestexec], ''.join(['env LD_LIBRARY_PATH=', os.path.dirname(ctestexec[0].path), ' ', ctestexec[0].path]))
     AlwaysBuild(ctest)
     testruns.append(ctest)
 
-Export("libhammer_static libhammer_shared")
+Export('libhammer_static libhammer_shared')
 
 for b in env['bindings']:
-    env.SConscript(["bindings/%s/SConscript" % b])
+    env.SConscript(['bindings/%s/SConscript' % b])

src/backends/regex.c

@@ -206,16 +206,43 @@ bool svm_stack_ensure_cap(HAllocator *mm__, HSVMContext *ctx, size_t addl) {
   return true;
 }
 
+/*
+ * GCC produces the following diagnostic on this function:
+ *
+ *    error: argument 'trace' might be clobbered by 'longjmp' or 'vfork' [-Werror=clobbered]
+ *
+ * However, this is spurious; what is happening is that the trace
+ * argument gets reused to store cur, and GCC doesn't know enough
+ * about setjmp to know that the second return only returns nonzero
+ * (and therefore the now-clobbered value of trace is invalid.)
+ *
+ * A side effect of disabling this warning is that we need to be
+ * careful about undefined behaviour involving automatic
+ * variables. Specifically, any automatic variable in this function
+ * whose value gets modified after setjmp has an undefined value after
+ * the second return; here, the only variables that could matter for
+ * are arena and ctx (because they're referenced in "goto fail").
+ */
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunknown-pragmas"
+#pragma GCC diagnostic ignored "-Wclobbered"
+#endif
 HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace, const uint8_t *input, int len) {
   // orig_prog is only used for the action table
-  HSVMContext ctx;
+  HSVMContext *ctx = NULL;
   HArena *arena = h_new_arena(mm__, 0);
-  ctx.stack_count = 0;
+  if (arena == NULL) {
-  ctx.stack_capacity = 16;
+    return NULL;
-  ctx.stack = h_new(HParsedToken*, ctx.stack_capacity);
+  }
+  ctx = h_new(HSVMContext, 1);
+  if (!ctx) goto fail;
+  ctx->stack_count = 0;
+  ctx->stack_capacity = 16;
+  ctx->stack = h_new(HParsedToken*, ctx->stack_capacity);
 
   // out of memory handling
-  if(!arena || !ctx.stack)
+  if(!arena || !ctx->stack)
     goto fail;
   jmp_buf except;
   h_arena_set_except(arena, &except);
@@ -227,20 +254,20 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
   for (cur = trace; cur; cur = cur->next) {
     switch (cur->opcode) {
     case SVM_PUSH:
-      if (!svm_stack_ensure_cap(mm__, &ctx, 1)) {
+      if (!svm_stack_ensure_cap(mm__, ctx, 1)) {
 	goto fail;
       }
       tmp_res = a_new(HParsedToken, 1);
       tmp_res->token_type = TT_MARK;
       tmp_res->index = cur->input_pos;
       tmp_res->bit_offset = 0;
-      ctx.stack[ctx.stack_count++] = tmp_res;
+      ctx->stack[ctx->stack_count++] = tmp_res;
       break;
     case SVM_NOP:
       break;
     case SVM_ACTION:
       // Action should modify stack appropriately
-      if (!orig_prog->actions[cur->arg].action(arena, &ctx, orig_prog->actions[cur->arg].env)) {
+      if (!orig_prog->actions[cur->arg].action(arena, ctx, orig_prog->actions[cur->arg].env)) {
 	
 	// action failed... abort somehow
 	goto fail;
@@ -249,9 +276,9 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
     case SVM_CAPTURE: 
       // Top of stack must be a mark
       // This replaces said mark in-place with a TT_BYTES.
-      assert(ctx.stack[ctx.stack_count-1]->token_type == TT_MARK);
+      assert(ctx->stack[ctx->stack_count-1]->token_type == TT_MARK);
       
-      tmp_res = ctx.stack[ctx.stack_count-1];
+      tmp_res = ctx->stack[ctx->stack_count-1];
       tmp_res->token_type = TT_BYTES;
       // TODO: Will need to copy if bit_offset is nonzero
       assert(tmp_res->bit_offset == 0);
@@ -260,25 +287,33 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       tmp_res->bytes.len = cur->input_pos - tmp_res->index;
       break;
     case SVM_ACCEPT:
-      assert(ctx.stack_count <= 1);
+      assert(ctx->stack_count <= 1);
       HParseResult *res = a_new(HParseResult, 1);
-      if (ctx.stack_count == 1) {
+      if (ctx->stack_count == 1) {
-	res->ast = ctx.stack[0];
+	res->ast = ctx->stack[0];
       } else {
 	res->ast = NULL;
       }
       res->bit_length = cur->input_pos * 8;
       res->arena = arena;
       h_arena_set_except(arena, NULL);
-      h_free(ctx.stack);
+      h_free(ctx->stack);
+      h_free(ctx);
       return res;
     }
   }
  fail:
   if (arena) h_delete_arena(arena);
-  if (ctx.stack) h_free(ctx.stack);
+  if (ctx) {
+    if (ctx->stack) h_free(ctx->stack);
+    h_free(ctx);
+  }
   return NULL;
 }
+// Reenable -Wclobber
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
 
 uint16_t h_rvm_create_action(HRVMProg *prog, HSVMActionFunc action_func, void* env) {
   for (uint16_t i = 0; i < prog->action_count; i++) {

src/bindings/ruby/README.md

@@ -21,12 +21,37 @@ Ruby bindings for [hammer](https://github.com/UpstandingHackers/hammer), a parsi
 
 ## Installation
 
-TODO
+1. Download the hammer source code, and make it available system wide with the bindings. 
   
+  `git clone https://github.com/UpstandingHackers/hammer`
+
+  `cd hammer`
+
+  `scons bindings=ruby`
+
+  `sudo scons bindings=ruby install`
+
+2. On linux, you will have to do 
+
+  `sudo ldconfig`
+  
+3. Build the gem
+  `gem build hammer-parser.gemspec`
+
+4. Install the gem
+  `gem install hammer-parser-x.x.x.gem`
 
 
 ## Examples
 
+Add hammer to your Gemfile. 
+
+  `gem 'hammer-parser'`
+
+Use hammer in your project.
+
+  `require 'hammer'`
+
 ### Building a parser
 
 ```ruby

src/bindings/ruby/hammer-parser.gemspec

@@ -5,9 +5,9 @@ Gem::Specification.new do |s|
   s.summary       = 'Ruby bindings to the hammer parsing library.'
   s.description   = s.summary # TODO: longer description?
   s.authors       = ['Meredith L. Patterson', 'TQ Hirsch', 'Jakob Rath']
-  # TODO:
+  s.email         = 'hammer@upstandinghackers.com'
-  # s.email = ...
+  s.homepage      = 'https://github.com/UpstandingHackers/hammer'
-  # s.homepage = ...
+  s.license       = 'GPL-2.0'
 
   files = []
   files << 'README.md'

src/datastructures.c

@@ -52,7 +52,6 @@ HSlist* h_slist_copy(HSlist *slist) {
     h_slist_push(ret, head->elem);
     tail = ret->head;
     head = head->next;
-  }
     while (head != NULL) {
       // append head item to tail in a new node
       HSlistNode *node = h_arena_malloc(slist->arena, sizeof(HSlistNode));
@@ -61,6 +60,7 @@ HSlist* h_slist_copy(HSlist *slist) {
       tail = tail->next = node;
       head = head->next;
     }
+  }
   return ret;
 }
 

src/hammer.h

@@ -778,10 +778,25 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results);
 //void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
 // }}}
 
+// {{{ result_buf printers (used by token type registry)
+
+struct result_buf;
+
+bool h_append_buf(struct result_buf *buf, const char* input, int len);
+bool h_append_buf_c(struct result_buf *buf, char v);
+bool h_append_buf_formatted(struct result_buf *buf, char* format, ...);
+
+// }}}
+
 // {{{ Token type registry
 /// Allocate a new, unused (as far as this function knows) token type.
 HTokenType h_allocate_token_type(const char* name);
 
+/// Allocate a new token type with an unambiguous print function.
+HTokenType h_allocate_token_new(
+    const char* name,
+    void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf));
+
 /// Get the token type associated with name. Returns -1 if name is unkown
 HTokenType h_get_token_type_number(const char* name);
 

src/internal.h

@@ -422,6 +422,18 @@ struct HParserVtable_ {
   bool higher; // false if primitive
 };
 
+// {{{ Token type registry internal
+
+typedef struct HTTEntry_ {
+  const char* name;
+  HTokenType value;
+  void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf);
+} HTTEntry;
+
+const HTTEntry* h_get_token_type_entry(HTokenType token_type);
+
+// }}}
+
 bool h_false(void*);
 bool h_true(void*);
 bool h_not_regular(HRVMProg*, void*);

src/parsers/choice.c

@@ -1,6 +1,20 @@
 #include <stdarg.h>
 #include "parser_internal.h"
 
+#if defined(__STDC_VERSION__) && (                                  \
+      (__STDC_VERSION__ >= 201112L && !defined(__STDC_NO_VLA__)) ||   \
+      (__STDC_VERSION__ >= 199901L) \
+    )
+#  define STACK_VLA(type,name,size) type name[size]
+#else
+#  if defined(_MSC_VER)
+#    include <malloc.h> // for _alloca
+#    define STACK_VLA(type,name,size) type* name = _alloca(size)
+#  else
+#    error "Missing VLA implementation for this compiler"
+#  endif
+#endif
+
 typedef struct {
   size_t len;
   HParser **p_array;
@@ -53,11 +67,14 @@ static void desugar_choice(HAllocator *mm__, HCFStack *stk__, void *env) {
 
 static bool choice_ctrvm(HRVMProg *prog, void* env) {
   HSequence *s = (HSequence*)env;
-  uint16_t gotos[s->len];
+  // NOTE(uucidl): stack allocation since this backend uses
+  // setjmp/longjmp for error handling.
+  STACK_VLA(uint16_t, gotos, s->len);
   for (size_t i=0; i<s->len; ++i) {
     uint16_t insn = h_rvm_insert_insn(prog, RVM_FORK, 0);
-    if (!h_compile_regex(prog, s->p_array[i]))
+    if (!h_compile_regex(prog, s->p_array[i])) {
       return false;
+    }
     gotos[i] = h_rvm_insert_insn(prog, RVM_GOTO, 65535);
     h_rvm_patch_arg(prog, insn, h_rvm_get_ip(prog));
   }

src/parsers/nothing.c

@@ -1,6 +1,8 @@
 #include "parser_internal.h"
 
-static HParseResult* parse_nothing() {
+static HParseResult* parse_nothing(void* x,HParseState* y) {
+  (void)(x);
+  (void)(y);
   // not a mistake, this parser always fails
   return NULL;
 }

src/pprint.c

@@ -96,7 +96,7 @@ static inline bool ensure_capacity(struct result_buf *buf, int amt) {
   return true;
 }
 
-static inline bool append_buf(struct result_buf *buf, const char* input, int len) {
+bool h_append_buf(struct result_buf *buf, const char* input, int len) {
   if (ensure_capacity(buf, len)) {
     memcpy(buf->output + buf->len, input, len);
     buf->len += len;
@@ -106,7 +106,7 @@ static inline bool append_buf(struct result_buf *buf, const char* input, int len
   }
 }
 
-static inline bool append_buf_c(struct result_buf *buf, char v) {
+bool h_append_buf_c(struct result_buf *buf, char v) {
   if (ensure_capacity(buf, 1)) {
     buf->output[buf->len++] = v;
     return true;
@@ -116,7 +116,7 @@ static inline bool append_buf_c(struct result_buf *buf, char v) {
 }
 
 /** append a formatted string to the result buffer */
-static inline bool append_buf_formatted(struct result_buf *buf, char* format, ...)
+bool h_append_buf_formatted(struct result_buf *buf, char* format, ...)
 {
   char* tmpbuf;
   int len;
@@ -125,7 +125,7 @@ static inline bool append_buf_formatted(struct result_buf *buf, char* format, ..
 
   va_start(ap, format);
   len = h_platform_vasprintf(&tmpbuf, format, ap);
-  result = append_buf(buf, tmpbuf, len);
+  result = h_append_buf(buf, tmpbuf, len);
   free(tmpbuf);
   va_end(ap);
 
@@ -134,52 +134,59 @@ static inline bool append_buf_formatted(struct result_buf *buf, char* format, ..
 
 static void unamb_sub(const HParsedToken* tok, struct result_buf *buf) {
   if (!tok) {
-    append_buf(buf, "NULL", 4);
+    h_append_buf(buf, "NULL", 4);
     return;
   }
   switch (tok->token_type) {
   case TT_NONE:
-    append_buf(buf, "null", 4);
+    h_append_buf(buf, "null", 4);
     break;
   case TT_BYTES:
     if (tok->bytes.len == 0)
-      append_buf(buf, "<>", 2);
+      h_append_buf(buf, "<>", 2);
     else {
       for (size_t i = 0; i < tok->bytes.len; i++) {
         const char *HEX = "0123456789abcdef";
-	append_buf_c(buf, (i == 0) ? '<': '.');
+        h_append_buf_c(buf, (i == 0) ? '<': '.');
         char c = tok->bytes.token[i];
-	append_buf_c(buf, HEX[(c >> 4) & 0xf]);
+        h_append_buf_c(buf, HEX[(c >> 4) & 0xf]);
-	append_buf_c(buf, HEX[(c >> 0) & 0xf]);
+        h_append_buf_c(buf, HEX[(c >> 0) & 0xf]);
       }
-      append_buf_c(buf, '>');
+      h_append_buf_c(buf, '>');
     }
     break;
   case TT_SINT:
     if (tok->sint < 0)
-      append_buf_formatted(buf, "s-%#" PRIx64, -tok->sint);
+      h_append_buf_formatted(buf, "s-%#" PRIx64, -tok->sint);
     else
-      append_buf_formatted(buf, "s%#" PRIx64, tok->sint);
+      h_append_buf_formatted(buf, "s%#" PRIx64, tok->sint);
     break;
   case TT_UINT:
-    append_buf_formatted(buf, "u%#" PRIx64, tok->uint);
+    h_append_buf_formatted(buf, "u%#" PRIx64, tok->uint);
     break;
   case TT_ERR:
-    append_buf(buf, "ERR", 3);
+    h_append_buf(buf, "ERR", 3);
     break;
   case TT_SEQUENCE: {
-    append_buf_c(buf, '(');
+    h_append_buf_c(buf, '(');
     for (size_t i = 0; i < tok->seq->used; i++) {
       if (i > 0)
-	append_buf_c(buf, ' ');
+        h_append_buf_c(buf, ' ');
       unamb_sub(tok->seq->elements[i], buf);
     }
-    append_buf_c(buf, ')');
+    h_append_buf_c(buf, ')');
   }
     break;
-  default:
+  default: {
-    fprintf(stderr, "Unexpected token type %d\n", tok->token_type);
+    const HTTEntry *e = h_get_token_type_entry(tok->token_type);
-    assert_message(0, "Should not reach here.");
+    if (e) {
+      h_append_buf_c(buf, '{');
+      e->unamb_sub(tok, buf);
+      h_append_buf_c(buf, '}');
+    } else {
+      assert_message(0, "Bogus token type.");
+    }
+  }
   }
 }
   
@@ -192,7 +199,7 @@ char* h_write_result_unamb(const HParsedToken* tok) {
   };
   assert(buf.output != NULL);
   unamb_sub(tok, &buf);
-  append_buf_c(&buf, 0);
+  h_append_buf_c(&buf, 0);
   return buf.output;
 }
   

src/registry.c

@@ -15,18 +15,19 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-#include <search.h>
 #include <stdlib.h>
 #include "hammer.h"
 #include "internal.h"
+#include "tsearch.h"
 
-typedef struct Entry_ {
+#if defined(_MSC_VER)
-  const char* name;
+#define h_strdup _strdup
-  HTokenType value;
+#else
-} Entry;
+#define h_strdup strdup
+#endif
 
 static void *tt_registry = NULL;
-static Entry** tt_by_id = NULL;
+static HTTEntry** tt_by_id = NULL;
 static unsigned int tt_by_id_sz = 0;
 #define TT_START TT_USER
 static HTokenType tt_next = TT_START;
@@ -34,23 +35,31 @@ static HTokenType tt_next = TT_START;
 /*
   // TODO: These are for the extension registry, which does not yet have a good name.
 static void *ext_registry = NULL;
-static Entry** ext_by_id = NULL;
+static HTTEntry** ext_by_id = NULL;
 static int ext_by_id_sz = 0;
 static int ext_next = 0;
 */
 
 
 static int compare_entries(const void* v1, const void* v2) {
-  const Entry *e1 = (Entry*)v1, *e2 = (Entry*)v2;
+  const HTTEntry *e1 = (HTTEntry*)v1, *e2 = (HTTEntry*)v2;
   return strcmp(e1->name, e2->name);
 }
 
-HTokenType h_allocate_token_type(const char* name) {
+static void default_unamb_sub(const HParsedToken* tok,
-  Entry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
+                              struct result_buf* buf) {
+  h_append_buf_formatted(buf, "XXX AMBIGUOUS USER TYPE %d", tok->token_type);
+}
+
+HTokenType h_allocate_token_new(
+    const char* name,
+    void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf)) {
+  HTTEntry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
   assert(new_entry != NULL);
   new_entry->name = name;
   new_entry->value = 0;
-  Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries);
+  new_entry->unamb_sub = unamb_sub;
+  HTTEntry* probe = *(HTTEntry**)tsearch(new_entry, &tt_registry, compare_entries);
   if (probe->value != 0) {
     // Token type already exists...
     // TODO: treat this as a bug?
@@ -58,7 +67,7 @@ HTokenType h_allocate_token_type(const char* name) {
     return probe->value;
   } else {
     // new value
-    probe->name = strdup(probe->name); // drop ownership of name
+    probe->name = h_strdup(probe->name); // drop ownership of name
     probe->value = tt_next++;
     if ((probe->value - TT_START) >= tt_by_id_sz) {
       if (tt_by_id_sz == 0) {
@@ -75,10 +84,13 @@ HTokenType h_allocate_token_type(const char* name) {
     return probe->value;
   }
 }
+HTokenType h_allocate_token_type(const char* name) {
+  return h_allocate_token_new(name, default_unamb_sub);
+}
 HTokenType h_get_token_type_number(const char* name) {
-  Entry e;
+  HTTEntry e;
   e.name = name;
-  Entry **ret = (Entry**)tfind(&e, &tt_registry, compare_entries);
+  HTTEntry **ret = (HTTEntry**)tfind(&e, &tt_registry, compare_entries);
   if (ret == NULL)
     return 0;
   else
@@ -90,3 +102,9 @@ const char* h_get_token_type_name(HTokenType token_type) {
   else
     return tt_by_id[token_type - TT_START]->name;
 }
+const HTTEntry* h_get_token_type_entry(HTokenType token_type) {
+  if (token_type >= tt_next || token_type < TT_START)
+    return NULL;
+  else
+    return tt_by_id[token_type - TT_START];
+}

src/search.h

@@ -0,0 +1,15 @@
+#if defined(_MSC_VER)
+/* find or insert datum into search tree */
+void *tsearch(const void *vkey, void **vrootp,
+              int (*compar)(const void *, const void *));
+
+/* delete node with given key */
+void * tdelete(const void *vkey, void **vrootp,
+               int (*compar)(const void *, const void *));
+
+/* Walk the nodes of a tree */
+void twalk(const void *vroot, void (*action)(const void *, VISIT, int));
+
+#else
+#include <search.h>
+#endif

src/tsearch.c

@@ -0,0 +1,141 @@
+/*	$OpenBSD: tsearch.c,v 1.9 2015/08/20 21:49:29 deraadt Exp $	*/
+
+/*
+ * Tree search generalized from Knuth (6.2.2) Algorithm T just like
+ * the AT&T man page says.
+ *
+ * The node_t structure is for internal use only
+ *
+ * Written by reading the System V Interface Definition, not the code.
+ *
+ * Totally public domain.
+ */
+
+#include <stdlib.h>
+#include "tsearch.h"
+
+typedef struct node_t {
+    char	  *key;
+    struct node_t *left, *right;
+} node;
+
+/* find or insert datum into search tree */
+void *
+tsearch(const void *vkey, void **vrootp,
+    int (*compar)(const void *, const void *))
+{
+    node *q;
+    char *key = (char *)vkey;
+    node **rootp = (node **)vrootp;
+
+    if (rootp == (struct node_t **)0)
+	return ((void *)0);
+    while (*rootp != (struct node_t *)0) {	/* Knuth's T1: */
+	int r;
+
+	if ((r = (*compar)(key, (*rootp)->key)) == 0)	/* T2: */
+	    return ((void *)*rootp);		/* we found it! */
+	rootp = (r < 0) ?
+	    &(*rootp)->left :		/* T3: follow left branch */
+	    &(*rootp)->right;		/* T4: follow right branch */
+    }
+    q = malloc(sizeof(node));	/* T5: key not found */
+    if (q != (struct node_t *)0) {	/* make new node */
+	*rootp = q;			/* link new node to old */
+	q->key = key;			/* initialize new node */
+	q->left = q->right = (struct node_t *)0;
+    }
+    return ((void *)q);
+}
+
+/* delete node with given key */
+void *
+tdelete(const void *vkey, void **vrootp,
+    int (*compar)(const void *, const void *))
+{
+    node **rootp = (node **)vrootp;
+    char *key = (char *)vkey;
+    node *p = (node *)1;
+    node *q;
+    node *r;
+    int cmp;
+
+    if (rootp == (struct node_t **)0 || *rootp == (struct node_t *)0)
+	return ((struct node_t *)0);
+    while ((cmp = (*compar)(key, (*rootp)->key)) != 0) {
+	p = *rootp;
+	rootp = (cmp < 0) ?
+	    &(*rootp)->left :		/* follow left branch */
+	    &(*rootp)->right;		/* follow right branch */
+	if (*rootp == (struct node_t *)0)
+	    return ((void *)0);		/* key not found */
+    }
+    r = (*rootp)->right;			/* D1: */
+    if ((q = (*rootp)->left) == (struct node_t *)0)	/* Left (struct node_t *)0? */
+	q = r;
+    else if (r != (struct node_t *)0) {		/* Right link is null? */
+	if (r->left == (struct node_t *)0) {	/* D2: Find successor */
+	    r->left = q;
+	    q = r;
+	} else {			/* D3: Find (struct node_t *)0 link */
+	    for (q = r->left; q->left != (struct node_t *)0; q = r->left)
+		r = q;
+	    r->left = q->right;
+	    q->left = (*rootp)->left;
+	    q->right = (*rootp)->right;
+	}
+    }
+    free((struct node_t *) *rootp);	/* D4: Free node */
+    *rootp = q;				/* link parent to new node */
+    return(p);
+}
+
+/* Walk the nodes of a tree */
+static void
+trecurse(node *root, void (*action)(const void *, VISIT, int), int level)
+{
+    if (root->left == (struct node_t *)0 && root->right == (struct node_t *)0)
+	(*action)(root, leaf, level);
+    else {
+	(*action)(root, preorder, level);
+	if (root->left != (struct node_t *)0)
+	    trecurse(root->left, action, level + 1);
+	(*action)(root, postorder, level);
+	if (root->right != (struct node_t *)0)
+	    trecurse(root->right, action, level + 1);
+	(*action)(root, endorder, level);
+    }
+}
+
+/* Walk the nodes of a tree */
+void
+twalk(const void *vroot, void (*action)(const void *, VISIT, int))
+{
+    node *root = (node *)vroot;
+
+    if (root != (node *)0 && action != (void (*)(const void *, VISIT, int))0)
+	trecurse(root, action, 0);
+}
+
+/*	$OpenBSD: tfind.c,v 1.6 2014/03/16 18:38:30 guenther Exp $	*/
+
+/* find a node, or return 0 */
+void *
+tfind(const void *vkey, void * const *vrootp,
+    int (*compar)(const void *, const void *))
+{
+    char *key = (char *)vkey;
+    node **rootp = (node **)vrootp;
+
+    if (rootp == (struct node_t **)0)
+	return ((struct node_t *)0);
+    while (*rootp != (struct node_t *)0) {	/* T1: */
+	int r;
+	if ((r = (*compar)(key, (*rootp)->key)) == 0)	/* T2: */
+	    return (*rootp);		/* key found */
+	rootp = (r < 0) ?
+	    &(*rootp)->left :		/* T3: follow left branch */
+	    &(*rootp)->right;		/* T4: follow right branch */
+    }
+    return (node *)0;
+}

src/tsearch.h

@@ -0,0 +1,26 @@
+#ifndef HAMMER_TSEARCH__H
+#define HAMMER_TSEARCH__H
+
+#if defined(_MSC_VER)
+typedef enum { preorder, postorder, endorder, leaf } VISIT;
+
+/* find or insert datum into search tree */
+void *tsearch(const void *vkey, void **vrootp,
+              int (*compar)(const void *, const void *));
+
+/* delete node with given key */
+void * tdelete(const void *vkey, void **vrootp,
+               int (*compar)(const void *, const void *));
+
+/* Walk the nodes of a tree */
+void twalk(const void *vroot, void (*action)(const void *, VISIT, int));
+
+/* find a node, or return 0 */
+void *tfind(const void *vkey, void * const *vrootp,
+            int (*compar)(const void *, const void *));
+
+#else
+#include <search.h>
+#endif
+
+#endif /* HAMMER_TSEARCH__H */

tools/windows/build.bat

@@ -24,8 +24,8 @@ cl.exe -nologo -FC -EHsc -Z7 -Oi -GR- -Gm- %CLFLAGS% -c ^
        -Fo%BUILD%\obj\
 if %errorlevel% neq 0 goto err
 
-lib.exe %BUILD%\obj\*.obj -OUT:%BUILD%\hammer.lib
+lib.exe %BUILD%\obj\*.obj -OUT:%BUILD%\lib\hammer_s.lib
-echo STATIC_LIBRARY %BUILD%\hammer.lib
+echo STATIC_LIBRARY %BUILD%\lib\hammer_s.lib
 if %errorlevel% neq 0 goto err
 popd
 

tools/windows/build_examples.bat

@@ -15,7 +15,7 @@ call %HEREPATH%\clvars.bat
 echo SRC=%SRC%, BUILD=%BUILD%
 echo CLFLAGS=%CLFLAGS%
 
-set HAMMERLIB=%BUILD%\hammer.lib
+set HAMMERLIB=%BUILD%\lib\hammer_s.lib
 
 REM Now let's build some example programs
 

tools/windows/clvars.bat

@@ -7,6 +7,16 @@ set WARNINGS=-W4 -Wall -WX
 REM c4464 (relative include path contains '..')
 set WARNINGS=%WARNINGS% -wd4464
 
+REM c4189 (local variable is initialized but not referenced)
+set WARNINGS=%WARNINGS% -wd4189
+
+REM c4018/c4388 (signed/unsigned mismatch)
+REM basically useless. Complains about obviously correct code like:
+REM     uint8_t x = 60;
+REM     size_t i = 9;
+REM     i < x/8
+set WARNINGS=%WARNINGS% -wd4018 -wd4388
+
 REM c4457 (declaration shadowing function parameter)
 REM FIXME(windows) TODO(uucidl): remove occurence of c4457 and reactivate
 REM FIXME(windows) TODO(uucidl): remove occurence of c4456 and reactivate

tools/windows/hammer_lib_src_list

@@ -4,17 +4,23 @@ benchmark.c
 bitreader.c 
 bitwriter.c 
 cfgrammar.c
+datastructures.c
 desugar.c 
 glue.c 
 hammer.c 
 pprint.c
+registry.c
 system_allocator.c
+tsearch.c
 parsers/action.c 
 parsers/and.c 
 parsers/attr_bool.c 
+parsers/bind.c
+parsers/bits.c
 parsers/butnot.c 
 parsers/ch.c 
 parsers/charset.c 
+parsers/choice.c
 parsers/difference.c 
 parsers/end.c 
 parsers/endianness.c 
@@ -25,6 +31,7 @@ parsers/indirect.c
 parsers/int_range.c 
 parsers/many.c 
 parsers/not.c 
+parsers/nothing.c
 parsers/optional.c 
 parsers/permutation.c 	
 parsers/sequence.c