emileclarkb/hammer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added support for the token type registry

Browse source
This commit is contained in:
Dan Hirsch 2014-01-11 01:09:45 +01:00
parent 753120f27e
commit 482e89adf8
3 changed files with 141 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/bindings/ruby/hammer-parser.gemspec
View file

@ -11,7 +11,15 @@ Gem::Specification.new do |s|
files = []
files << 'README.md'
files << Dir['{lib,test}/**/*.rb']
files << [
"lib/hammer/internal.rb",
"lib/hammer/parser.rb",
"lib/hammer/parser_builder.rb",
"lib/hammer.rb",
"lib/minitest/hamer-parser_plugin.rb",
"test/autogen_test.rb",
"test/parser_test.rb"
]
s.files = files
s.test_files = s.files.select { |path| path =~ /^test\/.*_test.rb/ }

121
src/bindings/ruby/lib/hammer/internal.rb
View file

@ -6,22 +6,123 @@ module Hammer
ffi_lib 'hammer'
class DynamicVariable
SYMBOL_PREFIX = "Hammer::Internal::DynamicVariable gensym "
@@current_symbol = 0
def initialize(default=nil, name=nil, &block)
# This can take either a default value or a block. If a
# default value is given, all threads' dynvars are initialized
# to that object. If a block is given, the block is lazilly
# called on each thread to generate the initial value. If
# both a block and a default value are passed, the block is
# called with the literal value.
@default = default
@block = block || Proc.new{|x| x}
@@current_symbol += 1
@sym = (SYMBOL_PREFIX + @@current_symbol.to_s).to_sym
end
def value
if Thread.current.key? @sym
return Thread.current[@sym]
else
return Thread.current[@sym] = @block.call(@default)
end
end
def value=(new_value)
Thread.current[@sym] = new_value
end
def with(new_value, &block)
old_value = value
begin
self.value = new_value
return block.call
ensure
self.value = old_value
end
end
end
# Maybe we can implement Hammer::Parser with FFI::DataConverter.
# That way, most hammer functions won't need to be wrapped.
# (Probably need to wrap token, sequence and choice only).
# See http://www.elabs.se/blog/61-advanced-topics-in-ruby-ffi
typedef :pointer, :h_parser
HTokenType = enum(:none, 1,
:bytes, 2,
:sint, 4,
:uint, 8,
:sequence, 16,
:reserved_1,
:err, 32,
:user, 64,
:max)
class HTokenType
extend FFI::DataConverter
@@known_type_map = {
:none => 1,
:bytes => 2,
:sint => 4,
:uint => 8,
:sequence => 16,
}
@@inverse_type_map = @@known_type_map.invert
def self.new(name)
if name.is_a?(Symbol)
name_sym = name
name_str = name.to_s
else
name_str = name.to_s
name_sym = name.to_sym
end
num = h_allocate_token_type(name_str)
@@known_type_map[name_sym] = num
@@inverse_type_map[num] = name
end
def self.from_name(name)
unless @@known_type_map.key? name
num = h_get_token_type_number(name.to_s)
if num <= 0
raise ArgumentError, "Unknown token type #{name}"
end
@@known_type_map[name] = num
@@inverse_type_map[num] = name
end
return @@known_type_map[name]
end
def self.from_num(num)
unless @@inverse_type_map.key? num
name = h_get_token_type_name(num)
if name.nil?
return nil
end
name = name.to_sym
@@known_type_map[name] = num
@@inverse_type_map_type_map[num] = name
end
return @@inverse_type_map[num]
end
def self.native_type
FFI::Type::INT
end
def self.to_native(val, ctx)
return val if val.is_a?(Integer)
return from_name(val)
end
def self.from_native(val, ctx)
return from_num(val) || val
end
end
# Define these as soon as possible, so that they can be used
# without fear elsewhere
attach_function :h_allocate_token_type, [:string], HTokenType
attach_function :h_get_token_type_number, [:string], HTokenType
attach_function :h_get_token_type_name, [HTokenType], :string
class HCountedArray < FFI::Struct
layout :capacity, :size_t,
:used, :size_t,
@ -213,5 +314,7 @@ module Hammer
attach_function :h_parse_result_free, [HParseResult.by_ref], :void
# TODO: Does the HParser* need to be freed?
# Token type registry
end
end

33
src/bindings/ruby/lib/hammer/parser.rb
View file

@ -1,6 +1,10 @@
require 'hammer/internal'
module Hammer
class Parser
@@saved_objects = Hammer::Internal::DynamicVariable.new nil, "Hammer parse-time pins"
# Don't create new instances with Hammer::Parser.new,
# use the constructor methods instead (i.e. Hammer::Parser.int64 etc.)
#
@ -26,17 +30,20 @@ module Hammer
raise ArgumentError, 'expecting a String' unless data.is_a? String # TODO: Not needed, FFI checks that.
ibuf = FFI::MemoryPointer.from_string(data)
result = Hammer::Internal.h_parse(@h_parser, ibuf, data.bytesize) # Don't include the trailing null
if result.null?
return nil
else
# NOTE:
# The parse result *must* hold a reference to the parser that created it!
# Otherwise, the parser might get garbage-collected while the result is still valid.
# Any pointers to token strings will then be invalid.
result.instance_variable_set :@parser, self
return result
end
@@saved_objects.with([]) do
result = Hammer::Internal.h_parse(@h_parser, ibuf, data.bytesize) # Don't include the trailing null
if result.null?
return nil
else
# NOTE:
# The parse result *must* hold a reference to the parser that created it!
# Otherwise, the parser might get garbage-collected while the result is still valid.
# Any pointers to token strings will then be invalid.
result.instance_variable_set :@parser, self
result.instance_variable_set :@pins, @@saved_objects.value
return result
end
end
end
# Binds an indirect parser.
@ -71,7 +78,7 @@ module Hammer
buffer = FFI::MemoryPointer.from_string(string)
h_parser = Hammer::Internal.h_token(buffer, buffer.size-1) # buffer.size includes the null byte at the end
return Hammer::Parser.new(:token, h_parser, buffer)
return Hammer::Parser.new(:token, h_parser, [buffer, string])
end
def self.marshal_ch_arg(num)
@ -100,7 +107,7 @@ module Hammer
def self.int_range(parser, i1, i2)
h_parser = Hammer::Internal.h_int_range(parser.h_parser, i1, i2)
return Hammer::Parser.new(:int_range, h_parser, nil)
return Hammer::Parser.new(:int_range, h_parser, [parser])
end
def self.in(charset)