From 30cc709b8cdbeaf4a83088eea16262f646ad15f4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Sun, 27 Dec 2015 14:45:21 +0100 Subject: [PATCH 01/11] add SLOB allocator --- src/SConscript | 6 +- src/hammer.h | 3 + src/sloballoc.c | 216 +++++++++++++++++++++++++++++++++++++++++++++++ src/sloballoc.h | 15 ++++ src/t_mm.c | 148 ++++++++++++++++++++++++++++++++ src/test_suite.c | 2 + src/test_suite.h | 1 + 7 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 src/sloballoc.c create mode 100644 src/sloballoc.h create mode 100644 src/t_mm.c diff --git a/src/SConscript b/src/SConscript index 7a1b9d4..3d25452 100644 --- a/src/SConscript +++ b/src/SConscript @@ -66,7 +66,8 @@ misc_hammer_parts = [ 'platform_bsdlike.c', 'pprint.c', 'registry.c', - 'system_allocator.c'] + 'system_allocator.c', + 'sloballoc.c'] ctests = ['t_benchmark.c', 't_bitreader.c', @@ -74,7 +75,8 @@ ctests = ['t_benchmark.c', 't_parser.c', 't_grammar.c', 't_misc.c', - 't_regression.c'] + 't_mm.c', + 't_regression.c'] libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts) libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts) diff --git a/src/hammer.h b/src/hammer.h index 1be297c..6c2bf49 100644 --- a/src/hammer.h +++ b/src/hammer.h @@ -789,6 +789,9 @@ HTokenType h_get_token_type_number(const char* name); const char* h_get_token_type_name(HTokenType token_type); // }}} +/// Make an allocator that draws from the given memory area. +HAllocator *h_sloballoc(void *mem, size_t size); + #ifdef __cplusplus } #endif diff --git a/src/sloballoc.c b/src/sloballoc.c new file mode 100644 index 0000000..8b6cbc5 --- /dev/null +++ b/src/sloballoc.c @@ -0,0 +1,216 @@ +// first-fit SLOB (simple list of blocks) allocator + +#include "sloballoc.h" +#include +#include + +struct alloc { + size_t size; + uint8_t data[]; +}; + +struct block { + struct alloc alloc; + struct block *next; +}; + +struct slob { + size_t size; + struct block *head; + uint8_t data[]; +}; + + +SLOB *slobinit(void *mem, size_t size) +{ + SLOB *slob = mem; + + assert(size >= sizeof(SLOB) + sizeof(struct block)); + assert(size < UINTPTR_MAX - (uintptr_t)mem); + + slob = mem; + slob->size = size - sizeof(SLOB); + slob->head = mem + sizeof(SLOB); + slob->head->alloc.size = slob->size - sizeof(struct alloc); + slob->head->next = NULL; + + return slob; +} + +void *sloballoc(SLOB *slob, size_t size) +{ + struct block *b, **p; + size_t fitblock, remblock; + + // size must be enough to extend to a struct block in case of free + fitblock = sizeof(struct block) - sizeof(struct alloc); + if(size < fitblock) size = fitblock; + + // need this much to fit another block in the remaining space + remblock = size + sizeof(struct block); + if(remblock < size) return NULL; // overflow + + // scan list for the first block of sufficient size + for(p=&slob->head; (b=*p); p=&b->next) { + if(b->alloc.size >= remblock) { + // cut from the end of the block + b->alloc.size -= sizeof(struct alloc) + size; + struct alloc *a = (void *)b->alloc.data + b->alloc.size; + a->size = size; + return a->data; + } else if(b->alloc.size >= size) { + // when a block fills, it converts directly to a struct alloc + *p = b->next; // unlink + return b->alloc.data; + } + } + + return NULL; +} + +void slobfree(SLOB *slob, void *a_) +{ + struct alloc *a = a_ - sizeof(struct alloc); + struct block *b, **p, *left=NULL, *right=NULL, **rightp; + + // sanity check: a lies inside slob + assert((void *)a >= (void *)slob->data); + assert((void *)a->data + a->size <= (void *)slob->data + slob->size); + + // scan list for blocks adjacent to a + for(p=&slob->head; (b=*p); p=&b->next) { + if((void *)a == b->alloc.data + b->alloc.size) { + assert(!left); + left = b; + } + if((void *)a->data + a->size == b) { + assert(!right); + right = b; + rightp = p; + } + + if(left && right) { + // extend left and unlink right + left->alloc.size += sizeof(*a) + a->size + + sizeof(right->alloc) + right->alloc.size; + *rightp = right->next; + return; + } + } + + if(left) { + // extend left to absorb a + left->alloc.size += sizeof(*a) + a->size; + } else if(right) { + // shift and extend right to absorb a + right->alloc.size += sizeof(*a) + a->size; + *rightp = (struct block *)a; **rightp = *right; + } else { + // spawn new block over a + struct block *b = (struct block *)a; + b->next = slob->head; slob->head = b; + } +} + +int slobcheck(SLOB *slob) +{ + // invariants: + // 1. memory area is divided seamlessly and exactly into n blocks + // 2. every block is large enough to hold a 'struct block'. + // 3. free list has at most n elements. + // 4. every element of the free list is one of the valid blocks. + // 5. every block appears at most once in the free list. + + void *p; + size_t nblocks=0, nfree=0; + + #define FORBLOCKS \ + for(p = slob->data; \ + p != slob->data + slob->size; \ + p += sizeof(struct alloc) + ((struct alloc *)p)->size) + + // 1. memory area is divided seamlessly and exactly into n blocks + FORBLOCKS { + if(p < (void *)slob->data) + return 1; + if(p > (void *)slob->data + slob->size) + return 2; + nblocks++; + + struct alloc *a = p; + if(a->size > UINTPTR_MAX - (uintptr_t)p) + return 3; + + // 2. every block is large enough to hold a 'struct block'. + if(a->size + sizeof(struct alloc) < sizeof(struct block)) + return 4; + } + + // 3. free list has at most n elements. + for(struct block *b=slob->head; b; b=b->next) { + nfree++; + if(nfree > nblocks) + return 5; + + // 4. every element of the free list is one of the valid blocks. + FORBLOCKS + if(p == b) break; + if(!p) + return 6; + } + + // 5. every block appears at most once in the free list. + FORBLOCKS { + size_t count=0; + for(struct block *b=slob->head; b; b=b->next) + if(p == b) count++; + if(count > 1) + return 7; + } + + #undef FORBLOCKS + return 0; +} + + +// hammer interface + +#include "hammer.h" + +static void *h_slob_alloc(HAllocator *mm, size_t size) +{ + SLOB *slob = (SLOB *)(mm+1); + return sloballoc(slob, size); +} + +static void h_slob_free(HAllocator *mm, void *p) +{ + SLOB *slob = (SLOB *)(mm+1); + slobfree(slob, p); +} + +static void *h_slob_realloc(HAllocator *mm, void *p, size_t size) +{ + SLOB *slob = (SLOB *)(mm+1); + + assert(((void)"XXX need realloc for SLOB allocator", 0)); + return NULL; +} + +HAllocator *h_sloballoc(void *mem, size_t size) +{ + if(size < sizeof(HAllocator)) + return NULL; + + HAllocator *mm = mem; + SLOB *slob = slobinit(mem + sizeof(HAllocator), size - sizeof(HAllocator)); + if(!slob) + return NULL; + assert(slob == (SLOB *)(mm+1)); + + mm->alloc = h_slob_alloc; + mm->realloc = h_slob_realloc; + mm->free = h_slob_free; + + return mm; +} diff --git a/src/sloballoc.h b/src/sloballoc.h new file mode 100644 index 0000000..ecdc479 --- /dev/null +++ b/src/sloballoc.h @@ -0,0 +1,15 @@ +#ifndef SLOBALLOC_H_SEEN +#define SLOBALLOC_H_SEEN + +#include + +typedef struct slob SLOB; + +SLOB *slobinit(void *mem, size_t size); +void *sloballoc(SLOB *slob, size_t size); +void slobfree(SLOB *slob, void *p); + +// consistency check (verify internal invariants); returns 0 on success +int slobcheck(SLOB *slob); + +#endif // SLOBALLOC_H_SEEN diff --git a/src/t_mm.c b/src/t_mm.c new file mode 100644 index 0000000..620d4e8 --- /dev/null +++ b/src/t_mm.c @@ -0,0 +1,148 @@ +#include +#include +#include "test_suite.h" +#include "sloballoc.h" +#include "hammer.h" + +#define check_sloballoc_invariants() do { \ + int err = slobcheck(slob); \ + if(err) { \ + g_test_message("SLOB invariant check failed on line %d, returned %d", \ + __LINE__, err); \ + g_test_fail(); \ + } \ + } while(0) + +#define check_sloballoc(VAR, SIZE, OFFSET) do { \ + check_sloballoc_invariants(); \ + VAR = sloballoc(slob, (SIZE)); \ + g_check_cmp_ptr(VAR, ==, mem + (OFFSET)); \ + } while(0) + +#define check_sloballoc_fail(SIZE) do { \ + check_sloballoc_invariants(); \ + void *p = sloballoc(slob, (SIZE)); \ + g_check_cmp_ptr(p, ==, NULL); \ + } while(0) + +#define check_slobfree(P) do { \ + check_sloballoc_invariants(); \ + slobfree(slob, P); \ + } while(0) + +#define N 1024 + +#define SLOBALLOC_FIXTURE \ + static uint8_t mem[N] = {0x58}; \ + SLOB *slob = slobinit(mem, N); \ + size_t max = N - 2*sizeof(size_t) - sizeof(void *); \ + (void)max; /* silence warning */ \ + if(!slob) { \ + g_test_message("SLOB allocator init failed on line %d", __LINE__); \ + g_test_fail(); \ + } + +static void test_sloballoc_size(void) +{ + SLOBALLOC_FIXTURE + void *p; + + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_fail(N); + check_sloballoc_fail(max+1); + + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_invariants(); +} + +static void test_sloballoc_merge(void) +{ + SLOBALLOC_FIXTURE + void *p, *q, *r; + + check_sloballoc(p, 100, N-100); + check_slobfree(p); + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc(p, 100, N-100); + check_sloballoc(q, 100, N-200-sizeof(size_t)); + check_slobfree(p); + check_sloballoc(p, 50, N-50); + check_sloballoc(r, 100, N-300-2*sizeof(size_t)); + check_slobfree(q); + check_sloballoc(q, 150, N-200-sizeof(size_t)); + check_slobfree(p); + check_slobfree(r); + check_slobfree(q); // merge left and right + + check_sloballoc_fail(max+1); + check_sloballoc(p, max, N-max); + check_slobfree(p); + + check_sloballoc_invariants(); +} + +static void test_sloballoc_small(void) +{ + SLOBALLOC_FIXTURE + void *p, *q, *r; + + check_sloballoc(p, 100, N-100); + check_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_sloballoc(r, 100, N-200-2*sizeof(size_t)-sizeof(void *)); + check_slobfree(q); + check_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_slobfree(p); + check_slobfree(r); + + check_sloballoc_invariants(); +} + +#define check_h_sloballoc(VAR, SIZE, OFFSET) do { \ + check_sloballoc_invariants(); \ + VAR = mm->alloc(mm, (SIZE)); \ + g_check_cmp_ptr(VAR, ==, mem + (OFFSET)); \ + } while(0) + +#define check_h_slobfree(P) do { \ + check_sloballoc_invariants(); \ + mm->free(mm, P); \ + } while(0) + +static void test_sloballoc_hammer(void) +{ + static uint8_t mem[N] = {0x58}; + HAllocator *mm = h_sloballoc(mem, N); int line = __LINE__; + SLOB *slob = ((void *)mm) + sizeof(HAllocator); + void *p, *q, *r; + + if(!mm) { + g_test_message("h_sloballoc() failed on line %d", line); + g_test_fail(); + } + + check_h_sloballoc(p, 100, N-100); + check_h_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_h_sloballoc(r, 100, N-200-2*sizeof(size_t)-sizeof(void *)); + check_h_slobfree(q); + check_h_sloballoc(q, 1, N-100-sizeof(size_t)-sizeof(void *)); + check_h_slobfree(p); + check_h_slobfree(r); + + check_sloballoc_invariants(); +} + +#undef N + +void register_mm_tests(void) { + g_test_add_func("/core/mm/sloballoc/size", test_sloballoc_size); + g_test_add_func("/core/mm/sloballoc/merge", test_sloballoc_merge); + g_test_add_func("/core/mm/sloballoc/small", test_sloballoc_small); + g_test_add_func("/core/mm/sloballoc/hammer", test_sloballoc_hammer); +} + diff --git a/src/test_suite.c b/src/test_suite.c index cba18e8..f569644 100644 --- a/src/test_suite.c +++ b/src/test_suite.c @@ -24,6 +24,7 @@ extern void register_bitwriter_tests(); extern void register_parser_tests(); extern void register_grammar_tests(); extern void register_misc_tests(); +extern void register_mm_tests(); extern void register_benchmark_tests(); extern void register_regression_tests(); @@ -36,6 +37,7 @@ int main(int argc, char** argv) { register_parser_tests(); register_grammar_tests(); register_misc_tests(); + register_mm_tests(); register_regression_tests(); if (g_test_slow() || g_test_perf()) register_benchmark_tests(); diff --git a/src/test_suite.h b/src/test_suite.h index 83359f9..ed640fd 100644 --- a/src/test_suite.h +++ b/src/test_suite.h @@ -321,6 +321,7 @@ #define g_check_cmp_int64(n1, op, n2) g_check_inttype("%" PRId64, int64_t, n1, op, n2) #define g_check_cmp_uint32(n1, op, n2) g_check_inttype("%u", uint32_t, n1, op, n2) #define g_check_cmp_uint64(n1, op, n2) g_check_inttype("%" PRIu64, uint64_t, n1, op, n2) +#define g_check_cmp_ptr(n1, op, n2) g_check_inttype("%p", void *, n1, op, n2) #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2) #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2) From d9c4492fd9581ef5b804269f0e48a628173f22c4 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Tue, 6 Dec 2016 18:21:59 +0100 Subject: [PATCH 02/11] silence gcc about "possible" (impossible) use of uninitialized variable --- src/sloballoc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sloballoc.c b/src/sloballoc.c index 8b6cbc5..d42f50f 100644 --- a/src/sloballoc.c +++ b/src/sloballoc.c @@ -71,7 +71,7 @@ void *sloballoc(SLOB *slob, size_t size) void slobfree(SLOB *slob, void *a_) { struct alloc *a = a_ - sizeof(struct alloc); - struct block *b, **p, *left=NULL, *right=NULL, **rightp; + struct block *b, **p, *left=NULL, *right=NULL, **rightp=NULL; // sanity check: a lies inside slob assert((void *)a >= (void *)slob->data); From fedb36ed897ac5be625ce17898e69e725507d676 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 7 Dec 2016 12:00:27 +0100 Subject: [PATCH 03/11] avoid arithmetic on void pointers to be more standard-conforming --- src/sloballoc.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sloballoc.c b/src/sloballoc.c index d42f50f..24537ff 100644 --- a/src/sloballoc.c +++ b/src/sloballoc.c @@ -55,7 +55,7 @@ void *sloballoc(SLOB *slob, size_t size) if(b->alloc.size >= remblock) { // cut from the end of the block b->alloc.size -= sizeof(struct alloc) + size; - struct alloc *a = (void *)b->alloc.data + b->alloc.size; + struct alloc *a = (struct alloc *)(b->alloc.data + b->alloc.size); a->size = size; return a->data; } else if(b->alloc.size >= size) { @@ -74,16 +74,16 @@ void slobfree(SLOB *slob, void *a_) struct block *b, **p, *left=NULL, *right=NULL, **rightp=NULL; // sanity check: a lies inside slob - assert((void *)a >= (void *)slob->data); - assert((void *)a->data + a->size <= (void *)slob->data + slob->size); + assert((uint8_t *)a >= slob->data); + assert(a->data + a->size <= slob->data + slob->size); // scan list for blocks adjacent to a for(p=&slob->head; (b=*p); p=&b->next) { - if((void *)a == b->alloc.data + b->alloc.size) { + if((uint8_t *)a == b->alloc.data + b->alloc.size) { assert(!left); left = b; } - if((void *)a->data + a->size == b) { + if(a->data + a->size == (uint8_t *)b) { assert(!right); right = b; rightp = p; @@ -121,7 +121,7 @@ int slobcheck(SLOB *slob) // 4. every element of the free list is one of the valid blocks. // 5. every block appears at most once in the free list. - void *p; + uint8_t *p; size_t nblocks=0, nfree=0; #define FORBLOCKS \ @@ -131,13 +131,13 @@ int slobcheck(SLOB *slob) // 1. memory area is divided seamlessly and exactly into n blocks FORBLOCKS { - if(p < (void *)slob->data) + if(p < slob->data) return 1; - if(p > (void *)slob->data + slob->size) + if(p > slob->data + slob->size) return 2; nblocks++; - struct alloc *a = p; + struct alloc *a = (struct alloc *)p; if(a->size > UINTPTR_MAX - (uintptr_t)p) return 3; @@ -154,7 +154,7 @@ int slobcheck(SLOB *slob) // 4. every element of the free list is one of the valid blocks. FORBLOCKS - if(p == b) break; + if(p == (uint8_t *)b) break; if(!p) return 6; } @@ -163,7 +163,7 @@ int slobcheck(SLOB *slob) FORBLOCKS { size_t count=0; for(struct block *b=slob->head; b; b=b->next) - if(p == b) count++; + if(p == (uint8_t *)b) count++; if(count > 1) return 7; } From ec8249513c0c5da126972e737ba3ba3dd600e413 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 7 Dec 2016 12:06:53 +0100 Subject: [PATCH 04/11] avoid more void pointer arithmetic --- src/sloballoc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sloballoc.c b/src/sloballoc.c index 24537ff..d16a879 100644 --- a/src/sloballoc.c +++ b/src/sloballoc.c @@ -30,7 +30,7 @@ SLOB *slobinit(void *mem, size_t size) slob = mem; slob->size = size - sizeof(SLOB); - slob->head = mem + sizeof(SLOB); + slob->head = (struct block *)((uint8_t *)mem + sizeof(SLOB)); slob->head->alloc.size = slob->size - sizeof(struct alloc); slob->head->next = NULL; @@ -70,7 +70,7 @@ void *sloballoc(SLOB *slob, size_t size) void slobfree(SLOB *slob, void *a_) { - struct alloc *a = a_ - sizeof(struct alloc); + struct alloc *a = (struct alloc *)((uint8_t *)a_ - sizeof(struct alloc)); struct block *b, **p, *left=NULL, *right=NULL, **rightp=NULL; // sanity check: a lies inside slob @@ -203,7 +203,7 @@ HAllocator *h_sloballoc(void *mem, size_t size) return NULL; HAllocator *mm = mem; - SLOB *slob = slobinit(mem + sizeof(HAllocator), size - sizeof(HAllocator)); + SLOB *slob = slobinit((uint8_t *)mem + sizeof(HAllocator), size - sizeof(HAllocator)); if(!slob) return NULL; assert(slob == (SLOB *)(mm+1)); From 631ca40f9170b10491db442ff3395b8f26cff4c3 Mon Sep 17 00:00:00 2001 From: "Sven M. Hallberg" Date: Wed, 7 Dec 2016 12:39:49 +0100 Subject: [PATCH 05/11] avoid embedding data[] in struct block to satisfy windows build --- src/sloballoc.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/sloballoc.c b/src/sloballoc.c index d16a879..e164cab 100644 --- a/src/sloballoc.c +++ b/src/sloballoc.c @@ -10,7 +10,7 @@ struct alloc { }; struct block { - struct alloc alloc; + size_t size; // read: struct alloc struct block *next; }; @@ -31,7 +31,7 @@ SLOB *slobinit(void *mem, size_t size) slob = mem; slob->size = size - sizeof(SLOB); slob->head = (struct block *)((uint8_t *)mem + sizeof(SLOB)); - slob->head->alloc.size = slob->size - sizeof(struct alloc); + slob->head->size = slob->size - sizeof(struct alloc); slob->head->next = NULL; return slob; @@ -52,16 +52,16 @@ void *sloballoc(SLOB *slob, size_t size) // scan list for the first block of sufficient size for(p=&slob->head; (b=*p); p=&b->next) { - if(b->alloc.size >= remblock) { + if(b->size >= remblock) { // cut from the end of the block - b->alloc.size -= sizeof(struct alloc) + size; - struct alloc *a = (struct alloc *)(b->alloc.data + b->alloc.size); + b->size -= sizeof(struct alloc) + size; + struct alloc *a = (struct alloc *)(((struct alloc *)b)->data + b->size); a->size = size; return a->data; - } else if(b->alloc.size >= size) { + } else if(b->size >= size) { // when a block fills, it converts directly to a struct alloc *p = b->next; // unlink - return b->alloc.data; + return ((struct alloc *)b)->data; } } @@ -79,7 +79,7 @@ void slobfree(SLOB *slob, void *a_) // scan list for blocks adjacent to a for(p=&slob->head; (b=*p); p=&b->next) { - if((uint8_t *)a == b->alloc.data + b->alloc.size) { + if((uint8_t *)a == ((struct alloc *)b)->data + b->size) { assert(!left); left = b; } @@ -91,8 +91,8 @@ void slobfree(SLOB *slob, void *a_) if(left && right) { // extend left and unlink right - left->alloc.size += sizeof(*a) + a->size + - sizeof(right->alloc) + right->alloc.size; + left->size += sizeof(*a) + a->size + + sizeof(struct alloc) + right->size; *rightp = right->next; return; } @@ -100,10 +100,10 @@ void slobfree(SLOB *slob, void *a_) if(left) { // extend left to absorb a - left->alloc.size += sizeof(*a) + a->size; + left->size += sizeof(*a) + a->size; } else if(right) { // shift and extend right to absorb a - right->alloc.size += sizeof(*a) + a->size; + right->size += sizeof(*a) + a->size; *rightp = (struct block *)a; **rightp = *right; } else { // spawn new block over a From 0f3cadcc3eaa9add2961219fd5e52919499ba487 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 21:13:32 +0100 Subject: [PATCH 06/11] Enable absolute imports, true division, & print() These have no effect in Python 3.x, they are the default. Enabling them in Python 2.x, enabling them in Python 2.x allows single source compatiblity. --- SConstruct | 9 ++++++--- examples/SConscript | 2 ++ examples/base64.py | 2 +- examples/base64_sem1.py | 2 +- examples/base64_sem2.py | 2 +- src/SConscript | 3 +++ src/bindings/cpp/SConscript | 3 +++ src/bindings/dotnet/SConscript | 3 +++ src/bindings/perl/SConscript | 3 +++ src/bindings/php/SConscript | 3 +++ src/bindings/python/SConscript | 3 +++ src/bindings/python/hammer_tests.py | 2 ++ src/bindings/ruby/SConscript | 3 +++ tools/csharp/__init__.py | 4 +++- tools/csharp/csharp.py | 2 ++ tools/csharp/mono.py | 2 ++ tools/scanreplace.py | 2 ++ 17 files changed, 43 insertions(+), 7 deletions(-) diff --git a/SConstruct b/SConstruct index 41b467e..ae85cda 100644 --- a/SConstruct +++ b/SConstruct @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os import os.path import platform @@ -43,9 +46,9 @@ env['prefix'] = os.path.abspath(env['prefix']) if 'DESTDIR' in env: env['DESTDIR'] = os.path.abspath(env['DESTDIR']) if rel_prefix: - print >>sys.stderr, '--!!-- You used a relative prefix with a DESTDIR. This is probably not what you' - print >>sys.stderr, '--!!-- you want; files will be installed in' - print >>sys.stderr, '--!!-- %s' % (calcInstallPath('$prefix'),) + print('--!!-- You used a relative prefix with a DESTDIR. This is probably not what you', file=sys.stderr) + print('--!!-- you want; files will be installed in', file=sys.stderr) + print('--!!-- %s' % (calcInstallPath('$prefix'),), file=sys.stderr) env['libpath'] = calcInstallPath('$prefix', 'lib') diff --git a/examples/SConscript b/examples/SConscript index 0694721..b34b85a 100644 --- a/examples/SConscript +++ b/examples/SConscript @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function + Import('env') example = env.Clone() diff --git a/examples/base64.py b/examples/base64.py index 3ffe304..0233ff7 100644 --- a/examples/base64.py +++ b/examples/base64.py @@ -10,7 +10,7 @@ # base64_sem1.py and base64_sem2.py for examples how to attach appropriate # semantic actions to the grammar. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import sys diff --git a/examples/base64_sem1.py b/examples/base64_sem1.py index f0676eb..2c4d6e0 100644 --- a/examples/base64_sem1.py +++ b/examples/base64_sem1.py @@ -13,7 +13,7 @@ # transform the parse tree in small steps in a bottom-up fashion. Compare # base64_sem2.py for an alternative approach using a single top-level action. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import sys diff --git a/examples/base64_sem2.py b/examples/base64_sem2.py index 6b5f8db..3b023dd 100644 --- a/examples/base64_sem2.py +++ b/examples/base64_sem2.py @@ -14,7 +14,7 @@ # for an alternative approach using a fine-grained piece-by-piece # transformation. -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import sys diff --git a/src/SConscript b/src/SConscript index a46ddac..4bd6c28 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import('env testruns') diff --git a/src/bindings/cpp/SConscript b/src/bindings/cpp/SConscript index 9555b98..385759e 100644 --- a/src/bindings/cpp/SConscript +++ b/src/bindings/cpp/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/dotnet/SConscript b/src/bindings/dotnet/SConscript index 94f874e..62e4b22 100644 --- a/src/bindings/dotnet/SConscript +++ b/src/bindings/dotnet/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/perl/SConscript b/src/bindings/perl/SConscript index 49b693a..e3e37c6 100644 --- a/src/bindings/perl/SConscript +++ b/src/bindings/perl/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/src/bindings/php/SConscript b/src/bindings/php/SConscript index 34728af..2846800 100644 --- a/src/bindings/php/SConscript +++ b/src/bindings/php/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os, os.path Import('env libhammer_shared testruns') diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index dac2d95..d5d092d 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os, os.path Import('env libhammer_shared testruns targets') diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index 45a63f4..ee79caa 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function + import unittest import hammer as h diff --git a/src/bindings/ruby/SConscript b/src/bindings/ruby/SConscript index 6d85a93..d50b644 100644 --- a/src/bindings/ruby/SConscript +++ b/src/bindings/ruby/SConscript @@ -1,4 +1,7 @@ # -*- python -*- + +from __future__ import absolute_import, division, print_function + import os.path Import("env libhammer_shared testruns targets") diff --git a/tools/csharp/__init__.py b/tools/csharp/__init__.py index af4f519..fbc9d30 100644 --- a/tools/csharp/__init__.py +++ b/tools/csharp/__init__.py @@ -21,4 +21,6 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from csharp import exists, generate +from __future__ import absolute_import, division, print_function + +from .csharp import exists, generate diff --git a/tools/csharp/csharp.py b/tools/csharp/csharp.py index 6b38b45..f10fc71 100644 --- a/tools/csharp/csharp.py +++ b/tools/csharp/csharp.py @@ -30,6 +30,8 @@ # This is an attempt to meld to two based initially on the Microsoft C# tool with amendmnets from the Mono # tool. +from __future__ import absolute_import, division, print_function + import os.path import SCons.Builder import SCons.Node.FS diff --git a/tools/csharp/mono.py b/tools/csharp/mono.py index a2cc380..99c509c 100644 --- a/tools/csharp/mono.py +++ b/tools/csharp/mono.py @@ -25,6 +25,8 @@ # This C# Tool for Mono taken from http://www.scons.org/wiki/CsharpBuilder. +from __future__ import absolute_import, division, print_function + import os.path import SCons.Builder import SCons.Node.FS diff --git a/tools/scanreplace.py b/tools/scanreplace.py index 5321e48..81a167e 100644 --- a/tools/scanreplace.py +++ b/tools/scanreplace.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function + from string import Template def replace_action(target, source, env): From 287f71d5614dc091fb8a810854f82ceaafd73752 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 21:28:35 +0100 Subject: [PATCH 07/11] Fix uses of retired builtins and builtin methods In Python 3.x - int and long types are unified. The unified type is called int. - the text string type (unicode) is renamed to str. - the byte string type (str) is renamed to bytes. - chr returns a text string (i.e. str) - xrange is renamed to range. - dict.has_key() is removed - --- examples/base64_sem1.py | 6 +++--- examples/base64_sem2.py | 2 +- src/bindings/swig/hammer.i | 29 +++++++++++++++++++++-------- tools/csharp/csharp.py | 16 ++++++++-------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/examples/base64_sem1.py b/examples/base64_sem1.py index 2c4d6e0..1665ccb 100644 --- a/examples/base64_sem1.py +++ b/examples/base64_sem1.py @@ -26,7 +26,7 @@ import hammer as h def act_bsfdig(p, user_data=None): # FIXME See the note in init_parser() - c = p if isinstance(p, (int, long)) else ord(p) + c = p if isinstance(p, h.INTEGER_TYPES) else ord(p) if 0x41 <= c <= 0x5A: # A-Z return c - 0x41 @@ -65,14 +65,14 @@ def act_base64_n(n, p, user_data=None): x = 0 bits = 0 - for i in xrange(0, n+1): + for i in range(0, n+1): x <<= 6 x |= p[i] or 0 bits += 6 x >>= bits % 8 # align, i.e. cut off extra bits - for i in xrange(n): + for i in range(n): item = x & 0xFF res[n-1-i] = item # output the last byte and diff --git a/examples/base64_sem2.py b/examples/base64_sem2.py index 3b023dd..9915889 100644 --- a/examples/base64_sem2.py +++ b/examples/base64_sem2.py @@ -28,7 +28,7 @@ import hammer as h def bsfdig_value(p): """Return the numeric value of a parsed base64 digit. """ - c = p if isinstance(p, (int, long)) else ord(p) + c = p if isinstance(p, h.INTEGER_TYPES) else ord(p) if c: if 0x41 <= c <= 0x5A: # A-Z return c - 0x41 diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 122ffe4..7ba7be2 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -25,6 +25,20 @@ } %pythoncode %{ + try: + INTEGER_TYPES = (int, long) + except NameError: + INTEGER_TYPES = (int,) + + try: + TEXT_TYPE = unicode + def bchr(i): + return chr(i) + except NameError: + TEXT_TYPE = str + def bchr(i): + return bytes([i]) + class Placeholder(object): """The python equivalent of TT_NONE""" def __str__(self): @@ -250,36 +264,35 @@ } %pythoncode %{ - def action(p, act): return _h_action(p, act) def attr_bool(p, pred): return _h_attr_bool(p, pred) def ch(ch): - if isinstance(ch, str) or isinstance(ch, unicode): + if isinstance(ch, (bytes, TEXT_TYPE)): return token(ch) else: return _h_ch(ch) def ch_range(c1, c2): - dostr = isinstance(c1, str) - dostr2 = isinstance(c2, str) - if isinstance(c1, unicode) or isinstance(c2, unicode): + dostr = isinstance(c1, bytes) + dostr2 = isinstance(c2, bytes) + if isinstance(c1, TEXT_TYPE) or isinstance(c2, TEXT_TYPE): raise TypeError("ch_range only works on bytes") if dostr != dostr2: raise TypeError("Both arguments to ch_range must be the same type") if dostr: - return action(_h_ch_range(c1, c2), chr) + return action(_h_ch_range(c1, c2), bchr) else: return _h_ch_range(c1, c2) def epsilon_p(): return _h_epsilon_p() def end_p(): return _h_end_p() def in_(charset): - return action(_h_in(charset), chr) + return action(_h_in(charset), bchr) def not_in(charset): - return action(_h_not_in(charset), chr) + return action(_h_not_in(charset), bchr) def not_(p): return _h_not(p) def int_range(p, i1, i2): return _h_int_range(p, i1, i2) diff --git a/tools/csharp/csharp.py b/tools/csharp/csharp.py index f10fc71..101d3f6 100644 --- a/tools/csharp/csharp.py +++ b/tools/csharp/csharp.py @@ -205,7 +205,7 @@ def AddToModPaths(env, files, **kw): def cscFlags(target, source, env, for_signature): listCmd = [] - if (env.has_key('WINEXE')): + if ('WINEXE' in env): if (env['WINEXE'] == 1): listCmd.append('-t:winexe') return listCmd @@ -245,7 +245,7 @@ def cscSourcesNoResources(target, source, env, for_signature): def cscRefs(target, source, env, for_signature): listCmd = [] - if (env.has_key('ASSEMBLYREFS')): + if ('ASSEMBLYREFS' in env): refs = SCons.Util.flatten(env['ASSEMBLYREFS']) for ref in refs: if SCons.Util.is_String(ref): @@ -258,7 +258,7 @@ def cscRefs(target, source, env, for_signature): def cscMods(target, source, env, for_signature): listCmd = [] - if (env.has_key('NETMODULES')): + if ('NETMODULES' in env): mods = SCons.Util.flatten(env['NETMODULES']) for mod in mods: listCmd.append('-addmodule:%s' % mod) @@ -276,7 +276,7 @@ def alLinkSources(target, source, env, for_signature): # just treat this as a generic unidentified source file listCmd.append('-link:%s' % s.get_string(for_signature)) - if env.has_key('VERSION'): + if 'VERSION' in env: version = parseVersion(env) listCmd.append('-version:%d.%d.%d.%d' % version) @@ -298,7 +298,7 @@ def cliLinkSources(target, source, env, for_signature): return listCmd def add_version(target, source, env): - if env.has_key('VERSION'): + if 'VERSION' in env: if SCons.Util.is_String(target[0]): versionfile = target[0] + '_VersionInfo.cs' else: @@ -321,14 +321,14 @@ def lib_emitter(target, source, env): def add_depends(target, source, env): """Add dependency information before the build order is established""" - if (env.has_key('NETMODULES')): + if ('NETMODULES' in env): mods = SCons.Util.flatten(env['NETMODULES']) for mod in mods: # add as dependency for t in target: env.Depends(t, mod) - if (env.has_key('ASSEMBLYREFS')): + if ('ASSEMBLYREFS' in env): refs = SCons.Util.flatten(env['ASSEMBLYREFS']) for ref in refs: # add as dependency @@ -419,7 +419,7 @@ res_action = SCons.Action.Action('$CLIRCCOM', '$CLIRCCOMSTR') def res_emitter(target, source, env): # prepend NAMESPACE if provided - if (env.has_key('NAMESPACE')): + if ('NAMESPACE' in env): newtargets = [] for t in target: tname = t.name From c82390941d7d0d2677d5a6142bf37bf847c663b3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 21:38:49 +0100 Subject: [PATCH 08/11] Allow Python interpreter to be specified during build This allows the library to be built and tested with a non-default version of CPython, e.g. scons bindings=python python=python3.6 scons bindings=python python=python3.6 testpython --- SConstruct | 1 + src/bindings/python/SConscript | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index ae85cda..f05e00d 100644 --- a/SConstruct +++ b/SConstruct @@ -15,6 +15,7 @@ vars = Variables(None, ARGUMENTS) vars.Add(PathVariable('DESTDIR', 'Root directory to install in (useful for packaging scripts)', None, PathVariable.PathIsDirCreate)) vars.Add(PathVariable('prefix', 'Where to install in the FHS', default_install_dir, PathVariable.PathAccept)) vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby'])) +vars.Add('python', 'Python interpreter', 'python') tools = ['default', 'scanreplace'] if 'dotnet' in ARGUMENTS.get('bindings', []): diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript index d5d092d..2ca8ea0 100644 --- a/src/bindings/python/SConscript +++ b/src/bindings/python/SConscript @@ -10,17 +10,18 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0) swig = pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE")) setup = ['setup.py'] pydir = os.path.join(env['BUILD_BASE'], 'src/bindings/python') -libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --inplace') +pysetup = os.path.join(pydir, 'setup.py') +libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], '%s %s build_ext --inplace' % (env['python'], pysetup)) Default(libhammer_python) pytestenv = pythonenv.Clone() pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0])) pytests = ['hammer_tests.py'] -pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=" + os.path.dirname(str(libhammer_shared[0])) + " nosetests -vv $SOURCE") +pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=%s %s -mnose -vv $SOURCE" % (os.path.dirname(str(libhammer_shared[0])), env['python'])) pytest = Alias("testpython", [pytestexec], pytestexec) AlwaysBuild(pytestexec) testruns.append(pytest) -pyinstallexec = pythonenv.Command(None, libhammer_python, 'python ' + os.path.join(pydir, 'setup.py ') + ' install') +pyinstallexec = pythonenv.Command(None, libhammer_python, '%s %s install' % (env['python'], pysetup)) pyinstall = Alias("installpython", [pyinstallexec], pyinstallexec) targets.append(pyinstall) From 8b4b8ddc574fb6496242b6b1b9b355ea0ce88d53 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 21:43:00 +0100 Subject: [PATCH 09/11] Use PyBytes_* Python CAPI functions This removes any doubts about what type of string is in use. --- src/bindings/swig/hammer.i | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i index 7ba7be2..0097bdc 100644 --- a/src/bindings/swig/hammer.i +++ b/src/bindings/swig/hammer.i @@ -1,4 +1,7 @@ %module hammer +%begin %{ +#define SWIG_PYTHON_STRICT_BYTE_CHAR +%} %nodefaultctor; @@ -83,11 +86,11 @@ PyErr_SetString(PyExc_ValueError, "Expecting a string"); return NULL; } else { - $1 = *(uint8_t*)PyString_AsString($input); + $1 = *(uint8_t*)PyBytes_AsString($input); } } %typemap(out) HBytes* { - $result = PyString_FromStringAndSize((char*)$1->token, $1->len); + $result = PyBytes_FromStringAndSize((char*)$1->token, $1->len); } %typemap(out) struct HCountedArray_* { int i; @@ -187,7 +190,7 @@ return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL); break; case TT_BYTES: - return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); + return PyBytes_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len); case TT_SINT: // TODO: return PyINT if appropriate return PyLong_FromLong(token->token_data.sint); From 59ba68ef844e43cd73f38cf4e0f110cda6e031b1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 21:54:07 +0100 Subject: [PATCH 10/11] Use byte literals in examples and unit tests In Python 2.x an unprefixed string literal produces a byte string. In Python 3.x an unprefixed string literal produces a textual string. To produce a byte string in both a b prefix is needed, e.g. b'foo'. Since I believe Hammer works predominantly with byte strings I have used b prefixes throughout. --- examples/base64.py | 10 +- examples/base64_sem1.py | 16 +- examples/base64_sem2.py | 16 +- src/bindings/python/hammer_tests.py | 364 ++++++++++++++-------------- 4 files changed, 203 insertions(+), 203 deletions(-) diff --git a/examples/base64.py b/examples/base64.py index 0233ff7..f7b68ab 100644 --- a/examples/base64.py +++ b/examples/base64.py @@ -23,13 +23,13 @@ def init_parser(): alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.ch('=') + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.ch(b'=') bsfdig = h.choice(alpha, digit, plus, slash) - bsfdig_4bit = h.in_('AEIMQUYcgkosw048') - bsfdig_2bit = h.in_('AQgw') + bsfdig_4bit = h.in_(b'AEIMQUYcgkosw048') + bsfdig_2bit = h.in_(b'AQgw') base64_3 = h.repeat_n(bsfdig, 4) base64_2 = h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals) base64_1 = h.sequence(bsfdig, bsfdig_2bit, equals, equals) diff --git a/examples/base64_sem1.py b/examples/base64_sem1.py index 1665ccb..84cb2d7 100644 --- a/examples/base64_sem1.py +++ b/examples/base64_sem1.py @@ -34,9 +34,9 @@ def act_bsfdig(p, user_data=None): return c - 0x61 + 26 elif 0x30 <= c <= 0x39: # 0-9 return c - 0x30 + 52 - elif c == '+': + elif c == b'+': return 62 - elif c == '/': + elif c == b'/': return 63 else: raise ValueError @@ -118,16 +118,16 @@ def init_parser(): # literals, or integers digit = h.ch_range(0x30, 0x39) alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) - space = h.in_(" \t\n\r\f\v") + space = h.in_(b" \t\n\r\f\v") # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.action(h.ch('='), act_equals) + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.action(h.ch(b'='), act_equals) bsfdig = h.action(h.choice(alpha, digit, plus, slash), act_bsfdig) - bsfdig_4bit = h.action(h.in_("AEIMQUYcgkosw048"), act_bsfdig_4bit) - bsfdig_2bit = h.action(h.in_("AQgw"), act_bsfdig_2bit) + bsfdig_4bit = h.action(h.in_(b"AEIMQUYcgkosw048"), act_bsfdig_4bit) + bsfdig_2bit = h.action(h.in_(b"AQgw"), act_bsfdig_2bit) base64_3 = h.action(h.repeat_n(bsfdig, 4), act_base64_3) base64_2 = h.action(h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals), act_base64_2) diff --git a/examples/base64_sem2.py b/examples/base64_sem2.py index 9915889..2e33774 100644 --- a/examples/base64_sem2.py +++ b/examples/base64_sem2.py @@ -36,9 +36,9 @@ def bsfdig_value(p): return c - 0x61 + 26 elif 0x30 <= c <= 0x39: # 0-9 return c - 0x30 + 52 - elif c == '+': + elif c == b'+': return 62 - elif c == '/': + elif c == b'/': return 63 return 0 @@ -109,16 +109,16 @@ def init_parser(): # CORE digit = h.ch_range(0x30, 0x39) alpha = h.choice(h.ch_range(0x41, 0x5a), h.ch_range(0x61, 0x7a)) - space = h.in_(" \t\n\r\f\v") + space = h.in_(b" \t\n\r\f\v") # AUX. - plus = h.ch('+') - slash = h.ch('/') - equals = h.ch('=') + plus = h.ch(b'+') + slash = h.ch(b'/') + equals = h.ch(b'=') bsfdig = h.choice(alpha, digit, plus, slash) - bsfdig_4bit = h.in_("AEIMQUYcgkosw048") - bsfdig_2bit = h.in_("AQgw") + bsfdig_4bit = h.in_(b"AEIMQUYcgkosw048") + bsfdig_2bit = h.in_(b"AQgw") base64_3 = h.repeat_n(bsfdig, 4) base64_2 = h.sequence(bsfdig, bsfdig, bsfdig_4bit, equals) base64_1 = h.sequence(bsfdig, bsfdig_2bit, equals, equals) diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py index ee79caa..0aead60 100644 --- a/src/bindings/python/hammer_tests.py +++ b/src/bindings/python/hammer_tests.py @@ -6,215 +6,215 @@ import hammer as h class TestTokenParser(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.token("95\xa2") + cls.parser = h.token(b"95\xa2") def test_success(self): - self.assertEqual(self.parser.parse("95\xa2"), "95\xa2") + self.assertEqual(self.parser.parse(b"95\xa2"), b"95\xa2") def test_partial_fails(self): - self.assertEqual(self.parser.parse("95"), None) + self.assertEqual(self.parser.parse(b"95"), None) class TestChParser(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser_int = h.ch(0xa2) - cls.parser_chr = h.ch("\xa2") + cls.parser_chr = h.ch(b"\xa2") def test_success(self): - self.assertEqual(self.parser_int.parse("\xa2"), 0xa2) - self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2") + self.assertEqual(self.parser_int.parse(b"\xa2"), 0xa2) + self.assertEqual(self.parser_chr.parse(b"\xa2"), b"\xa2") def test_failure(self): - self.assertEqual(self.parser_int.parse("\xa3"), None) - self.assertEqual(self.parser_chr.parse("\xa3"), None) + self.assertEqual(self.parser_int.parse(b"\xa3"), None) + self.assertEqual(self.parser_chr.parse(b"\xa3"), None) class TestChRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.ch_range("a", "c") + cls.parser = h.ch_range(b"a", b"c") def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(self.parser.parse(b"d"), None) class TestInt64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int64() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) + self.assertEqual(self.parser.parse(b"\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None) + self.assertEqual(self.parser.parse(b"\xff\xff\xff\xfe\x00\x00\x00"), None) class TestInt32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int32() def test_success(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000) - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse(b"\xff\xfe\x00\x00"), -0x20000) + self.assertEqual(self.parser.parse(b"\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\xff\xfe\x00"), None) - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(self.parser.parse(b"\xff\xfe\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x02\x00"), None) class TestInt16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int16() def test_success(self): - self.assertEqual(self.parser.parse("\xfe\x00"), -0x200) - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(self.parser.parse(b"\xfe\x00"), -0x200) + self.assertEqual(self.parser.parse(b"\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\xfe"), None) - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(self.parser.parse(b"\xfe"), None) + self.assertEqual(self.parser.parse(b"\x02"), None) class TestInt8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int8() def test_success(self): - self.assertEqual(self.parser.parse("\x88"), -0x78) + self.assertEqual(self.parser.parse(b"\x88"), -0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestUint64(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint64() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) + self.assertEqual(self.parser.parse(b"\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x00\x00\x02\x00\x00\x00"), None) class TestUint32(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint32() def test_success(self): - self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000) + self.assertEqual(self.parser.parse(b"\x00\x02\x00\x00"), 0x20000) def test_failure(self): - self.assertEqual(self.parser.parse("\x00\x02\x00"), None) + self.assertEqual(self.parser.parse(b"\x00\x02\x00"), None) class TestUint16(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint16() def test_success(self): - self.assertEqual(self.parser.parse("\x02\x00"), 0x200) + self.assertEqual(self.parser.parse(b"\x02\x00"), 0x200) def test_failure(self): - self.assertEqual(self.parser.parse("\x02"), None) + self.assertEqual(self.parser.parse(b"\x02"), None) class TestUint8(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.uint8() def test_success(self): - self.assertEqual(self.parser.parse("\x78"), 0x78) + self.assertEqual(self.parser.parse(b"\x78"), 0x78) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestIntRange(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.int_range(h.uint8(), 3, 10) def test_success(self): - self.assertEqual(self.parser.parse("\x05"), 5) + self.assertEqual(self.parser.parse(b"\x05"), 5) def test_failure(self): - self.assertEqual(self.parser.parse("\x0b"), None) + self.assertEqual(self.parser.parse(b"\x0b"), None) class TestWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.whitespace(h.ch("a")) + cls.parser = h.whitespace(h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse(" a"), "a") - self.assertEqual(self.parser.parse("\ta"), "a") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b" a"), b"a") + self.assertEqual(self.parser.parse(b" a"), b"a") + self.assertEqual(self.parser.parse(b"\ta"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("_a"), None) + self.assertEqual(self.parser.parse(b"_a"), None) class TestWhitespaceEnd(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = h.whitespace(h.end_p()) def test_success(self): - self.assertEqual(self.parser.parse(""), None) # empty string - self.assertEqual(self.parser.parse(" "), None) # empty string + self.assertEqual(self.parser.parse(b""), None) # empty string + self.assertEqual(self.parser.parse(b" "), None) # empty string def test_failure(self): - self.assertEqual(self.parser.parse(" x"), None) + self.assertEqual(self.parser.parse(b" x"), None) class TestLeft(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.left(h.ch("a"), h.ch(" ")) + cls.parser = h.left(h.ch(b"a"), h.ch(b" ")) def test_success(self): - self.assertEqual(self.parser.parse("a "), "a") + self.assertEqual(self.parser.parse(b"a "), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestRight(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.right(h.ch(" "), h.ch("a")) + cls.parser = h.right(h.ch(b" "), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse(" a"), "a") + self.assertEqual(self.parser.parse(b" a"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse("ba"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b"ba"), None) class TestMiddle(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" ")) + cls.parser = h.middle(h.ch(b" "), h.ch(b"a"), h.ch(b" ")) def test_success(self): - self.assertEqual(self.parser.parse(" a "), "a") + self.assertEqual(self.parser.parse(b" a "), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse(" "), None) - self.assertEqual(self.parser.parse(" a"), None) - self.assertEqual(self.parser.parse("a "), None) - self.assertEqual(self.parser.parse(" b "), None) - self.assertEqual(self.parser.parse("ba "), None) - self.assertEqual(self.parser.parse(" ab"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b" "), None) + self.assertEqual(self.parser.parse(b" a"), None) + self.assertEqual(self.parser.parse(b"a "), None) + self.assertEqual(self.parser.parse(b" b "), None) + self.assertEqual(self.parser.parse(b"ba "), None) + self.assertEqual(self.parser.parse(b" ab"), None) class TestAction(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), - h.choice(h.ch("b"), h.ch("B"))), + cls.parser = h.action(h.sequence(h.choice(h.ch(b"a"), h.ch(b"A")), + h.choice(h.ch(b"b"), h.ch(b"B"))), lambda x: [y.upper() for y in x]) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ["A", "B"]) - self.assertEqual(self.parser.parse("AB"), ["A", "B"]) + self.assertEqual(self.parser.parse(b"ab"), [b"A", b"B"]) + self.assertEqual(self.parser.parse(b"AB"), [b"A", b"B"]) def test_failure(self): - self.assertEqual(self.parser.parse("XX"), None) + self.assertEqual(self.parser.parse(b"XX"), None) class TestIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.in_("abc") + cls.parser = h.in_(b"abc") def test_success(self): - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("d"), None) + self.assertEqual(self.parser.parse(b"d"), None) class TestNotIn(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.not_in("abc") + cls.parser = h.not_in(b"abc") def test_success(self): - self.assertEqual(self.parser.parse("d"), "d") + self.assertEqual(self.parser.parse(b"d"), b"d") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestEndP(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.end_p()) + cls.parser = h.sequence(h.ch(b"a"), h.end_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): - self.assertEqual(self.parser.parse("aa"), None) + self.assertEqual(self.parser.parse(b"aa"), None) class TestNothingP(unittest.TestCase): @classmethod @@ -223,244 +223,244 @@ class TestNothingP(unittest.TestCase): def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestSequence(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ('a','b')) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) - self.assertEqual(self.parser.parse("b"), None) + self.assertEqual(self.parser.parse(b"a"), None) + self.assertEqual(self.parser.parse(b"b"), None) class TestSequenceWhitespace(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b"))) + cls.parser = h.sequence(h.ch(b"a"), h.whitespace(h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ('a','b')) - self.assertEqual(self.parser.parse("a b"), ('a','b')) - self.assertEqual(self.parser.parse("a b"), ('a','b')) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) + self.assertEqual(self.parser.parse(b"a b"), (b"a", b"b")) + self.assertEqual(self.parser.parse(b"a b"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a c"), None) + self.assertEqual(self.parser.parse(b"a c"), None) class TestChoice(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.choice(h.ch("a"), h.ch("b")) + cls.parser = h.choice(h.ch(b"a"), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("b"), "b") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b"b"), b"b") def test_failure(self): - self.assertEqual(self.parser.parse("c"), None) + self.assertEqual(self.parser.parse(b"c"), None) class TestButNot(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch("a"), h.token("ab")) + cls.parser = h.butnot(h.ch(b"a"), h.token(b"ab")) def test_success(self): - self.assertEqual(self.parser.parse("a"), "a") - self.assertEqual(self.parser.parse("aa"), "a") + self.assertEqual(self.parser.parse(b"a"), b"a") + self.assertEqual(self.parser.parse(b"aa"), b"a") def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestButNotRange(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6")) + cls.parser = h.butnot(h.ch_range(b"0", b"9"), h.ch(b"6")) def test_success(self): - self.assertEqual(self.parser.parse("4"), "4") + self.assertEqual(self.parser.parse(b"4"), b"4") def test_failure(self): - self.assertEqual(self.parser.parse("6"), None) + self.assertEqual(self.parser.parse(b"6"), None) class TestDifference(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.difference(h.token("ab"), h.ch("a")) + cls.parser = h.difference(h.token(b"ab"), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), "ab") + self.assertEqual(self.parser.parse(b"ab"), b"ab") def test_failure(self): - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestXor(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9")) + cls.parser = h.xor(h.ch_range(b"0", b"6"), h.ch_range(b"5", b"9")) def test_success(self): - self.assertEqual(self.parser.parse("0"), "0") - self.assertEqual(self.parser.parse("9"), "9") + self.assertEqual(self.parser.parse(b"0"), b"0") + self.assertEqual(self.parser.parse(b"9"), b"9") def test_failure(self): - self.assertEqual(self.parser.parse("5"), None) - self.assertEqual(self.parser.parse("a"), None) + self.assertEqual(self.parser.parse(b"5"), None) + self.assertEqual(self.parser.parse(b"a"), None) class TestMany(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.many(h.choice(h.ch(b"a"), h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse(""), ()) - self.assertEqual(self.parser.parse("a"), ('a',)) - self.assertEqual(self.parser.parse("b"), ('b',)) - self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a')) + self.assertEqual(self.parser.parse(b""), ()) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"b"), (b"b",)) + self.assertEqual(self.parser.parse(b"aabbaba"), (b"a", b"a", b"b", b"b", b"a", b"b", b"a")) def test_failure(self): pass class TestMany1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b"))) + cls.parser = h.many1(h.choice(h.ch(b"a"), h.ch(b"b"))) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) - self.assertEqual(self.parser.parse("b"), ("b",)) - self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a")) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"b"), (b"b",)) + self.assertEqual(self.parser.parse(b"aabbaba"), (b"a", b"a", b"b", b"b", b"a", b"b", b"a")) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) - self.assertEqual(self.parser.parse("daabbabadef"), None) + self.assertEqual(self.parser.parse(b""), None) + self.assertEqual(self.parser.parse(b"daabbabadef"), None) class TestRepeatN(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2) + cls.parser = h.repeat_n(h.choice(h.ch(b"a"), h.ch(b"b")), 2) def test_success(self): - self.assertEqual(self.parser.parse("abdef"), ('a', 'b')) + self.assertEqual(self.parser.parse(b"abdef"), (b"a", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("adef"), None) - self.assertEqual(self.parser.parse("dabdef"), None) + self.assertEqual(self.parser.parse(b"adef"), None) + self.assertEqual(self.parser.parse(b"dabdef"), None) class TestOptional(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d")) + cls.parser = h.sequence(h.ch(b"a"), h.optional(h.choice(h.ch(b"b"), h.ch(b"c"))), h.ch(b"d")) def test_success(self): - self.assertEqual(self.parser.parse("abd"), ('a','b','d')) - self.assertEqual(self.parser.parse("acd"), ('a','c','d')) - self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd')) + self.assertEqual(self.parser.parse(b"abd"), (b"a", b"b", b"d")) + self.assertEqual(self.parser.parse(b"acd"), (b"a", b"c", b"d")) + self.assertEqual(self.parser.parse(b"ad"), (b"a", h.Placeholder(), b"d")) def test_failure(self): - self.assertEqual(self.parser.parse("aed"), None) - self.assertEqual(self.parser.parse("ab"), None) - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(self.parser.parse(b"aed"), None) + self.assertEqual(self.parser.parse(b"ab"), None) + self.assertEqual(self.parser.parse(b"ac"), None) class TestIgnore(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c")) + cls.parser = h.sequence(h.ch(b"a"), h.ignore(h.ch(b"b")), h.ch(b"c")) def test_success(self): - self.assertEqual(self.parser.parse("abc"), ("a","c")) + self.assertEqual(self.parser.parse(b"abc"), (b"a",b"c")) def test_failure(self): - self.assertEqual(self.parser.parse("ac"), None) + self.assertEqual(self.parser.parse(b"ac"), None) class TestSepBy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.sepBy(h.choice(h.ch(b"1"), h.ch(b"2"), h.ch(b"3")), h.ch(b",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) - self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) - self.assertEqual(self.parser.parse("1,3"), ('1','3')) - self.assertEqual(self.parser.parse("3"), ('3',)) - self.assertEqual(self.parser.parse(""), ()) + self.assertEqual(self.parser.parse(b"1,2,3"), (b"1", b"2", b"3")) + self.assertEqual(self.parser.parse(b"1,3,2"), (b"1", b"3", b"2")) + self.assertEqual(self.parser.parse(b"1,3"), (b"1", b"3")) + self.assertEqual(self.parser.parse(b"3"), (b"3",)) + self.assertEqual(self.parser.parse(b""), ()) def test_failure(self): pass class TestSepBy1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(",")) + cls.parser = h.sepBy1(h.choice(h.ch(b"1"), h.ch(b"2"), h.ch(b"3")), h.ch(b",")) def test_success(self): - self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3')) - self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2')) - self.assertEqual(self.parser.parse("1,3"), ('1','3')) - self.assertEqual(self.parser.parse("3"), ('3',)) + self.assertEqual(self.parser.parse(b"1,2,3"), (b"1", b"2", b"3")) + self.assertEqual(self.parser.parse(b"1,3,2"), (b"1", b"3", b"2")) + self.assertEqual(self.parser.parse(b"1,3"), (b"1", b"3")) + self.assertEqual(self.parser.parse(b"3"), (b"3",)) def test_failure(self): - self.assertEqual(self.parser.parse(""), None) + self.assertEqual(self.parser.parse(b""), None) class TestEpsilonP1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.epsilon_p(), h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("ab"), ("a", "b")) + self.assertEqual(self.parser.parse(b"ab"), (b"a", b"b")) def test_failure(self): pass class TestEpsilonP2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.epsilon_p(), h.ch("a")) + cls.parser = h.sequence(h.epsilon_p(), h.ch(b"a")) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): pass class TestEpsilonP3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.epsilon_p()) + cls.parser = h.sequence(h.ch(b"a"), h.epsilon_p()) def test_success(self): - self.assertEqual(self.parser.parse("a"), ("a",)) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) def test_failure(self): pass class TestAttrBool(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), + cls.parser = h.attr_bool(h.many1(h.choice(h.ch(b"a"), h.ch(b"b"))), lambda x: x[0] == x[1]) def test_success(self): - self.assertEqual(self.parser.parse("aa"), ("a", "a")) - self.assertEqual(self.parser.parse("bb"), ("b", "b")) + self.assertEqual(self.parser.parse(b"aa"), (b"a", b"a")) + self.assertEqual(self.parser.parse(b"bb"), (b"b", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("ab"), None) + self.assertEqual(self.parser.parse(b"ab"), None) class TestAnd1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0")) + cls.parser = h.sequence(h.and_(h.ch(b"0")), h.ch(b"0")) def test_success(self): - self.assertEqual(self.parser.parse("0"), ("0",)) + self.assertEqual(self.parser.parse(b"0"), (b"0",)) def test_failure(self): pass class TestAnd2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1")) + cls.parser = h.sequence(h.and_(h.ch(b"0")), h.ch(b"1")) def test_success(self): pass def test_failure(self): - self.assertEqual(self.parser.parse("0"), None) + self.assertEqual(self.parser.parse(b"0"), None) class TestAnd3(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2"))) + cls.parser = h.sequence(h.ch(b"1"), h.and_(h.ch(b"2"))) def test_success(self): - self.assertEqual(self.parser.parse("12"), ('1',)) + self.assertEqual(self.parser.parse(b"12"), (b"1",)) def test_failure(self): pass class TestNot1(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), - h.choice(h.ch("+"), h.token("++")), - h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), + h.choice(h.ch(b"+"), h.token(b"++")), + h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b")) + self.assertEqual(self.parser.parse(b"a+b"), (b"a", b"+", b"b")) def test_failure(self): - self.assertEqual(self.parser.parse("a++b"), None) + self.assertEqual(self.parser.parse(b"a++b"), None) class TestNot2(unittest.TestCase): @classmethod def setUpClass(cls): - cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), - h.token("++")), - h.ch("b")) + cls.parser = h.sequence(h.ch(b"a"), h.choice(h.sequence(h.ch(b"+"), h.not_(h.ch(b"+"))), + h.token(b"++")), + h.ch(b"b")) def test_success(self): - self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b')) - self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b')) + self.assertEqual(self.parser.parse(b"a+b"), (b"a", (b"+",), b"b")) + self.assertEqual(self.parser.parse(b"a++b"), (b"a", b"++", b"b")) def test_failure(self): pass @@ -469,12 +469,12 @@ class TestNot2(unittest.TestCase): # # @classmethod # # def setUpClass(cls): # # cls.parser = h.indirect() -# # a = h.ch("a") +# # a = h.ch(b"a") # # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a)) # # def test_success(self): -# # self.assertEqual(self.parser.parse("a"), "a") -# # self.assertEqual(self.parser.parse("aa"), ["a", "a"]) -# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"]) +# # self.assertEqual(self.parser.parse(b"a"), b"a") +# # self.assertEqual(self.parser.parse(b"aa"), [b"a", b"a"]) +# # self.assertEqual(self.parser.parse(b"aaa"), [b"a", b"a", b"a"]) # # def test_failure(self): # # pass @@ -482,15 +482,15 @@ class TestNot2(unittest.TestCase): class TestRightrec(unittest.TestCase): @classmethod def setUpClass(cls): - #raise unittest.SkipTest("Bind doesn't work right now") + #raise unittest.SkipTest(b"Bind doesn't work right now") cls.parser = h.indirect() - a = h.ch("a") + a = h.ch(b"a") cls.parser.bind(h.choice(h.sequence(a, cls.parser), h.epsilon_p())) def test_success(self): - self.assertEqual(self.parser.parse("a"), ('a',)) - self.assertEqual(self.parser.parse("aa"), ('a', ('a',))) - self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',)))) + self.assertEqual(self.parser.parse(b"a"), (b"a",)) + self.assertEqual(self.parser.parse(b"aa"), (b"a", (b"a",))) + self.assertEqual(self.parser.parse(b"aaa"), (b"a", (b"a", (b"a",)))) def test_failure(self): pass @@ -499,13 +499,13 @@ class TestRightrec(unittest.TestCase): # # @classmethod # # def setUpClass(cls): # # cls.parser = h.indirect() -# # d = h.ch("d") -# # p = h.ch("+") +# # d = h.ch(b"d") +# # p = h.ch(b"+") # # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d)) # # # this is supposed to be flattened # # def test_success(self): -# # self.assertEqual(self.parser.parse("d"), ["d"]) -# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"]) -# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"]) +# # self.assertEqual(self.parser.parse(b"d"), [b"d"]) +# # self.assertEqual(self.parser.parse(b"d+d"), [b"d", b"+", b"d"]) +# # self.assertEqual(self.parser.parse(b"d+d+d"), [b"d", b"+", b"d", b"+", b"d"]) # # def test_failure(self): -# # self.assertEqual(self.parser.parse("d+"), None) +# # self.assertEqual(self.parser.parse(b"d+"), None) From 2ec062cd7c751baa73ab055a57ac7179a41ba13f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 10 May 2019 22:08:41 +0100 Subject: [PATCH 11/11] Update README with Python 3.x support --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee62558..d122fc8 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ Installing * pkg-config (for `scons test`) * glib-2.0 (>= 2.29) (for `scons test`) * glib-2.0-dev (for `scons test`) -* [swig](http://swig.org/) (for Python/Perl/PHP bindings; Perl requires >= 2.0.8) -* python2.7-dev (for Python bindings) +* [swig](http://swig.org/) (for Python/Perl/PHP bindings; Perl requires >= 2.0.8; Python 3.x requires >= 3.0.0) +* python2.7-dev (for Python 2 bindings) +* python3-dev (>= 3.5) (for Python 3 bindings) * a JDK (for Java bindings) * a working [phpenv](https://github.com/CHH/phpenv) configuration (for PHP bindings) * [Ruby](https://www.ruby-lang.org/) >= 1.9.3 and bundler, for the Ruby bindings @@ -73,7 +74,7 @@ The `examples/` directory contains some simple examples, currently including: Known Issues ============ -The Python bindings only work with Python 2.7. SCons doesn't work with Python 3, and PyCapsule isn't available in 2.6 and below, so 2.7 is all you get. Sorry about that. +The Python bindings work with Python 2.7, and Python 3.5+. The requirement for SWIG >= 2.0.8 for Perl bindings is due to a [known bug](http://sourceforge.net/p/swig/patches/324/) in SWIG. [ppa:dns/irc](https://launchpad.net/~dns/+archive/irc) has backports of SWIG 2.0.8 for Ubuntu versions 10.04-12.10; you can also [build SWIG from source](http://www.swig.org/download.html).