diff --git a/make/generate-flex-bison.mk b/make/generate-flex-bison.mk index d32abce..c8f8361 100644 --- a/make/generate-flex-bison.mk +++ b/make/generate-flex-bison.mk @@ -4,8 +4,9 @@ NAMESPACE_IN_GENERATED ?= $(FB_NAME) # These types are meant to be cut off the tree and turned into hand coded flex # regexes -#TRIM_SYMBOLS = blah -TRIM_SYMBOLS ?= +TRIM_SYMBOLS ?= +CUT_SYMBOLS ?= +IRRELEVANT_SYMBOLS ?= GENERATE_LOG_LEVEL ?= notice FIX_EXTENSIONS ?= discard CHECK_SYMBOLS ?= all @@ -24,6 +25,8 @@ GENERATE = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL --replace-whitespace \ $(OPT_CHECK_SYMBOLS) \ --trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/ */,/g') \ + --cut-symbols=$(shell echo $(CUT_SYMBOLS) | sed 's/ */,/g') \ + --irrelevant-symbols=$(shell echo $(IRRELEVANT_SYMBOLS) | sed 's/ */,/g') \ --namespace=$(NAMESPACE_IN_GENERATED) \ --includes=$(shell echo $(INCLUDED_BY_GENERATED) | sed 's/ */,/g') \ $(CREATE_EXTRA_ARGS) diff --git a/tools/python/jwutils/grammar.py b/tools/python/jwutils/grammar.py index f29eede..1218092 100644 --- a/tools/python/jwutils/grammar.py +++ b/tools/python/jwutils/grammar.py @@ -138,7 +138,7 @@ def tok2sym(tok): if term is not None: if term in special_terminals.keys(): return "T_" + special_terminals[term].upper() - return "T_" + term.upper() + return "T_" + re.sub('[^a-zA-Z0-9]', '_', term).upper() return tok def tok2regex(tok): @@ -319,6 +319,7 @@ class Symbol: def __init__(self, token, tp = None, rules = None): self.reset(token, tp, rules) + self.set_is_payload(True) def reset(self, token, tp = None, rules = None): if tp == None: @@ -339,6 +340,9 @@ class Symbol: self.rules = rules self.set_type(tp) + def set_is_payload(self, onoff): + self.is_payload = onoff + def set_type(self, tp): if tp == p_ruleset: self.term = None @@ -551,6 +555,8 @@ def grammar_get_types(grammar): pp = grammar[c.token] if pp.tp is p_terminal: continue + if not pp.is_payload: + continue members.append(tok2sym(c.token)) if True or len(members): rules.append(members) @@ -1035,6 +1041,12 @@ def grammar_trim_symbols(grammar, symbols): return grammar +# flag symbols as non-payload +def grammar_irrelevant_symbols(grammar, symbols): + for s in symbols: + grammar[s].set_is_payload(False) + + return grammar def grammar_create_ebnf(grammar, opts): indent = 40 slog(INFO, "creating ebnf from grammar of size", len(grammar)) @@ -1050,6 +1062,9 @@ def grammar_create_ebnf(grammar, opts): out += ' ' * indent + ' ;\n' return out +def format_token(sym, tp): + return misc.pad('%token <' + sym + '>', 27) + misc.pad(sym, 20) + '/* ' + tp + ' */' + def grammar_create_y(grammar, opts): indent = ' ' * 40 width = 0 @@ -1134,7 +1149,7 @@ def grammar_create_y(grammar, opts): out += '\n' for t, p in grammar.iteritems(): if p.tp == p_terminal: - out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n' + out += format_token(p.sym, t) +'\n' # special tokens out += '\n' @@ -1142,19 +1157,19 @@ def grammar_create_y(grammar, opts): if p.tp == p_special: if p.token == '?': # TODO: why is this among the symbols anyway? continue - out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n' + out += format_token(p.sym, t) +'\n' # regex tokens out += '\n' for t, p in grammar.iteritems(): if p.tp == p_literal: - out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n' + out += format_token(p.sym, t) +'\n' # types out += '\n' for t, p in grammar.iteritems(): if p.tp == p_ruleset: - out += '%type <' + tok2sym(p.token) + '> ' + t + (40 - len(t)) * ' ' + '/* ' + t + ' */' +'\n' + out += misc.pad('%type <' + tok2sym(p.token) + '>', 40) + misc.pad(t, 35) + '/* ' + t + ' */' +'\n' out += textwrap.dedent("""\ @@ -1207,6 +1222,8 @@ def grammar_create_y(grammar, opts): # continue if p.tp not in [ p_ruleset ]: continue + if not p.is_payload: + continue tp = tok2name(c.token) suffix = '' if tokens.count(c.token) > 1: @@ -1499,6 +1516,7 @@ class GrammarCmd(jwutils.Cmd): p.add_argument('-w', '--replace-whitespace', help='replace white space in tokens by underscore characters', action='store_true', default=False) p.add_argument('--check-symbols', help='check symbols, comma-separated or "all"', nargs='?', default='') p.add_argument('-t', '--trim-symbols', help='trim grammar tree at symbol', nargs='?', default='') + p.add_argument('-r', '--irrelevant-symbols', help='exclude symbol from output payload', nargs='?', default='') p.add_argument('-c', '--cut-symbols', help='cut grammar tree at symbol', nargs='?', default='') return p @@ -1518,12 +1536,14 @@ class GrammarCmd(jwutils.Cmd): args.check_symbols = '' check_symbols = args.check_symbols.split() grammar_check(grammar, check_symbols) + if args.replace_whitespace: + grammar = grammar_replace_whitespace(grammar) if len(args.trim_symbols): grammar = grammar_trim_symbols(grammar, args.trim_symbols.split(',')) if len(args.cut_symbols): grammar = grammar_cut_symbols(grammar, args.cut_symbols.split(',')) - if args.replace_whitespace: - grammar = grammar_replace_whitespace(grammar) + if len(args.irrelevant_symbols): + grammar = grammar_irrelevant_symbols(grammar, args.irrelevant_symbols.split(',')) return grammar # ------------------------------------------------- TODO: clean this up > diff --git a/tools/python/jwutils/log.py b/tools/python/jwutils/log.py index 9790689..9d5e967 100644 --- a/tools/python/jwutils/log.py +++ b/tools/python/jwutils/log.py @@ -3,6 +3,7 @@ import syslog import sys import inspect from os.path import basename +import misc EMERG = syslog.LOG_EMERG ALERT = syslog.LOG_ALERT @@ -63,15 +64,6 @@ prio_colors = { EMERG : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_MAGENTA, CONSOLE_FONT_OFF ], } -def __pad(token, total_size, right_align = False): - add = total_size - len(token) - if add <= 0: - return token - space = ' ' * add - if right_align: - return space + token - return token + space - def get_caller_pos(up = 1): assert(up == 1) # TODO: implement this caller_frame = inspect.currentframe().f_back.f_back @@ -95,7 +87,7 @@ def slog(prio, *args, **kwargs): # export name, line = kwargs['caller'] else: name, line = get_caller_pos(1) - msg += __pad(name, 20) + '[' + __pad(str(line), 4, True) + ']' + msg += misc.pad(name, 20) + '[' + misc.pad(str(line), 4, True) + ']' if f_color in flags: color_on, color_off = prio_colors[prio] diff --git a/tools/python/jwutils/misc.py b/tools/python/jwutils/misc.py index 848629b..f70d146 100644 --- a/tools/python/jwutils/misc.py +++ b/tools/python/jwutils/misc.py @@ -6,3 +6,12 @@ def silentremove(filename): #export except OSError as e: if e.errno != errno.ENOENT: raise # re-raise exception if a different error occurred + +def pad(token, total_size, right_align = False): + add = total_size - len(token) + if add <= 0: + return token + space = ' ' * add + if right_align: + return space + token + return token + space