janware/jw-python
1
mirror of ssh://git.janware.com/srv/git/janware/proj/jw-python synced 2026-01-15 09:53:32 +01:00
Code Releases Activity

grammar.py: Add support for --replace-whitespace

Browse source 
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2017-10-29 13:56:52 +01:00
commit 4fee6ac06e
2 changed files with 23 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
test/grammar/Makefile
View file

@ -22,6 +22,7 @@ GENERATE = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL
--unroll-lists \ --unroll-lists \
--unroll-options \ --unroll-options \
--unroll-alternatives \ --unroll-alternatives \
--replace-whitespace \
$(CHECK_SYMBOLS) \ $(CHECK_SYMBOLS) \
--trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/ */,/g') \ --trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/ */,/g') \
$(CREATE_EXTRA_ARGS) $(CREATE_EXTRA_ARGS)

22
tools/python/jwutils/grammar.py
View file

@ -314,6 +314,9 @@ class State:
class Symbol: class Symbol:
def __init__(self, token, tp = None, rules = None): def __init__(self, token, tp = None, rules = None):
self.reset(token, tp, rules)
def reset(self, token, tp = None, rules = None):
if tp == None: if tp == None:
if is_terminal(token) is not None: if is_terminal(token) is not None:
tp = p_terminal tp = p_terminal
@ -786,6 +789,22 @@ def grammar_unroll_alternatives(grammar):
grammar[tok].rules = rules_unroll_alternatives(p.rules) grammar[tok].rules = rules_unroll_alternatives(p.rules)
return grammar return grammar
def grammar_replace_whitespace(grammar):
r = OrderedDict()
for tok, s in grammar.iteritems():
newrules = []
for rule in s.rules:
newrule = []
for c in rule:
newc = RuleComp(c.token.replace(' ', '_'), tp=c.tp, line=c.line)
newrule.append(newc)
newrules.append(newrule)
newtok = tok.replace(' ', '_')
s.reset(newtok, tp=s.tp, rules=newrules)
r[newtok] = s
slog(INFO, "added symbol", newtok)
return r
def step_out(grammar, terminals, orphans, lexicals, tok, depth, checked = None, found = None): def step_out(grammar, terminals, orphans, lexicals, tok, depth, checked = None, found = None):
if checked is None: if checked is None:
checked = set() checked = set()
@ -1447,6 +1466,7 @@ class GrammarCmd(jwutils.Cmd):
p.add_argument('-e', '--fix-extensions', help='fix EBNF prefix extensions (' + '|'.join(fix_extensions_mode) + ')', default=mode_concat) p.add_argument('-e', '--fix-extensions', help='fix EBNF prefix extensions (' + '|'.join(fix_extensions_mode) + ')', default=mode_concat)
p.add_argument('-o', '--unroll-options', help='unroll EBNF options', action='store_true', default=False) p.add_argument('-o', '--unroll-options', help='unroll EBNF options', action='store_true', default=False)
p.add_argument('-a', '--unroll-alternatives', help='unroll EBNF alternatives', action='store_true', default=False) p.add_argument('-a', '--unroll-alternatives', help='unroll EBNF alternatives', action='store_true', default=False)
p.add_argument('-w', '--replace-whitespace', help='replace white space in tokens by underscore characters', action='store_true', default=False)
p.add_argument('--check-symbols', help='check symbols, comma-separated or "all"', nargs='?', default='') p.add_argument('--check-symbols', help='check symbols, comma-separated or "all"', nargs='?', default='')
p.add_argument('-t', '--trim-symbols', help='trim grammar tree at symbol', nargs='?', default='') p.add_argument('-t', '--trim-symbols', help='trim grammar tree at symbol', nargs='?', default='')
p.add_argument('-c', '--cut-symbols', help='cut grammar tree at symbol', nargs='?', default='') p.add_argument('-c', '--cut-symbols', help='cut grammar tree at symbol', nargs='?', default='')
@ -1472,4 +1492,6 @@ class GrammarCmd(jwutils.Cmd):
grammar = grammar_trim_symbols(grammar, args.trim_symbols.split(',')) grammar = grammar_trim_symbols(grammar, args.trim_symbols.split(','))
if len(args.cut_symbols): if len(args.cut_symbols):
grammar = grammar_cut_symbols(grammar, args.cut_symbols.split(',')) grammar = grammar_cut_symbols(grammar, args.cut_symbols.split(','))
if args.replace_whitespace:
grammar = grammar_replace_whitespace(grammar)
return grammar return grammar