grammar.py: Add support for --replace-whitespace

Signed-off-by: Jan Lindemann <jan@janware.com>

test/grammar/Makefile

@@ -22,6 +22,7 @@ GENERATE              = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL
 				--unroll-lists \
 				--unroll-options \
 				--unroll-alternatives \
+				--replace-whitespace \
 				$(CHECK_SYMBOLS) \
 				--trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/  */,/g') \
 				$(CREATE_EXTRA_ARGS)

tools/python/jwutils/grammar.py

@@ -314,6 +314,9 @@ class State:
 class Symbol:
 
     def __init__(self, token, tp = None, rules = None):
+        self.reset(token, tp, rules)
+
+    def reset(self, token, tp = None, rules = None):
         if tp == None:
             if is_terminal(token) is not None:
                 tp = p_terminal
@@ -786,6 +789,22 @@ def grammar_unroll_alternatives(grammar):
         grammar[tok].rules = rules_unroll_alternatives(p.rules)
     return grammar
 
+def grammar_replace_whitespace(grammar):
+    r = OrderedDict()
+    for tok, s in grammar.iteritems():
+        newrules = []
+        for rule in s.rules:
+            newrule = []
+            for c in rule:
+                newc = RuleComp(c.token.replace(' ', '_'), tp=c.tp, line=c.line)
+                newrule.append(newc)
+            newrules.append(newrule)
+        newtok = tok.replace(' ', '_')
+        s.reset(newtok, tp=s.tp, rules=newrules)
+        r[newtok] = s
+        slog(INFO, "added symbol", newtok)
+    return r
+
 def step_out(grammar, terminals, orphans, lexicals, tok, depth, checked = None, found = None):
     if checked is None:
         checked = set()
@@ -1447,6 +1466,7 @@ class GrammarCmd(jwutils.Cmd):
         p.add_argument('-e', '--fix-extensions', help='fix EBNF prefix extensions (' + '|'.join(fix_extensions_mode) + ')', default=mode_concat)
         p.add_argument('-o', '--unroll-options', help='unroll EBNF options', action='store_true', default=False)
         p.add_argument('-a', '--unroll-alternatives', help='unroll EBNF alternatives', action='store_true', default=False)
+        p.add_argument('-w', '--replace-whitespace', help='replace white space in tokens by underscore characters', action='store_true', default=False)
         p.add_argument('--check-symbols', help='check symbols, comma-separated or "all"', nargs='?', default='')
         p.add_argument('-t', '--trim-symbols', help='trim grammar tree at symbol', nargs='?', default='')
         p.add_argument('-c', '--cut-symbols', help='cut grammar tree at symbol', nargs='?', default='')
@@ -1472,4 +1492,6 @@ class GrammarCmd(jwutils.Cmd):
             grammar = grammar_trim_symbols(grammar, args.trim_symbols.split(','))
         if len(args.cut_symbols):
             grammar = grammar_cut_symbols(grammar, args.cut_symbols.split(','))
+        if args.replace_whitespace:
+            grammar = grammar_replace_whitespace(grammar)
         return grammar