grammar.py: Add support for --irrelevant-symbols

Arguments to --irrelevant-symbols are not meant to be represented in the
AST resulting from parsing.

Also, add pad() to misc.py.

Signed-off-by: Jan Lindemann <jan@janware.com>

tools/python/jwutils/grammar.py

@@ -138,7 +138,7 @@ def tok2sym(tok):
     if term is not None:
         if term in special_terminals.keys():
            return "T_" + special_terminals[term].upper()
-        return "T_" + term.upper()
+        return "T_" + re.sub('[^a-zA-Z0-9]', '_', term).upper()
     return tok
 
 def tok2regex(tok):
@@ -319,6 +319,7 @@ class Symbol:
 
     def __init__(self, token, tp = None, rules = None):
         self.reset(token, tp, rules)
+        self.set_is_payload(True)
 
     def reset(self, token, tp = None, rules = None):
         if tp == None:
@@ -339,6 +340,9 @@ class Symbol:
             self.rules = rules
         self.set_type(tp)
 
+    def set_is_payload(self, onoff):
+        self.is_payload = onoff
+
     def set_type(self, tp):
         if tp == p_ruleset:
             self.term = None
@@ -551,6 +555,8 @@ def grammar_get_types(grammar):
                 pp = grammar[c.token]
                 if pp.tp is p_terminal:
                     continue
+                if not pp.is_payload:
+                    continue
                 members.append(tok2sym(c.token))
             if True or len(members):
                 rules.append(members)
@@ -1035,6 +1041,12 @@ def grammar_trim_symbols(grammar, symbols):
 
     return grammar
 
+# flag symbols as non-payload
+def grammar_irrelevant_symbols(grammar, symbols):
+    for s in symbols:
+        grammar[s].set_is_payload(False)
+
+    return grammar
 def grammar_create_ebnf(grammar, opts):
     indent = 40
     slog(INFO, "creating ebnf from grammar of size", len(grammar))
@@ -1050,6 +1062,9 @@ def grammar_create_ebnf(grammar, opts):
 	out += ' ' * indent + ' ;\n'
     return out
 
+def format_token(sym, tp):
+    return misc.pad('%token <' + sym + '>', 27) + misc.pad(sym, 20) + '/* ' + tp + ' */'
+
 def grammar_create_y(grammar, opts):
     indent = ' ' * 40
     width = 0
@@ -1134,7 +1149,7 @@ def grammar_create_y(grammar, opts):
     out += '\n'
     for t, p in grammar.iteritems():
         if p.tp == p_terminal:
-            out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n'
+            out += format_token(p.sym, t) +'\n'
 
     # special tokens
     out += '\n'
@@ -1142,19 +1157,19 @@ def grammar_create_y(grammar, opts):
         if p.tp == p_special:
             if p.token == '?': # TODO: why is this among the symbols anyway?
                 continue
-            out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n'
+            out += format_token(p.sym, t) +'\n'
 
     # regex tokens
     out += '\n'
     for t, p in grammar.iteritems():
         if p.tp == p_literal:
-            out += '%token <' + p.sym + '> ' + p.sym + (20 - len(p.sym)) * ' ' + '/* ' + t + ' */' +'\n'
+            out += format_token(p.sym, t) +'\n'
 
     # types
     out += '\n'
     for t, p in grammar.iteritems():
         if p.tp == p_ruleset:
-            out += '%type <' + tok2sym(p.token) + '> ' + t + (40 - len(t)) * ' ' + '/* ' + t + ' */' +'\n'
+            out += misc.pad('%type <' + tok2sym(p.token) + '>', 40) + misc.pad(t, 35) + '/* ' + t + ' */' +'\n'
 
     out += textwrap.dedent("""\
 
@@ -1207,6 +1222,8 @@ def grammar_create_y(grammar, opts):
                 #    continue
                 if p.tp not in [ p_ruleset ]:
                     continue
+                if not p.is_payload:
+                    continue
                 tp = tok2name(c.token)
                 suffix = ''
                 if tokens.count(c.token) > 1:
@@ -1499,6 +1516,7 @@ class GrammarCmd(jwutils.Cmd):
         p.add_argument('-w', '--replace-whitespace', help='replace white space in tokens by underscore characters', action='store_true', default=False)
         p.add_argument('--check-symbols', help='check symbols, comma-separated or "all"', nargs='?', default='')
         p.add_argument('-t', '--trim-symbols', help='trim grammar tree at symbol', nargs='?', default='')
+        p.add_argument('-r', '--irrelevant-symbols', help='exclude symbol from output payload', nargs='?', default='')
         p.add_argument('-c', '--cut-symbols', help='cut grammar tree at symbol', nargs='?', default='')
         return p
 
@@ -1518,12 +1536,14 @@ class GrammarCmd(jwutils.Cmd):
                 args.check_symbols = ''
             check_symbols = args.check_symbols.split()
             grammar_check(grammar, check_symbols)
+        if args.replace_whitespace:
+            grammar = grammar_replace_whitespace(grammar)
         if len(args.trim_symbols):
             grammar = grammar_trim_symbols(grammar, args.trim_symbols.split(','))
         if len(args.cut_symbols):
             grammar = grammar_cut_symbols(grammar, args.cut_symbols.split(','))
-        if args.replace_whitespace:
-            grammar = grammar_replace_whitespace(grammar)
+        if len(args.irrelevant_symbols):
+            grammar = grammar_irrelevant_symbols(grammar, args.irrelevant_symbols.split(','))
         return grammar
 
 # ------------------------------------------------- TODO: clean this up >