grammar.py et al: Centralize more code

More code is removed from the special parser directories and centralized
into grammar.py, Cmd.py, and generate-flex-bison.mk.

Signed-off-by: Jan Lindemann <jan@janware.com>

test/grammar/Makefile

@@ -1,75 +1,10 @@
 TOPDIR = ../..
 
-USE_PROJECT_LIB = true
+USE_PROJECT_LIB        = true
 MEMBERS               += local.a($(OBJ))
-
-GENERATED_STD         = grammartest.l grammartest.y grammartest.ebnf include/grammartest.h
-
-# These types are meant to be cut off the tree and turned into hand coded flex
-# regexes
-#TRIM_SYMBOLS          = blah
-TRIM_SYMBOLS          = 
-GENERATE_LOG_LEVEL   ?= notice
-FIX_EXTENSIONS       ?= discard
-CHECK_SYMBOLS        ?= --check-symbols=all
-
-
-GRAMMAR_INPUT        ?= grammartest-input.ebnf
-GENERATED             = grammartest-dense.ebnf $(GENERATED_STD)
-GENERATE_PY           = ./generate.py
-GENERATE              = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \
-				--fix-extensions $(FIX_EXTENSIONS) \
-				--unroll-lists \
-				--unroll-options \
-				--unroll-alternatives \
-				--replace-whitespace \
-				$(CHECK_SYMBOLS) \
-				--trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/  */,/g') \
-				$(CREATE_EXTRA_ARGS)
-CHECK_SYMBOLS         ?= special_character
-
-FB_NAME_PREFIX       ?= grammartest_
-FB_HDRDIR            ?= include
-FB_BISON_OUT_EXT     ?= cpp
-FB_FLEX_OUT_EXT      ?= cpp
-FB_CASE_INSENSITIVE  ?= true
-FB_SRC               ?= $(filter %.y %.l,$(GENERATED))
+PREREQ_BUILD          += ytools
+FB_NAME                = grammartest
+NAMESPACE_IN_GENERATED = gt
 
 include $(TOPDIR)/make/proj.mk
-include $(MODDIR)/make/flex-bison.mk
-include $(MODDIR)/make/py-defs.mk
-
-all:
-debug-all:
-	GENERATE_LOG_LEVEL=debug make all 2>&1 | tee run.out
-
-generate: $(GENERATED)
-
-grammartest.y: include/grammartest.h
-lex.grammartest.c: grammartest.l
-
-check: $(GRAMMAR_INPUT) $(GENERATE_PY) Makefile
-	python ./$(GENERATE_PY) --log-level info check --fix-extensions unroll --unroll-lists --unroll-options --check-symbols='$(CHECK_SYMBOLS)' $<
-
-grammartest-dense.ebnf: $(GRAMMAR_INPUT) $(GENERATE_PY)
-	python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create --fix-extensions keep $< grammartest.ebnf > $@.tmp
-	mv $@.tmp $@
-
-define generate_rule
-$(1): $$(GRAMMAR_INPUT) $$(GENERATE_PY) Makefile
-	$$(GENERATE) $$< $$(patsubst grammartest.%,grammartest.%,$$@) > $$@.tmp
-	mv $$@.tmp $$@
-endef
-$(foreach target,$(GENERATED_STD),$(eval $(call generate_rule,$(target))))
-
-clean.generated:
-	rm -f $(GENERATED)
-clean: clean.generated
-echo-generated:
-	@echo GENERATED = $(GENERATED)
-
-help:
-	$(GENERATE) --help
-
-expand-macros:
-	make 2>/dev/null | sed '/g++/ !d; s/g++\|gcc//; s/-o .*//' | xargs g++ -E -C | indent
+include $(TOPDIR)/make/generate-flex-bison.mk

test/grammar/generate.py

@@ -1,79 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
-import argparse
-import sys
-import re
-import textwrap
-from collections import OrderedDict
-from abc import abstractmethod
-
 import jwutils
 
-from jwutils.log import *
-from jwutils import grammar
-
-base = 'grammartest'
-mip = '_JW_PYTHON_' + base + base.upper()
-namespace = base
-
-def create_grammartest_ebnf(grammar):
-    print(jwutils.grammar.create_ebnf(grammar))
-
-def create_grammartest_y(grammar):
-    print(jwutils.grammar.create_yacc(grammar))
-
-def create_grammartest_l(grammar):
-    print(jwutils.grammar.create_lex(grammar))
-
-def create_include_grammartest_h(grammar):
-    print(jwutils.grammar.create_header(grammar, mip=mip, namespace=namespace))
-
-class GrammarCmd(jwutils.grammar.GrammarCmd):
-
-    def __init__(self, name, help):
-        super(GrammarCmd, self).__init__(name, help=help)
-
-    @abstractmethod
-    def _run(self, grammar):
-        pass
-
-    def add_parser(self, parsers):
-        p = super(GrammarCmd, self).add_parser(parsers)
-        return p
-
-    def run(self, args):
-        with open(args.input, 'r') as infile:
-            contents = infile.read()
-        grammar = jwutils.grammar.grammar_parse_ebnf(contents)
-        grammar = super(GrammarCmd, self).processGrammar(args, grammar)
-        self._run(args, grammar)
-
-class CmdCreate(GrammarCmd):
-
-    def __init__(self):
-        super(CmdCreate, self).__init__("create", help="Create a file")
-
-    def add_parser(self, parsers):
-        p = super(CmdCreate, self).add_parser(parsers)
-        p.add_argument("output", help="output file")
-        return p
-
-    def _run(self, args, grammar):
-        cmd = getattr(sys.modules[__name__], 'create_' + re.sub(r'[-./]', '_', args.output))
-        cmd(grammar)
-
-class CmdCheck(GrammarCmd):
-
-    def __init__(self):
-        super(CmdCheck, self).__init__("check", help="Check grammar")
-
-    def add_parser(self, parsers):
-        p = super(CmdCheck, self).add_parser(parsers)
-        return p
-
-    def _run(self, args, grammar):
-        pass
-
-jwutils.run_sub_commands('generate Test parser files')
+jwutils.run_sub_commands('generate Test parser files', modules = ['jwutils.grammar'])