Continue implementation of grammar.py

Signed-off-by: Jan Lindemann <jan@janware.com>

test/grammar/Makefile

@@ -0,0 +1,70 @@
+TOPDIR = ../..
+
+GENERATED_STD         = grammartest.l grammartest.y grammartest.ebnf include/grammartest.h
+
+# These types are meant to be cut off the tree and turned into hand coded flex
+# regexes
+#TRIM_SYMBOLS          = blah
+TRIM_SYMBOLS          = 
+GENERATE_LOG_LEVEL   ?= notice
+FIX_EXTENSIONS       ?= discard
+CHECK_SYMBOLS        ?= --check-symbols=all
+
+
+GRAMMAR_INPUT        ?= grammartest-input.ebnf
+GENERATED             = grammartest-dense.ebnf $(GENERATED_STD)
+GENERATE_PY           = ./generate.py
+GENERATE              = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \
+				--fix-extensions $(FIX_EXTENSIONS) \
+				--unroll-lists \
+				--unroll-options \
+				$(CHECK_SYMBOLS) \
+				--trim-symbols=$(shell echo $(TRIM_SYMBOLS) | sed 's/  */,/g') \
+				$(CREATE_EXTRA_ARGS)
+CHECK_SYMBOLS         ?= special_character
+
+FB_NAME_PREFIX       ?= grammartest_
+FB_HDRDIR            ?= include
+FB_BISON_OUT_EXT     ?= cpp
+FB_FLEX_OUT_EXT      ?= cpp
+FB_CASE_INSENSITIVE  ?= true
+FB_SRC               ?= $(filter %.y %.l,$(GENERATED))
+
+include $(TOPDIR)/make/proj.mk
+include $(MODDIR)/make/flex-bison.mk
+include $(MODDIR)/make/py-defs.mk
+
+all:
+debug-all:
+	GENERATE_LOG_LEVEL=debug make all 2>&1 | tee run.out
+
+generate: $(GENERATED)
+
+grammartest.y: include/grammartest.h
+lex.grammartest.c: grammartest.l
+
+check: $(GRAMMAR_INPUT) $(GENERATE_PY) Makefile
+	python ./$(GENERATE_PY) --log-level info check --fix-extensions unroll --unroll-lists --unroll-options --check-symbols='$(CHECK_SYMBOLS)' $<
+
+grammartest-dense.ebnf: $(GRAMMAR_INPUT) $(GENERATE_PY)
+	python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create --fix-extensions keep $< grammartest.ebnf > $@.tmp
+	mv $@.tmp $@
+
+define generate_rule
+$(1): $$(GRAMMAR_INPUT) $$(GENERATE_PY) Makefile
+	$$(GENERATE) $$< $$(patsubst grammartest.%,grammartest.%,$$@) > $$@.tmp
+	mv $$@.tmp $$@
+endef
+$(foreach target,$(GENERATED_STD),$(eval $(call generate_rule,$(target))))
+
+clean.generated:
+	rm -f $(GENERATED)
+clean: clean.generated
+echo-generated:
+	@echo GENERATED = $(GENERATED)
+
+help:
+	$(GENERATE) --help
+
+expand-macros:
+	make 2>/dev/null | sed '/g++/ !d; s/g++\|gcc//; s/-o .*//' | xargs g++ -E -C | indent

test/grammar/generate.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+import argparse
+import sys
+import re
+import textwrap
+from collections import OrderedDict
+from abc import abstractmethod
+
+import jwutils
+
+from jwutils.log import *
+from jwutils import grammar
+
+base = 'grammartest'
+mip = '_JW_PYTHON_' + base + base.upper()
+namespace = base
+
+def create_grammartest_ebnf(grammar):
+    print(jwutils.grammar.create_ebnf(grammar))
+
+def create_grammartest_y(grammar):
+    print(jwutils.grammar.create_yacc(grammar))
+
+def create_grammartest_l(grammar):
+    print(jwutils.grammar.create_lex(grammar))
+
+def create_include_grammartest_h(grammar):
+    print(jwutils.grammar.create_header(grammar, mip=mip, namespace=namespace))
+
+class GrammarCmd(jwutils.grammar.GrammarCmd):
+
+    def __init__(self, name, help):
+        super(GrammarCmd, self).__init__(name, help=help)
+
+    @abstractmethod
+    def _run(self, grammar):
+        pass
+
+    def add_parser(self, parsers):
+        p = super(GrammarCmd, self).add_parser(parsers)
+        return p
+
+    def run(self, args):
+        with open(args.input, 'r') as infile:
+            contents = infile.read()
+        grammar = jwutils.grammar.grammar_parse_ebnf(contents)
+
+        slog(INFO, "grammar size is", len(grammar))
+        for t in grammar.keys():
+            slog(INFO, "key =", t)
+        slog(INFO, "grammar size is", len(grammar))
+        jwutils.grammar.dump_grammar(INFO, grammar)
+        grammar = super(GrammarCmd, self).processGrammar(args, grammar)
+        self._run(args, grammar)
+
+class CmdCreate(GrammarCmd):
+
+    def __init__(self):
+        super(CmdCreate, self).__init__("create", help="Create a file")
+
+    def add_parser(self, parsers):
+        p = super(CmdCreate, self).add_parser(parsers)
+        p.add_argument("output", help="output file")
+        return p
+
+    def _run(self, args, grammar):
+        cmd = getattr(sys.modules[__name__], 'create_' + re.sub(r'[-./]', '_', args.output))
+        cmd(grammar)
+
+class CmdCheck(GrammarCmd):
+
+    def __init__(self):
+        super(CmdCheck, self).__init__("check", help="Check grammar")
+
+    def add_parser(self, parsers):
+        p = super(CmdCheck, self).add_parser(parsers)
+        return p
+
+    def _run(self, args, grammar):
+        pass
+
+jwutils.run_sub_commands('generate Test parser files')

test/grammar/grammartest-input.ebnf

@@ -0,0 +1,16 @@
+ (* a simple program syntax in EBNF − Wikipedia *)
+ program = 'PROGRAM', white space, identifier, white space, 
+            'BEGIN', white space, 
+            { assignment, ";", white space }, 
+            'END.' ;
+ identifier = alphabetic character, { alphabetic character | digit } ;
+ number = [ "-" ], digit, { digit } ;
+ string = '"' , { all characters - '"' }, '"' ;
+ assignment = identifier , ":=" , ( number | identifier | string ) ;
+ alphabetic character = "A" | "B" | "C" | "D" | "E" | "F" | "G"
+                      | "H" | "I" | "J" | "K" | "L" | "M" | "N"
+                      | "O" | "P" | "Q" | "R" | "S" | "T" | "U"
+                      | "V" | "W" | "X" | "Y" | "Z" ;
+ digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" ;
+ white space = ? white space characters ? ;
+ all characters = ? all visible characters ? ;

test/grammar/grammartest.code

@@ -0,0 +1,10 @@
+ PROGRAM DEMO1
+ BEGIN
+   A:=3;
+   B:=45;
+   H:=-100023;
+   C:=A;
+   D123:=B34A;
+   BABOON:=GIRAFFE;
+   TEXT:="Hello world!";
+ END.

test/grammar/include/defs.h

@@ -0,0 +1,32 @@
+#ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
+#define _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
+
+#define YY_NO_INPUT
+#define YY_NO_UNPUT
+// #define YY_NO_UNISTD_H
+
+struct context {
+	int line;
+};
+
+union YYSTYPE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* defined in grammartest-parser.l */
+struct vp_scanner;
+struct vp_scanner *grammartest_default_init_scanner(const char *str);
+void *grammartest_default_scanner_get_data(const struct vp_scanner *scanner);
+void grammartest_default_cleanup_scanner(struct vp_scanner *scanner);
+
+void FB_SYM(error)(struct context *context, void *scanner, const char *s);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner)
+
+#endif /* #ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H */