grammar.py and friends: Make test/grammar compile and start

Doesn't successfully parse grammartest.code, yet, it errors out with a
syntax error on whitespace. But at least it compiles and starts.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/generate-flex-bison.mk

@@ -1,5 +1,3 @@
-GENERATED_STD          += $(FB_NAME).l $(FB_NAME).y $(FB_NAME).ebnf include/$(FB_NAME).h
-INCLUDED_BY_GENERATED  += include/defs.h include/$(FB_NAME).h include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
 NAMESPACE_IN_GENERATED ?= $(FB_NAME)
 
 # These types are meant to be cut off the tree and turned into hand coded flex
@@ -14,10 +12,22 @@ ifneq ($(CHECK_SYMBOLS),)
   OPT_CHECK_SYMBOLS  ?= --check-symbols='$(CHECK_SYMBOLS)'
 endif
 
+GENERATED_STD          += $(FB_NAME).l $(FB_NAME).y $(FB_NAME).ebnf $(FB_COMMON_H)
+GENERATED              += $(FB_NAME)-dense.ebnf $(GENERATED_STD)
 GRAMMAR_INPUT        ?= $(FB_NAME)-input.ebnf
-GENERATED             = $(FB_NAME)-dense.ebnf $(GENERATED_STD)
+FB_NAME_PREFIX       ?= $(FB_NAME)_
-GENERATE_PY           = ./generate.py
+FB_HDRDIR            ?= include
-GENERATE              = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \
+FB_BISON_OUT_EXT     ?= cpp
+FB_FLEX_OUT_EXT      ?= cpp
+FB_CASE_INSENSITIVE  ?= true
+FB_SRC               ?= $(filter %.y %.l,$(GENERATED))
+FB_COMMON_H          ?= $(FB_HDRDIR)/$(FB_NAME).h
+
+INCLUDED_BY_GENERATED  += include/defs.h $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
+
+
+GENERATE_PY          ?= ./generate.py
+GENERATE             ?= python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \
 				--fix-extensions $(FIX_EXTENSIONS) \
 				--unroll-lists \
 				--unroll-options \
@@ -30,13 +40,6 @@ GENERATE              = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL
 				--namespace=$(NAMESPACE_IN_GENERATED) \
 				--includes=$(shell echo $(INCLUDED_BY_GENERATED) | sed 's/  */,/g') \
 				$(CREATE_EXTRA_ARGS)
-FB_NAME_PREFIX       ?= $(FB_NAME)_
-FB_HDRDIR            ?= include
-FB_BISON_OUT_EXT     ?= cpp
-FB_FLEX_OUT_EXT      ?= cpp
-FB_CASE_INSENSITIVE  ?= true
-FB_SRC               ?= $(filter %.y %.l,$(GENERATED))
-
 include $(TOPDIR)/make/proj.mk
 include $(MODDIR)/make/flex-bison.mk
 include $(MODDIR)/make/py-defs.mk
@@ -46,9 +49,8 @@ debug-all:
 	GENERATE_LOG_LEVEL=debug make all 2>&1 | tee run.out
 
 generate: $(GENERATED)
-
+#$(FB_NAME).y: $(FB_COMMON_H) lex.$(FB_NAME).$(FB_FLEX_OUT_EXT)
-$(FB_NAME).y: include/$(FB_NAME).h
+#lex.$(FB_NAME).$(FB_FLEX_OUT_EXT): $(FB_NAME).l
-lex.$(FB_NAME).c: $(FB_NAME).l
 
 check: $(GRAMMAR_INPUT) $(GENERATE_PY) Makefile
 	python ./$(GENERATE_PY) --log-level info check --fix-extensions unroll --unroll-lists --unroll-options $(OPT_CHECK_SYMBOLS) $<

test/grammar/Makefile

@@ -1,10 +1,10 @@
 TOPDIR = ../..
 
-USE_PROJECT_LIB        = true
+EXE_ARGS              ?= grammartest.code
-MEMBERS               += local.a($(OBJ))
 PREREQ_BUILD          += ytools
 FB_NAME                = grammartest
 NAMESPACE_IN_GENERATED = gt
 
 include $(TOPDIR)/make/proj.mk
 include $(TOPDIR)/make/generate-flex-bison.mk
+include $(MODDIR)/make/exe.mk

test/grammar/grammartest.code

@@ -1,10 +1,10 @@
- PROGRAM DEMO1
+PROGRAM DEMO1
- BEGIN
+BEGIN
-   A:=3;
+  A:=3;
-   B:=45;
+  B:=45;
-   H:=-100023;
+  H:=-100023;
-   C:=A;
+  C:=A;
-   D123:=B34A;
+  D123:=B34A;
-   BABOON:=GIRAFFE;
+  BABOON:=GIRAFFE;
-   TEXT:="Hello world!";
+  TEXT:="Hello world!";
- END.
+END.

test/grammar/main.cpp

@@ -0,0 +1,44 @@
+#include <string>
+
+#include <stdio.h>
+
+#include <misc.h>
+#include <slog.h>
+#include <YMisc.h>
+
+// TODO: this should be included from grammartest.tab.h automatically
+#include "include/grammartest.h"
+
+#include "include/defs.h"
+#include "include/grammartest.tab.h"
+
+using namespace std;
+
+int main(int argc, const char *argv[])
+{
+	if (argc < 2) {
+		fprintf(stderr, "usage: %s input-file\n", filenotdir(argv[0]));
+		return 1;
+	}
+
+	const char *path = argv[1];
+	string content;
+	if (YMisc::suck_in_file(path, content)<0) {
+		slog(PRI_ERR, "failed to read [%s] (%s)", path, err());
+		return 1;
+	}
+
+	struct context context = {
+		line: 0
+	};
+
+	struct vp_scanner *scanner = FB_SYM(init_scanner)(content.c_str());
+	int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner));
+	FB_SYM(cleanup_scanner)(scanner);
+	if (status) {
+		slog(PRI_ERR, "failed to parse [%s] (%s)", path, err());
+		return -1;
+	}
+
+	return 0;
+}

tools/python/jwutils/grammar.py

@@ -1096,10 +1096,6 @@ def grammar_create_y(grammar, opts):
 
     for f in opts['includes']:
         out += '#include "' + f + '"' + '\n'
-    #include "include/defs.h"
-    #include "include/vhdl2017.h"
-    #include "include/lex.vhdl2017.h"
-    #include "include/vhdl2017.tab.h"
 
     out += "\nusing namespace " + opts['namespace'] + ';\n'
 
@@ -1261,30 +1257,28 @@ def grammar_create_l(grammar, opts):
 
     for f in opts['includes']:
         out += '#include "' + f + '"' + '\n'
-    #include "include/defs.h"
-    #include "include/vhdl2017.h"
-
-    #// #include "include/lex.vhdl2017.h"
-    #include "include/vhdl2017.tab.h"
 
     out += "\nusing namespace " + opts['namespace'] + ';\n'
 
+    #out += textwrap.dedent("""\
+
+    #    /* This is too late in the Flex generated file to work. Still lots of
+    #     * prototypes are spat into it above it, which end up with C++ linkage, of
+    #     * course, causing the linkages to be inconsistent to the functions below this
+    #     * extern "C". Only way I found was to use C++ is to use it on Bison only, and
+    #     * have Flex use C instead. */
+    #    #ifdef __cplusplus
+    #    // extern "C" {
+    #    #endif
+
+    #    #ifdef _REMOVE_ME
+    #    static void get_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip);
+    #    static void get_based_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip);
+    #    #endif
+
+    #    %}
+
     out += textwrap.dedent("""\
-
-        /* This is too late in the Flex generated file to work. Still lots of
-         * prototypes are spat into it above it, which end up with C++ linkage, of
-         * course, causing the linkages to be inconsistent to the functions below this
-         * extern "C". Only way I found was to use C++ is to use it on Bison only, and
-         * have Flex use C instead. */
-        #ifdef __cplusplus
-        // extern "C" {
-        #endif
-
-        #ifdef _REMOVE_ME
-        static void get_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip);
-        static void get_based_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip);
-        #endif
-
         %}
 
         %%
@@ -1300,22 +1294,22 @@ def grammar_create_l(grammar, opts):
             assert p.term[-1] in [ '"', "'" ], p.term
             out += re.escape(p.term[1:-1]) + ' { return ' + p.sym + '; }\n'
 
-    out += textwrap.dedent("""\
+    #out += textwrap.dedent("""\
-
+    #
-        %{/* basic_identifier */%}
+    #    %{/* basic_identifier */%}
-        %{/* extended_identifier */%}
+    #    %{/* extended_identifier */%}
-        %{/* based_integer */%}
+    #    %{/* based_integer */%}
-        %{/* bit_value */%}
+    #    %{/* bit_value */%}
-        %{/* numeric_literal */%}
+    #    %{/* numeric_literal */%}
-        %{/* enumeration_literal */%}
+    #    %{/* enumeration_literal */%}
-        %{/* string_literal */%}
+    #    %{/* string_literal */%}
-        %{/* bit_string_literal */%}
+    #    %{/* bit_string_literal */%}
-        %{/* character_literal */%}
+    #    %{/* character_literal */%}
-        %{/* graphic_character */%}
+    #    %{/* graphic_character */%}
-        %{/* basic_character */%}
+    #    %{/* basic_character */%}
-        %{/* integer */%}
+    #    %{/* integer */%}
-
+    #
-	""")
+    #	""")
 
     ignore += textwrap.dedent("""\
 
@@ -1378,8 +1372,7 @@ def grammar_create_l(grammar, opts):
         void FB_SYM(error)(struct context *context, void *scanner, const char *msg)
         {
         	struct yyguts_t *yyg =(struct yyguts_t*)scanner;
-        	// vp_log(context->vp, VP_LOG_ERR, "%s at \\"%s\\" in line %d.\\n\\n", msg, yytext, context->lineno);
+        	set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d", msg, yytext, context->line);
-        	slog(PRI_ERR, "%s at \\"%s\\" in line %d.\\n\\n", msg, yytext, context->line);
         }
 
         int FB_SYM(wrap)(void *scanner)
@@ -1395,7 +1388,7 @@ def grammar_create_l(grammar, opts):
 
         /* utilities which need to be placed here, because I can't find
          * yylex_init() / _destroy() in any generated header file (??) */
-        struct vp_scanner *vhdl_default_init_scanner(const char *str)
+        struct vp_scanner *FB_SYM(init_scanner)(const char *str)
         {
         	struct vp_scanner *r = (struct vp_scanner *)calloc(1, sizeof(*r));
 
@@ -1408,12 +1401,12 @@ def grammar_create_l(grammar, opts):
         	return r;
         }
 
-        void *vhdl_default_scanner_get_data(const struct vp_scanner *scanner)
+        void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner)
         {
         	return scanner->scanner;
         }
 
-        void vhdl_default_cleanup_scanner(struct vp_scanner *scanner)
+        void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner)
         {
         	free(scanner->str);
         	yy_delete_buffer(scanner->buf, scanner->scanner);
@@ -1421,12 +1414,14 @@ def grammar_create_l(grammar, opts):
         	free(scanner);
         }
 
-        #ifdef __cplusplus
-        // } // extern "C"
-        #endif
-
 	""")
 
+    #    #ifdef __cplusplus
+    #    // } // extern "C"
+    #    #endif
+    #
+    #	""")
+
     return out
 
 def grammar_create_h(grammar, opts):
@@ -1497,7 +1492,18 @@ def grammar_create_h(grammar, opts):
 
     if ns is not None:
         out += '\n} /* namespace ' + ns + '*/'
-    out += '\n#endif /* #ifndef + ' + opts['mip'] + ' */'
+
+    out += textwrap.dedent("""\
+
+        struct vp_scanner;
+
+        struct vp_scanner *FB_SYM(init_scanner)(const char *str);
+        void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner);
+        void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner);
+
+	""")
+
+    out += '\n\n#endif /* #ifndef + ' + opts['mip'] + ' */'
 
     return out