From 16ce7abd9306be1a317508efa122e2678ab7be1e Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 30 Oct 2017 13:05:22 +0100 Subject: [PATCH] grammar.py and friends: Make test/grammar compile and start Doesn't successfully parse grammartest.code, yet, it errors out with a syntax error on whitespace. But at least it compiles and starts. Signed-off-by: Jan Lindemann --- make/generate-flex-bison.mk | 32 +++++----- test/grammar/Makefile | 4 +- test/grammar/grammartest.code | 20 +++--- test/grammar/main.cpp | 44 +++++++++++++ tools/python/jwutils/grammar.py | 106 +++++++++++++++++--------------- 5 files changed, 129 insertions(+), 77 deletions(-) create mode 100644 test/grammar/main.cpp diff --git a/make/generate-flex-bison.mk b/make/generate-flex-bison.mk index c8f8361..5387554 100644 --- a/make/generate-flex-bison.mk +++ b/make/generate-flex-bison.mk @@ -1,5 +1,3 @@ -GENERATED_STD += $(FB_NAME).l $(FB_NAME).y $(FB_NAME).ebnf include/$(FB_NAME).h -INCLUDED_BY_GENERATED += include/defs.h include/$(FB_NAME).h include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h NAMESPACE_IN_GENERATED ?= $(FB_NAME) # These types are meant to be cut off the tree and turned into hand coded flex @@ -14,10 +12,22 @@ ifneq ($(CHECK_SYMBOLS),) OPT_CHECK_SYMBOLS ?= --check-symbols='$(CHECK_SYMBOLS)' endif +GENERATED_STD += $(FB_NAME).l $(FB_NAME).y $(FB_NAME).ebnf $(FB_COMMON_H) +GENERATED += $(FB_NAME)-dense.ebnf $(GENERATED_STD) GRAMMAR_INPUT ?= $(FB_NAME)-input.ebnf -GENERATED = $(FB_NAME)-dense.ebnf $(GENERATED_STD) -GENERATE_PY = ./generate.py -GENERATE = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \ +FB_NAME_PREFIX ?= $(FB_NAME)_ +FB_HDRDIR ?= include +FB_BISON_OUT_EXT ?= cpp +FB_FLEX_OUT_EXT ?= cpp +FB_CASE_INSENSITIVE ?= true +FB_SRC ?= $(filter %.y %.l,$(GENERATED)) +FB_COMMON_H ?= $(FB_HDRDIR)/$(FB_NAME).h + +INCLUDED_BY_GENERATED += include/defs.h $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h + + +GENERATE_PY ?= ./generate.py +GENERATE ?= python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \ --fix-extensions $(FIX_EXTENSIONS) \ --unroll-lists \ --unroll-options \ @@ -30,13 +40,6 @@ GENERATE = python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL --namespace=$(NAMESPACE_IN_GENERATED) \ --includes=$(shell echo $(INCLUDED_BY_GENERATED) | sed 's/ */,/g') \ $(CREATE_EXTRA_ARGS) -FB_NAME_PREFIX ?= $(FB_NAME)_ -FB_HDRDIR ?= include -FB_BISON_OUT_EXT ?= cpp -FB_FLEX_OUT_EXT ?= cpp -FB_CASE_INSENSITIVE ?= true -FB_SRC ?= $(filter %.y %.l,$(GENERATED)) - include $(TOPDIR)/make/proj.mk include $(MODDIR)/make/flex-bison.mk include $(MODDIR)/make/py-defs.mk @@ -46,9 +49,8 @@ debug-all: GENERATE_LOG_LEVEL=debug make all 2>&1 | tee run.out generate: $(GENERATED) - -$(FB_NAME).y: include/$(FB_NAME).h -lex.$(FB_NAME).c: $(FB_NAME).l +#$(FB_NAME).y: $(FB_COMMON_H) lex.$(FB_NAME).$(FB_FLEX_OUT_EXT) +#lex.$(FB_NAME).$(FB_FLEX_OUT_EXT): $(FB_NAME).l check: $(GRAMMAR_INPUT) $(GENERATE_PY) Makefile python ./$(GENERATE_PY) --log-level info check --fix-extensions unroll --unroll-lists --unroll-options $(OPT_CHECK_SYMBOLS) $< diff --git a/test/grammar/Makefile b/test/grammar/Makefile index a565ca1..fd991fc 100644 --- a/test/grammar/Makefile +++ b/test/grammar/Makefile @@ -1,10 +1,10 @@ TOPDIR = ../.. -USE_PROJECT_LIB = true -MEMBERS += local.a($(OBJ)) +EXE_ARGS ?= grammartest.code PREREQ_BUILD += ytools FB_NAME = grammartest NAMESPACE_IN_GENERATED = gt include $(TOPDIR)/make/proj.mk include $(TOPDIR)/make/generate-flex-bison.mk +include $(MODDIR)/make/exe.mk diff --git a/test/grammar/grammartest.code b/test/grammar/grammartest.code index 386e6ce..2545cbd 100644 --- a/test/grammar/grammartest.code +++ b/test/grammar/grammartest.code @@ -1,10 +1,10 @@ - PROGRAM DEMO1 - BEGIN - A:=3; - B:=45; - H:=-100023; - C:=A; - D123:=B34A; - BABOON:=GIRAFFE; - TEXT:="Hello world!"; - END. +PROGRAM DEMO1 +BEGIN + A:=3; + B:=45; + H:=-100023; + C:=A; + D123:=B34A; + BABOON:=GIRAFFE; + TEXT:="Hello world!"; +END. diff --git a/test/grammar/main.cpp b/test/grammar/main.cpp new file mode 100644 index 0000000..88bf135 --- /dev/null +++ b/test/grammar/main.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +// TODO: this should be included from grammartest.tab.h automatically +#include "include/grammartest.h" + +#include "include/defs.h" +#include "include/grammartest.tab.h" + +using namespace std; + +int main(int argc, const char *argv[]) +{ + if (argc < 2) { + fprintf(stderr, "usage: %s input-file\n", filenotdir(argv[0])); + return 1; + } + + const char *path = argv[1]; + string content; + if (YMisc::suck_in_file(path, content)<0) { + slog(PRI_ERR, "failed to read [%s] (%s)", path, err()); + return 1; + } + + struct context context = { + line: 0 + }; + + struct vp_scanner *scanner = FB_SYM(init_scanner)(content.c_str()); + int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner)); + FB_SYM(cleanup_scanner)(scanner); + if (status) { + slog(PRI_ERR, "failed to parse [%s] (%s)", path, err()); + return -1; + } + + return 0; +} diff --git a/tools/python/jwutils/grammar.py b/tools/python/jwutils/grammar.py index 1218092..de10305 100644 --- a/tools/python/jwutils/grammar.py +++ b/tools/python/jwutils/grammar.py @@ -1096,10 +1096,6 @@ def grammar_create_y(grammar, opts): for f in opts['includes']: out += '#include "' + f + '"' + '\n' - #include "include/defs.h" - #include "include/vhdl2017.h" - #include "include/lex.vhdl2017.h" - #include "include/vhdl2017.tab.h" out += "\nusing namespace " + opts['namespace'] + ';\n' @@ -1261,30 +1257,28 @@ def grammar_create_l(grammar, opts): for f in opts['includes']: out += '#include "' + f + '"' + '\n' - #include "include/defs.h" - #include "include/vhdl2017.h" - - #// #include "include/lex.vhdl2017.h" - #include "include/vhdl2017.tab.h" out += "\nusing namespace " + opts['namespace'] + ';\n' + #out += textwrap.dedent("""\ + + # /* This is too late in the Flex generated file to work. Still lots of + # * prototypes are spat into it above it, which end up with C++ linkage, of + # * course, causing the linkages to be inconsistent to the functions below this + # * extern "C". Only way I found was to use C++ is to use it on Bison only, and + # * have Flex use C instead. */ + # #ifdef __cplusplus + # // extern "C" { + # #endif + + # #ifdef _REMOVE_ME + # static void get_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip); + # static void get_based_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip); + # #endif + + # %} + out += textwrap.dedent("""\ - - /* This is too late in the Flex generated file to work. Still lots of - * prototypes are spat into it above it, which end up with C++ linkage, of - * course, causing the linkages to be inconsistent to the functions below this - * extern "C". Only way I found was to use C++ is to use it on Bison only, and - * have Flex use C instead. */ - #ifdef __cplusplus - // extern "C" { - #endif - - #ifdef _REMOVE_ME - static void get_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip); - static void get_based_string(YYSTYPE *yylval_param, yyscan_t yyscanner, int skip); - #endif - %} %% @@ -1300,22 +1294,22 @@ def grammar_create_l(grammar, opts): assert p.term[-1] in [ '"', "'" ], p.term out += re.escape(p.term[1:-1]) + ' { return ' + p.sym + '; }\n' - out += textwrap.dedent("""\ - - %{/* basic_identifier */%} - %{/* extended_identifier */%} - %{/* based_integer */%} - %{/* bit_value */%} - %{/* numeric_literal */%} - %{/* enumeration_literal */%} - %{/* string_literal */%} - %{/* bit_string_literal */%} - %{/* character_literal */%} - %{/* graphic_character */%} - %{/* basic_character */%} - %{/* integer */%} - - """) + #out += textwrap.dedent("""\ + # + # %{/* basic_identifier */%} + # %{/* extended_identifier */%} + # %{/* based_integer */%} + # %{/* bit_value */%} + # %{/* numeric_literal */%} + # %{/* enumeration_literal */%} + # %{/* string_literal */%} + # %{/* bit_string_literal */%} + # %{/* character_literal */%} + # %{/* graphic_character */%} + # %{/* basic_character */%} + # %{/* integer */%} + # + # """) ignore += textwrap.dedent("""\ @@ -1378,8 +1372,7 @@ def grammar_create_l(grammar, opts): void FB_SYM(error)(struct context *context, void *scanner, const char *msg) { struct yyguts_t *yyg =(struct yyguts_t*)scanner; - // vp_log(context->vp, VP_LOG_ERR, "%s at \\"%s\\" in line %d.\\n\\n", msg, yytext, context->lineno); - slog(PRI_ERR, "%s at \\"%s\\" in line %d.\\n\\n", msg, yytext, context->line); + set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d", msg, yytext, context->line); } int FB_SYM(wrap)(void *scanner) @@ -1395,7 +1388,7 @@ def grammar_create_l(grammar, opts): /* utilities which need to be placed here, because I can't find * yylex_init() / _destroy() in any generated header file (??) */ - struct vp_scanner *vhdl_default_init_scanner(const char *str) + struct vp_scanner *FB_SYM(init_scanner)(const char *str) { struct vp_scanner *r = (struct vp_scanner *)calloc(1, sizeof(*r)); @@ -1408,12 +1401,12 @@ def grammar_create_l(grammar, opts): return r; } - void *vhdl_default_scanner_get_data(const struct vp_scanner *scanner) + void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner) { return scanner->scanner; } - void vhdl_default_cleanup_scanner(struct vp_scanner *scanner) + void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner) { free(scanner->str); yy_delete_buffer(scanner->buf, scanner->scanner); @@ -1421,12 +1414,14 @@ def grammar_create_l(grammar, opts): free(scanner); } - #ifdef __cplusplus - // } // extern "C" - #endif - """) + # #ifdef __cplusplus + # // } // extern "C" + # #endif + # + # """) + return out def grammar_create_h(grammar, opts): @@ -1497,7 +1492,18 @@ def grammar_create_h(grammar, opts): if ns is not None: out += '\n} /* namespace ' + ns + '*/' - out += '\n#endif /* #ifndef + ' + opts['mip'] + ' */' + + out += textwrap.dedent("""\ + + struct vp_scanner; + + struct vp_scanner *FB_SYM(init_scanner)(const char *str); + void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner); + void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner); + + """) + + out += '\n\n#endif /* #ifndef + ' + opts['mip'] + ' */' return out