From 3cba245b775ccf50708bc63c99c19d4636d59991 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 3 Nov 2017 12:54:19 +0100 Subject: [PATCH] grammar.py and friends: Improve line numbers and headers Line numbers of parsed input are now automatically maintained. More needed data structures are now automatically created in the API header file. Signed-off-by: Jan Lindemann --- make/generate-flex-bison.mk | 2 +- test/grammar/generate.conf | 1 + test/grammar/include/defs.h | 33 ------------- test/grammar/main.cpp | 26 ++-------- tools/python/jwutils/grammar.py | 85 +++++++++++++++++++++++++++------ 5 files changed, 76 insertions(+), 71 deletions(-) delete mode 100644 test/grammar/include/defs.h diff --git a/make/generate-flex-bison.mk b/make/generate-flex-bison.mk index 1d34848..246b0a9 100644 --- a/make/generate-flex-bison.mk +++ b/make/generate-flex-bison.mk @@ -27,7 +27,7 @@ FB_CASE_INSENSITIVE ?= true FB_SRC ?= $(filter %.y %.l,$(GENERATED)) FB_COMMON_H ?= $(FB_HDRDIR)/$(FB_NAME).h -INCLUDED_BY_GENERATED += include/defs.h $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h +INCLUDED_BY_GENERATED += $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h GENERATE_PY ?= ./generate.py GENERATE ?= python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \ diff --git a/test/grammar/generate.conf b/test/grammar/generate.conf index 1f63307..8bc1e28 100644 --- a/test/grammar/generate.conf +++ b/test/grammar/generate.conf @@ -2,6 +2,7 @@ [white_space[ type = token + lex_extra_action = "if memchr(yytext, '\n', yyleng) context->line++;" regex = "[ \n\t\r]+" ] diff --git a/test/grammar/include/defs.h b/test/grammar/include/defs.h deleted file mode 100644 index 7ce3ba8..0000000 --- a/test/grammar/include/defs.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H -#define _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H - -#define YY_NO_INPUT -#define YY_NO_UNPUT -// #define YY_NO_UNISTD_H - -struct context { - int line; - int column; -}; - -union YYSTYPE; - -#ifdef __cplusplus -extern "C" { -#endif - -/* defined in grammartest-parser.l */ -struct vp_scanner; -struct vp_scanner *grammartest_default_init_scanner(const char *str); -void *grammartest_default_scanner_get_data(const struct vp_scanner *scanner); -void grammartest_default_cleanup_scanner(struct vp_scanner *scanner); - -void FB_SYM(error)(struct context *context, void *scanner, const char *s); - -#ifdef __cplusplus -} // extern "C" -#endif - -#define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner) - -#endif /* #ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H */ diff --git a/test/grammar/main.cpp b/test/grammar/main.cpp index e25b430..719f520 100644 --- a/test/grammar/main.cpp +++ b/test/grammar/main.cpp @@ -1,19 +1,10 @@ #include -#include - -#include #include #include -// TODO: this should be included from grammartest.tab.h automatically #include "include/grammartest.h" -#include "include/defs.h" -#include "include/grammartest.tab.h" - -extern int FB_SYM(debug); - using namespace std; int main(int argc, const char *argv[]) @@ -30,20 +21,9 @@ int main(int argc, const char *argv[]) return 1; } - // TODO: Initialize this in a generated function - struct context context = { - line: 1, - column: 0 - }; - - FB_SYM(debug) = 1; - - struct vp_scanner *scanner = FB_SYM(init_scanner)(content.c_str()); - int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner)); - FB_SYM(cleanup_scanner)(scanner); - if (status) { - slog(PRI_ERR, "failed to parse [%s] (%s)", path, err()); - return -1; + if (FB_SYM(create_ast)(content.c_str())<0) { + slog(PRI_ERR, "failed to create AST from [%s] (%s)", path, err()); + return 1; } return 0; diff --git a/tools/python/jwutils/grammar.py b/tools/python/jwutils/grammar.py index 39f6cf8..b151bb3 100644 --- a/tools/python/jwutils/grammar.py +++ b/tools/python/jwutils/grammar.py @@ -1295,6 +1295,17 @@ def grammar_create_l(grammar, opts): %{ #include + #define YY_USER_ACTION \\ + context->first_line = context->last_line; \\ + context->first_column = context->last_column; \\ + for(int i = 0; yytext[i] != '\\0'; i++) { \\ + if(yytext[i] == '\\n') { \\ + context->last_line++; \\ + context->last_column = 0; \\ + } else { \\ + context->last_column++; \\ + } \\ + } """) for f in opts['includes']: @@ -1325,9 +1336,6 @@ def grammar_create_l(grammar, opts): %% - \\n { context->line++; context->column = 0; REJECT; } - . { context->column++; REJECT; } - """) for t, p in grammar.iteritems(): @@ -1426,7 +1434,7 @@ def grammar_create_l(grammar, opts): void FB_SYM(error)(struct context *context, void *scanner, const char *msg) { struct yyguts_t *yyg =(struct yyguts_t*)scanner; - set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->line, context->column); + set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->last_line, context->last_column); } int FB_SYM(wrap)(void *scanner) @@ -1468,6 +1476,29 @@ def grammar_create_l(grammar, opts): free(scanner); } + int FB_SYM(create_ast)(const char *str) + { + // TODO: Initialize this in a generated function + struct context context = { + first_line: 1, + last_line: 1, + first_column: 0, + last_column: 0 + }; + + FB_SYM(debug) = 1; + + struct vp_scanner *scanner = FB_SYM(init_scanner)(str); + int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner)); + FB_SYM(cleanup_scanner)(scanner); + if (status) { + slog(PRI_ERR, "failed to parse (%s)", err()); + return -1; + } + + return 0; + } + """) # #ifdef __cplusplus @@ -1483,6 +1514,42 @@ def grammar_create_h(grammar, opts): ns = opts['namespace'] tokens = symbols_from_config(opts['config'], "all") + out += textwrap.dedent("""\ + + #define YY_NO_INPUT + #define YY_NO_UNPUT + // #define YY_NO_UNISTD_H + + struct context { + int first_line; + int last_line; + int first_column; + int last_column; + }; + + union YYSTYPE; + + #ifdef __cplusplus + extern "C" { + #endif + + struct vp_scanner; + + struct vp_scanner *FB_SYM(init_scanner)(const char *str); + void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner); + void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner); + int FB_SYM(create_ast)(const char *str); + + void FB_SYM(error)(struct context *context, void *scanner, const char *s); + + #ifdef __cplusplus + } // extern "C" + #endif + + #define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner) + + """) + if ns is not None: out += 'namespace ' + ns + '{\n\n' @@ -1556,16 +1623,6 @@ def grammar_create_h(grammar, opts): if ns is not None: out += '\n} /* namespace ' + ns + '*/' - out += textwrap.dedent("""\ - - struct vp_scanner; - - struct vp_scanner *FB_SYM(init_scanner)(const char *str); - void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner); - void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner); - - """) - out += '\n\n#endif /* #ifndef + ' + opts['mip'] + ' */' return out