grammar.py and friends: Improve line numbers and headers

Line numbers of parsed input are now automatically maintained. More
needed data structures are now automatically created in the API header
file.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/generate-flex-bison.mk

@@ -27,7 +27,7 @@ FB_CASE_INSENSITIVE  ?= true
 FB_SRC               ?= $(filter %.y %.l,$(GENERATED))
 FB_COMMON_H          ?= $(FB_HDRDIR)/$(FB_NAME).h
 
-INCLUDED_BY_GENERATED  += include/defs.h $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
+INCLUDED_BY_GENERATED  += $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
 
 GENERATE_PY          ?= ./generate.py
 GENERATE             ?= python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \

test/grammar/generate.conf

@@ -2,6 +2,7 @@
 
   [white_space[
     type = token
+    lex_extra_action = "if memchr(yytext, '\n', yyleng) context->line++;"
     regex = "[ \n\t\r]+" 
   ]
 

test/grammar/include/defs.h

@@ -1,33 +0,0 @@
-#ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
-#define _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
-
-#define YY_NO_INPUT
-#define YY_NO_UNPUT
-// #define YY_NO_UNISTD_H
-
-struct context {
-	int line;
-	int column;
-};
-
-union YYSTYPE;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* defined in grammartest-parser.l */
-struct vp_scanner;
-struct vp_scanner *grammartest_default_init_scanner(const char *str);
-void *grammartest_default_scanner_get_data(const struct vp_scanner *scanner);
-void grammartest_default_cleanup_scanner(struct vp_scanner *scanner);
-
-void FB_SYM(error)(struct context *context, void *scanner, const char *s);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif
-
-#define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner)
-
-#endif /* #ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H */

test/grammar/main.cpp

@@ -1,19 +1,10 @@
 #include <string>
 
-#include <stdio.h>
-
-#include <misc.h>
 #include <slog.h>
 #include <YMisc.h>
 
-// TODO: this should be included from grammartest.tab.h automatically
 #include "include/grammartest.h"
 
-#include "include/defs.h"
-#include "include/grammartest.tab.h"
-
-extern int FB_SYM(debug);
-
 using namespace std;
 
 int main(int argc, const char *argv[])
@@ -30,20 +21,9 @@ int main(int argc, const char *argv[])
 		return 1;
 	}
 
-	// TODO: Initialize this in a generated function
+	if (FB_SYM(create_ast)(content.c_str())<0) {
-	struct context context = {
+		slog(PRI_ERR, "failed to create AST from [%s] (%s)", path, err());
-		line: 1,
+		return 1;
-		column: 0
-	};
-
-	FB_SYM(debug) = 1;
-
-	struct vp_scanner *scanner = FB_SYM(init_scanner)(content.c_str());
-	int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner));
-	FB_SYM(cleanup_scanner)(scanner);
-	if (status) {
-		slog(PRI_ERR, "failed to parse [%s] (%s)", path, err());
-		return -1;
 	}
 
 	return 0;

tools/python/jwutils/grammar.py

@@ -1295,6 +1295,17 @@ def grammar_create_l(grammar, opts):
         %{
         #include <slog.h>
 
+        #define YY_USER_ACTION \\
+            context->first_line = context->last_line; \\
+            context->first_column = context->last_column; \\
+            for(int i = 0; yytext[i] != '\\0'; i++) { \\
+                if(yytext[i] == '\\n') { \\
+                    context->last_line++; \\
+                    context->last_column = 0; \\
+                } else { \\
+                    context->last_column++; \\
+                } \\
+            }
     """)
 
     for f in opts['includes']:
@@ -1325,9 +1336,6 @@ def grammar_create_l(grammar, opts):
 
         %%
 
-        \\n { context->line++; context->column = 0; REJECT; }
-        . { context->column++; REJECT; }
-
         """)
 
     for t, p in grammar.iteritems():
@@ -1426,7 +1434,7 @@ def grammar_create_l(grammar, opts):
         void FB_SYM(error)(struct context *context, void *scanner, const char *msg)
         {
         	struct yyguts_t *yyg =(struct yyguts_t*)scanner;
-                set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->line, context->column);
+                set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->last_line, context->last_column);
         }
 
         int FB_SYM(wrap)(void *scanner)
@@ -1468,6 +1476,29 @@ def grammar_create_l(grammar, opts):
         	free(scanner);
         }
 
+        int FB_SYM(create_ast)(const char *str)
+        {
+        	// TODO: Initialize this in a generated function
+        	struct context context = {
+        		first_line: 1,
+        		last_line: 1,
+        		first_column: 0,
+        		last_column: 0
+        	};
+        
+        	FB_SYM(debug) = 1;
+        
+        	struct vp_scanner *scanner = FB_SYM(init_scanner)(str);
+        	int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner));
+        	FB_SYM(cleanup_scanner)(scanner);
+        	if (status) {
+        		slog(PRI_ERR, "failed to parse (%s)", err());
+        		return -1;
+        	}
+
+        	return 0;
+        }
+
 	""")
 
     #    #ifdef __cplusplus
@@ -1483,6 +1514,42 @@ def grammar_create_h(grammar, opts):
     ns = opts['namespace']
     tokens = symbols_from_config(opts['config'], "all")
 
+    out += textwrap.dedent("""\
+    
+        #define YY_NO_INPUT
+        #define YY_NO_UNPUT
+        // #define YY_NO_UNISTD_H
+
+        struct context {
+        	int first_line;
+        	int last_line;
+        	int first_column;
+        	int last_column;
+        };
+
+        union YYSTYPE;
+
+        #ifdef __cplusplus
+        extern "C" {
+        #endif
+
+        struct vp_scanner;
+
+        struct vp_scanner *FB_SYM(init_scanner)(const char *str);
+        void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner);
+        void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner);
+        int FB_SYM(create_ast)(const char *str);
+
+        void FB_SYM(error)(struct context *context, void *scanner, const char *s);
+
+        #ifdef __cplusplus
+        } // extern "C"
+        #endif
+
+        #define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner)
+
+	""")
+
     if ns is not None:
         out += 'namespace ' + ns + '{\n\n'
 
@@ -1556,16 +1623,6 @@ def grammar_create_h(grammar, opts):
     if ns is not None:
         out += '\n} /* namespace ' + ns + '*/'
 
-    out += textwrap.dedent("""\
-
-        struct vp_scanner;
-
-        struct vp_scanner *FB_SYM(init_scanner)(const char *str);
-        void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner);
-        void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner);
-
-	""")
-
     out += '\n\n#endif /* #ifndef + ' + opts['mip'] + ' */'
 
     return out