janware/jw-python
1
mirror of ssh://git.janware.com/srv/git/janware/proj/jw-python synced 2026-01-15 01:52:56 +01:00
Code Releases Activity

grammar.py and friends: Improve line numbers and headers

Browse source 
Line numbers of parsed input are now automatically maintained. More
needed data structures are now automatically created in the API header
file.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2017-11-03 12:54:19 +01:00
commit 3cba245b77
5 changed files with 76 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

2
make/generate-flex-bison.mk
View file

@ -27,7 +27,7 @@ FB_CASE_INSENSITIVE ?= true
FB_SRC ?= $(filter %.y %.l,$(GENERATED))
FB_COMMON_H ?= $(FB_HDRDIR)/$(FB_NAME).h
INCLUDED_BY_GENERATED += include/defs.h $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
INCLUDED_BY_GENERATED += $(FB_COMMON_H) include/lex.$(FB_NAME).h include/$(FB_NAME).tab.h
GENERATE_PY ?= ./generate.py
GENERATE ?= python ./$(GENERATE_PY) --log-level $(GENERATE_LOG_LEVEL) create \

1
test/grammar/generate.conf
View file

@ -2,6 +2,7 @@
[white_space[
type = token
lex_extra_action = "if memchr(yytext, '\n', yyleng) context->line++;"
regex = "[ \n\t\r]+"
]

33
test/grammar/include/defs.h
View file

@ -1,33 +0,0 @@
#ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
#define _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H
#define YY_NO_INPUT
#define YY_NO_UNPUT
// #define YY_NO_UNISTD_H
struct context {
int line;
int column;
};
union YYSTYPE;
#ifdef __cplusplus
extern "C" {
#endif
/* defined in grammartest-parser.l */
struct vp_scanner;
struct vp_scanner *grammartest_default_init_scanner(const char *str);
void *grammartest_default_scanner_get_data(const struct vp_scanner *scanner);
void grammartest_default_cleanup_scanner(struct vp_scanner *scanner);
void FB_SYM(error)(struct context *context, void *scanner, const char *s);
#ifdef __cplusplus
} // extern "C"
#endif
#define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner)
#endif /* #ifndef _JW_PYTHON_GRAMMARTEST_PARSER_DEFS_H */

26
test/grammar/main.cpp
View file

@ -1,19 +1,10 @@
#include <string>
#include <stdio.h>
#include <misc.h>
#include <slog.h>
#include <YMisc.h>
// TODO: this should be included from grammartest.tab.h automatically
#include "include/grammartest.h"
#include "include/defs.h"
#include "include/grammartest.tab.h"
extern int FB_SYM(debug);
using namespace std;
int main(int argc, const char *argv[])
@ -30,20 +21,9 @@ int main(int argc, const char *argv[])
return 1;
}
// TODO: Initialize this in a generated function
struct context context = {
line: 1,
column: 0
};
FB_SYM(debug) = 1;
struct vp_scanner *scanner = FB_SYM(init_scanner)(content.c_str());
int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner));
FB_SYM(cleanup_scanner)(scanner);
if (status) {
slog(PRI_ERR, "failed to parse [%s] (%s)", path, err());
return -1;
if (FB_SYM(create_ast)(content.c_str())<0) {
slog(PRI_ERR, "failed to create AST from [%s] (%s)", path, err());
return 1;
}
return 0;

85
tools/python/jwutils/grammar.py
View file

@ -1295,6 +1295,17 @@ def grammar_create_l(grammar, opts):
%{
#include <slog.h>
#define YY_USER_ACTION \\
context->first_line = context->last_line; \\
context->first_column = context->last_column; \\
for(int i = 0; yytext[i] != '\\0'; i++) { \\
if(yytext[i] == '\\n') { \\
context->last_line++; \\
context->last_column = 0; \\
} else { \\
context->last_column++; \\
} \\
}
""")
for f in opts['includes']:
@ -1325,9 +1336,6 @@ def grammar_create_l(grammar, opts):
%%
\\n { context->line++; context->column = 0; REJECT; }
. { context->column++; REJECT; }
""")
for t, p in grammar.iteritems():
@ -1426,7 +1434,7 @@ def grammar_create_l(grammar, opts):
void FB_SYM(error)(struct context *context, void *scanner, const char *msg)
{
struct yyguts_t *yyg =(struct yyguts_t*)scanner;
set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->line, context->column);
set_error(PRI_ERR, EINVAL, "%s at \\"%s\\" in line %d:%d", msg, yytext, context->last_line, context->last_column);
}
int FB_SYM(wrap)(void *scanner)
@ -1468,6 +1476,29 @@ def grammar_create_l(grammar, opts):
free(scanner);
}
int FB_SYM(create_ast)(const char *str)
{
// TODO: Initialize this in a generated function
struct context context = {
first_line: 1,
last_line: 1,
first_column: 0,
last_column: 0
};
FB_SYM(debug) = 1;
struct vp_scanner *scanner = FB_SYM(init_scanner)(str);
int status = FB_SYM(parse)(&context, FB_SYM(scanner_get_data)(scanner));
FB_SYM(cleanup_scanner)(scanner);
if (status) {
slog(PRI_ERR, "failed to parse (%s)", err());
return -1;
}
return 0;
}
""")
# #ifdef __cplusplus
@ -1483,6 +1514,42 @@ def grammar_create_h(grammar, opts):
ns = opts['namespace']
tokens = symbols_from_config(opts['config'], "all")
out += textwrap.dedent("""\
#define YY_NO_INPUT
#define YY_NO_UNPUT
// #define YY_NO_UNISTD_H
struct context {
int first_line;
int last_line;
int first_column;
int last_column;
};
union YYSTYPE;
#ifdef __cplusplus
extern "C" {
#endif
struct vp_scanner;
struct vp_scanner *FB_SYM(init_scanner)(const char *str);
void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner);
void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner);
int FB_SYM(create_ast)(const char *str);
void FB_SYM(error)(struct context *context, void *scanner, const char *s);
#ifdef __cplusplus
} // extern "C"
#endif
#define YY_DECL int FB_SYM(lex)(YYSTYPE *yylval_param, struct context *context, void *yyscanner)
""")
if ns is not None:
out += 'namespace ' + ns + '{\n\n'
@ -1556,16 +1623,6 @@ def grammar_create_h(grammar, opts):
if ns is not None:
out += '\n} /* namespace ' + ns + '*/'
out += textwrap.dedent("""\
struct vp_scanner;
struct vp_scanner *FB_SYM(init_scanner)(const char *str);
void *FB_SYM(scanner_get_data)(const struct vp_scanner *scanner);
void FB_SYM(cleanup_scanner)(struct vp_scanner *scanner);
""")
out += '\n\n#endif /* #ifndef + ' + opts['mip'] + ' */'
return out