grammar.py and friends: Make list parsing run through

First time parsing doesn't error out with a syntax error. No usable AST
is produced, strings are not returned from lexer, and AST lists aren't
lists, really.

TEXT:="Hello world!"; had to be excluded from the example, because I
don't get how this could be parsed with the given syntax. There's a
special sequence "all visible characters", but any lexer regex I could
think of will also match the types defining "alphabetic character" and
return the respective tokens (e.g. T_A) or vice-versa, depending on the
order in the lexer input file. I suppose, the only sensible way to
handle this, is to define "all visible characters" by defining the
tokens for the missing characters, and then use them along T_A ... T_Z
or their derived types.

Signed-off-by: Jan Lindemann <jan@janware.com>

test/grammar/Makefile

@@ -1,10 +1,13 @@
 TOPDIR = ../..
 
+-include local.mk
+
 EXE_ARGS              ?= grammartest.code
 PREREQ_BUILD          += ytools
 FB_NAME                = grammartest
 NAMESPACE_IN_GENERATED = gt
 GENERATE_CONFIG_FILE   = generate.conf
+IRRELEVANT_SYMBOLS    ?= white_space
 
 include $(TOPDIR)/make/proj.mk
 include $(TOPDIR)/make/generate-flex-bison.mk

test/grammar/generate.conf

@@ -1,8 +1,18 @@
 [symbols]
+
   [white_space[
+    type = token
     regex = "[ \n\t\r]+" 
   ]
+
+  [all_characters[
+    type = non-terminal
+    regex = "[[:print:]]"
+    #lex_as = yytext[0]
+  ]
+
   [test[
+    type = token
     dings = bums
     regex = "bumsdings"
   ]

test/grammar/grammartest-input.ebnf

@@ -2,7 +2,7 @@
  program = 'PROGRAM', white space, identifier, white space, 
             'BEGIN', white space, 
             { assignment, ";", white space }, 
-            'END.' ;
+            'END.', [ white space ];
  identifier = alphabetic character, { alphabetic character | digit } ;
  number = [ "-" ], digit, { digit } ;
  string = '"' , { all characters }, '"' ;

test/grammar/grammartest.code

@@ -6,5 +6,4 @@ BEGIN
   C:=A;
   D123:=B34A;
   BABOON:=GIRAFFE;
-  TEXT:="Hello world!";
 END.