janware/jw-python
1
mirror of ssh://devgit.janware.com/janware/proj/jw-python synced 2026-06-17 07:06:37 +02:00
Code Releases Activity Actions

Makefile: Include py-topdir.mk

Browse source 
Include py-topdir.mk, which entails loads of fallout from make check. Fix it.
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-06-10 13:28:35 +02:00
commit e9845b5a1f
Signed by: Jan Lindemann
GPG key ID: 3750640C9E25DD61
45 changed files with 1796 additions and 1191 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -2,3 +2,4 @@ TOPDIR = .
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/topdir.mk
include $(JWBDIR)/make/py-topdir.mk

25
src/python/jw/util/ArgsContainer.py
View file

@ -1,20 +1,17 @@
# -*- coding: utf-8 -*-
import argparse
from collections import OrderedDict
from .log import slog
from .log import get_caller_pos, slog
class ArgsContainer: # export
class ArgsContainer: # export
__args: OrderedDict[str, str] = OrderedDict()
__kwargs: OrderedDict[str, str] = OrderedDict()
__kwargs: OrderedDict[str, dict[str, str]] = OrderedDict()
__values: dict[str, str] = {}
__specified_args: list[str] = list()
def __getattr__(self, name):
values = self.__values
def __getattr__(self, name: str) -> str:
if name in self.__values:
return self.__values[name]
if name in self.__kwargs.keys():
@ -25,7 +22,7 @@ class ArgsContainer: # export
raise Exception(f'No argument "{name}" defined')
def __setattr__(self, name, value):
if not name in self.__kwargs.keys():
if name not in self.__kwargs.keys():
raise Exception(f'No argument "{name}" defined')
self.__values[name] = value
self.__specified_args.append(name)
@ -41,7 +38,7 @@ class ArgsContainer: # export
else:
raise Exception('Missing argument name')
name = name.replace('-', '_')
self.__args[name] = args
self.__args[name] = arg
self.__kwargs[name] = kwargs
def keys(self):
@ -50,7 +47,7 @@ class ArgsContainer: # export
def args(self, name) -> str:
return self.__args[name]
def kwargs(self, name) -> str:
def kwargs(self, name) -> dict[str, str]:
return self.__kwargs[name]
def dump(self, prio, *args, **kwargs):
@ -59,15 +56,17 @@ class ArgsContainer: # export
val = None
try:
val = self.__getattr__(name)
except:
except Exception:
pass
slog(prio, f'{name}: {val}', caller=caller)
slog(prio, f'{name}: {val}', caller = caller)
@property
def specified_args(self):
return self.__specified_args
def add_argument(p: argparse.ArgumentParser|ArgsContainer, name: str, *args, **kwargs): # export
def add_argument( # export
p: argparse.ArgumentParser | ArgsContainer, name: str, *args, **kwargs
):
key = name.strip('--').replace('-', '_')
if isinstance(p, ArgsContainer):

4
src/python/jw/util/Bunch.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
from typing import Any
class Bunch: # export
class Bunch: # export
def __init__(self, **kwargs):
self.__dict__.update(kwargs)

34
src/python/jw/util/Cmd.py
View file

@ -1,39 +1,49 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import inspect, sys, re, abc, argparse
from argparse import ArgumentParser, _SubParsersAction
import abc
import argparse
import inspect
import re
import sys
from argparse import ArgumentParser
from typing import TYPE_CHECKING
from . import log
if TYPE_CHECKING:
from .Cmds import Cmds
# full blown example of one level of nested subcommands
# git -C project remote -v show -n myremote
class Cmd(abc.ABC): # export
class Cmd(abc.ABC): # export
@abc.abstractmethod
async def run(self, args):
pass
def __init__(self, name: str, help: str) -> None:
from . import Cmds
self.name = name
self.help = help
self.parent = None
self.children: list[Cmd] = []
self.child_classes: list[type[Cmd]] = []
self.app: Cmds|None = None
self.app: Cmds | None = None
async def _run(self, args):
pass
def add_parser(self, parsers) -> ArgumentParser:
r = parsers.add_parser(self.name, help=self.help,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
r.set_defaults(func=self.run)
r = parsers.add_parser(
self.name,
help = self.help,
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)
r.set_defaults(func = self.run)
return r
def add_subcommands(self, cmd: str|type[Cmd]|list[type[Cmd]]) -> None:
def add_subcommands(self, cmd: str | type[Cmd] | list[type[Cmd]]) -> None:
if isinstance(cmd, str):
sc = []
for name, obj in inspect.getmembers(sys.modules[self.__class__.__module__]):
@ -54,7 +64,7 @@ class Cmd(abc.ABC): # export
def add_arguments(self, parser: ArgumentParser) -> None:
pass
def conf_value(self, path, default=None):
def conf_value(self, path, default = None):
ret = None if self.app is None else self.app.conf_value(path, default)
if ret is None and default is not None:
return default

99
src/python/jw/util/Cmds.py
View file

@ -1,13 +1,21 @@
# -*- coding: utf-8 -*-
import argparse
import asyncio
import cProfile
import importlib
import inspect
import os
import re
import sys
import os, sys, argcomplete, argparse, importlib, inspect, re, pickle, asyncio, cProfile
from argparse import ArgumentParser
from pathlib import Path, PurePath
from .log import *
import argcomplete
from .log import DEBUG, ERR, NOTICE, add_log_file, set_flags, set_level, slog
from .stree import serdes
class Cmds: # export
class Cmds: # export
def __instantiate(self, cls):
try:
@ -15,7 +23,7 @@ class Cmds: # export
except Exception as e:
slog(ERR, f'Failed to instantiate command of type {cls}: {e}')
raise
r.cmds = self # TODO: Rename Cmds class to App, "Cmds" isn't very self-explanatory
r.cmds = self # TODO: Rename Cmds class to App, "Cmds" isn't self-explanatory
r.app = self
return r
@ -26,7 +34,9 @@ class Cmds: # export
for c in cmd.child_classes:
cmd.children.append(self.__instantiate(c))
if len(cmd.children) > 0:
subparsers = parser.add_subparsers(title='Available subcommands of ' + cmd.name, metavar='')
subparsers = parser.add_subparsers(
title = 'Available subcommands of ' + cmd.name, metavar = ''
)
for sub_cmd in cmd.children:
self.__add_cmd_to_parser(sub_cmd, subparsers)
@ -38,7 +48,13 @@ class Cmds: # export
slog(DEBUG, 'Reading configuration "{}"'.format(path))
return serdes.read(path, ''), [path]
def __init__(self, description: str = '', filter: str = '^Cmd.*', modules: None=None, eloop: None=None) -> None:
def __init__(
self,
description: str = '',
filter: str = '^Cmd.*',
modules: None = None,
eloop: None = None
) -> None:
self.__description = description
self.__filter = filter
self.__modules = modules
@ -68,18 +84,34 @@ class Cmds: # export
set_flags(log_flags)
set_level(log_level)
slog(DEBUG, "set log level to {}".format(log_level))
self.__parser = argparse.ArgumentParser(usage=os.path.basename(sys.argv[0]) + ' [options]',
formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=self.__description)
self.__parser.add_argument('--log-flags', help='Log flags', default=log_flags)
self.__parser.add_argument('--log-level', help='Log level', default=log_level)
self.__parser.add_argument('--backtrace', help='Show exception backtraces', action='store_true', default=False)
self.__parser.add_argument('--write-profile', help='Profile code and store output to file', default=None)
self.__parser.add_argument('--log-file', help='Log file', default=log_file)
if self.__modules == None:
self.__modules = [ '__main__' ]
self.__parser = argparse.ArgumentParser(
usage = os.path.basename(sys.argv[0]) + ' [options]',
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
description = self.__description
)
self.__parser.add_argument(
'--log-flags', help = 'Log flags', default = log_flags
)
self.__parser.add_argument(
'--log-level', help = 'Log level', default = log_level
)
self.__parser.add_argument(
'--backtrace',
help = 'Show exception backtraces',
action = 'store_true',
default = False
)
self.__parser.add_argument(
'--write-profile',
help = 'Profile code and store output to file',
default = None
)
self.__parser.add_argument('--log-file', help = 'Log file', default = log_file)
if self.__modules is None:
self.__modules = ['__main__']
subcmds = set()
slog(DEBUG, '-- searching for commands')
for m in self.__modules: # type: ignore
for m in self.__modules: # type: ignore
if m != '__main__':
importlib.import_module(m)
for name, c in inspect.getmembers(sys.modules[m], inspect.isclass):
@ -96,24 +128,27 @@ class Cmds: # export
subcmds.update(cmd.child_classes)
cmds = [cmd for cmd in self.__cmds if type(cmd) not in subcmds]
subparsers = self.__parser.add_subparsers(title='Available commands', metavar='')
subparsers = self.__parser.add_subparsers(
title = 'Available commands', metavar = ''
)
for cmd in cmds:
slog(DEBUG, f'Adding top-level command {cmd} to parser')
self.__add_cmd_to_parser(cmd, subparsers)
# Run all sub-commands. Overwrite if you want to do anything before or after
async def _run(self, argv=None):
async def _run(self, argv = None):
return await self.args.func(self.args)
async def __run(self, argv=None):
async def __run(self, argv = None):
argcomplete.autocomplete(self.__parser)
self.args = self.__parser.parse_args(args=argv)
self.args = self.__parser.parse_args(args = argv)
set_flags(self.args.log_flags)
set_level(self.args.log_level)
self.__back_trace = self.args.backtrace
exit_status = 0
# This is the toplevel parser, i.e. no func member has been added to the args via
# This is the toplevel parser, i.e. no func member has been added to the args
# yet, via
#
# Cmds.__init__()
# Cmds.__add_cmd_to_parser(cmd, subparsers)
@ -135,17 +170,15 @@ class Cmds: # export
add_log_file(self.args.log_file)
try:
ret = await self._run(self.args)
await self._run(self.args)
except Exception as e:
if hasattr(e, 'message'):
slog(ERR, e.message)
else:
slog(ERR, f'Exception: {type(e)}: {e}')
slog(ERR, f'Exception: {type(e)}: {str(e)}')
exit_status = 1
if self.__back_trace:
raise
finally:
if pr is not None:
assert self.args.write_profile is not None, 'args.write_profile'
pr.disable()
slog(NOTICE, f'Writing profile statistics to {self.args.write_profile}')
pr.dump_stats(self.args.write_profile)
@ -160,7 +193,7 @@ class Cmds: # export
self.eloop = None
self.__own_eloop = False
def conf_value(self, path, default=None):
def conf_value(self, path, default = None):
ret = None if self.__conf is None else self.__conf.value(path)
if ret is None and default is not None:
return default
@ -169,10 +202,12 @@ class Cmds: # export
def parser(self) -> ArgumentParser:
return self.__parser
def run(self, argv=None) -> None:
def run(self, argv = None) -> None:
#return self.__run()
return self.eloop.run_until_complete(self.__run(argv)) # type: ignore
return self.eloop.run_until_complete(self.__run(argv)) # type: ignore
def run_sub_commands(description = '', filter = '^Cmd.*', modules=None, argv=None): # export
def run_sub_commands( # export
description = '', filter = '^Cmd.*', modules = None, argv = None
):
cmds = Cmds(description, filter, modules)
return cmds.run(argv=argv)
return cmds.run(argv = argv)

89
src/python/jw/util/Config.py
View file

@ -1,14 +1,16 @@
# -*- coding: utf-8 -*-
import glob
import os
import re
import sys
from typing import Optional, Dict, cast
import os, re, glob, sys
from pathlib import Path, PosixPath
from pathlib import Path
from typing import Dict, Optional, cast
from . import stree
from .stree import serdes
from .log import DEBUG, ERR, slog, get_caller_pos
from .stree.StringTree import StringTree
from .log import *
class Config(): # export
class Config(): # export
def __load(self, search_dirs, glob_paths, refuse_mode_mask):
@ -33,15 +35,15 @@ class Config(): # export
for path in glob_paths:
dirs = search_dirs
if dirs is None:
dirs = [''] if __is_abs(path) else [ str(Path.home()), str(Path.cwd()) ]
dirs = [''] if __is_abs(path) else [str(Path.home()), str(Path.cwd())]
for d in dirs:
g = d + '/' + path if len(d) else path
slog(DEBUG, 'Looking for config "{}"'.format(g))
for f in glob.glob(g):
slog(DEBUG, 'Reading config "{}"'.format(f))
paths_buf = []
tree = stree.read(f, paths_buf=paths_buf)
assert(len(paths_buf))
tree = serdes.read(f, paths_buf = paths_buf)
assert (len(paths_buf))
if refuse_mode_mask is not None:
for p in paths_buf:
st = os.stat(p)
@ -49,37 +51,44 @@ class Config(): # export
for item in tree.child_list():
if item.content is None:
continue
if not re.search('password|secret', cast(str, item.content), flags=re.IGNORECASE):
if not re.search('password|secret',
cast('str', item.content),
flags = re.IGNORECASE):
continue
msg = "Config files define secret, but at least one has file permissions open for world"
msg = (
'Config files define secret, but at least one '
'has file permissions open for world'
)
slog(ERR, f'{msg}:')
for pp in paths_buf:
slog(ERR, f' {((os.stat(pp).st_mode) & 0o7777):o} {pp}')
mode = (os.stat(pp).st_mode) & 0o7777
slog(ERR, f' {mode:o} {pp}')
raise Exception(msg)
tree.dump(DEBUG, f)
ret.add("", tree)
return ret
def __init__(self,
search_dirs: Optional[list[str]]=None,
glob_paths: Optional[list[str]]=None,
glob_paths_env_key: Optional[str]=None,
defaults: Optional[Dict[str, str]]=None,
tree: Optional[StringTree]=None,
parent=None,
root_section=None,
refuse_mode_mask=0o0027
) -> None:
def __init__(
self,
search_dirs: Optional[list[str]] = None,
glob_paths: Optional[list[str]] = None,
glob_paths_env_key: Optional[str] = None,
defaults: Optional[Dict[str, str]] = None,
tree: Optional[StringTree] = None,
parent = None,
root_section = None,
refuse_mode_mask = 0o0027
) -> None:
self.__parent = parent
if tree is not None:
assert(search_dirs is None)
assert(glob_paths is None)
assert(glob_paths_env_key is None)
assert (search_dirs is None)
assert (glob_paths is None)
assert (glob_paths_env_key is None)
self.__conf = tree
else:
assert(tree is None)
assert (tree is None)
if glob_paths_env_key is not None:
glob_paths_env = os.getenv(glob_paths_env_key)
if glob_paths_env is not None:
@ -87,8 +96,11 @@ class Config(): # export
glob_paths = []
glob_paths.extend(glob_paths_env.split(':'))
self.__conf = self.__load(search_dirs=search_dirs, glob_paths=glob_paths,
refuse_mode_mask=refuse_mode_mask)
self.__conf = self.__load(
search_dirs = search_dirs,
glob_paths = glob_paths,
refuse_mode_mask = refuse_mode_mask
)
if root_section is not None:
tmp = self.__conf.get(root_section)
@ -141,7 +153,11 @@ class Config(): # export
def value(self, key: str, default = None) -> Optional[str]:
return self.get(key, default)
def branch(self, path: str, throw: bool=True): # type: ignore # Optional[Config]: FIXME: Don't know how to get hold of this type here
def branch(
self,
path: str,
throw: bool = True
): # type: ignore # Optional[Config]: FIXME: Don't know how to get hold of this type here
if self.__conf:
tree = self.__conf.get(path)
if tree is None:
@ -151,19 +167,24 @@ class Config(): # export
return None
self.dump(ERR, msg)
raise Exception(msg)
return Config(tree=tree, parent=self) # type: ignore
return Config(tree = tree, parent = self) # type: ignore
return None
def dump(self, prio: int, *args, **kwargs) -> None:
caller = get_caller_pos(1, kwargs)
self.__conf.dump(prio, caller=caller, *args, **kwargs)
self.__conf.dump(prio, caller = caller, *args, **kwargs)
@property
def name(self):
return self.__conf.content
def find(self, key: str|None, val: str|None, match:StringTree.Match=StringTree.Match.Equal) -> list[str]:
return self.__conf.find(key, val, match=match)
def find(
self,
key: str | None,
val: str | None,
match: StringTree.Match = StringTree.Match.Equal
) -> list[str]:
return self.__conf.find(key, val, match = match)
#def __getattr__(self, name: str):
# return getattr(self.__conf, name)

21
src/python/jw/util/CppState.py
View file

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
class CppState: # export
class CppState: # export
def __init__(self):
self.__pair_square = ['[', ']']
@ -33,37 +31,39 @@ class CppState: # export
self.things.append(self.__pair_square)
elif tok == ']':
self.square -= 1
assert(self.things.pop() == self.__pair_square)
assert (self.things.pop() == self.__pair_square)
elif tok == '{':
self.curly += 1
self.things.append(self.__pair_curly)
elif tok == '}':
self.curly -= 1
assert(self.things.pop() == self.__pair_curly)
assert (self.things.pop() == self.__pair_curly)
elif tok == '(':
self.paren += 1
self.things.append(self.__pair_paren)
elif tok == ')':
self.paren -= 1
assert(self.things.pop() == self.__pair_paren)
assert (self.things.pop() == self.__pair_paren)
elif tok == '<':
self.ext += 1
self.things.append(self.__pair_ext)
elif tok == '>':
self.ext -= 1
assert(self.things.pop() == self.__pair_ext)
assert (self.things.pop() == self.__pair_ext)
elif tok == '?':
if not self.in_special:
self.in_special = True
self.things.append(self.__pair_special)
else:
self.in_special = False
assert(self.things.pop() == self.__pair_special)
assert (self.things.pop() == self.__pair_special)
elif tok == '/*':
self.in_c_comment = True
self.things.append(self.__pair_c_comment)
elif tok == '*/':
raise Exception("Unmatched closing C-style comment mark", tok, "in line", line)
raise Exception(
"Unmatched closing C-style comment mark", tok, "in line", line
)
else:
if self.in_cpp_comment:
if tok == '\n':
@ -72,7 +72,7 @@ class CppState: # export
if tok == '/*':
raise Exception("Nested C-style comment", tok, "in line", line)
elif tok == '*/':
assert(self.things.pop() == self.__pair_c_comment)
assert (self.things.pop() == self.__pair_c_comment)
self.in_c_comment = False
if self.curly < 0 or self.square < 0 or self.ext < 0 or self.paren < 0:
@ -101,4 +101,3 @@ class CppState: # export
def is_optional(self):
return self.in_list() or self.in_option()

2
src/python/jw/util/Makefile
View file

@ -1,6 +1,4 @@
TOPDIR = ../../../..
PY_UPDATE_INIT_PY ?= false
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-mod.mk

8
src/python/jw/util/Object.py
View file

@ -1,16 +1,14 @@
# -*- coding: utf-8 -*-
from __future__ import print_function
from . import log
class Object(object): # export
class Object(object): # export
def __init__(self):
self.log_level = log.level
self.log_level = log.log_level()
def log(self, prio, *args):
if self.log_level == log.level:
if self.log_level == log.log_level():
log.slog(prio, args)
return
if prio <= self.log_level:

69
src/python/jw/util/Options.py
View file

@ -1,11 +1,13 @@
import re
import json
from collections import OrderedDict
from .log import *
import re
import shlex
import traceback
class Options: # export
from collections import OrderedDict
from .log import ERR, get_caller_pos, slog, slog_m
class Options: # export
class OrderedData:
@ -30,8 +32,8 @@ class Options: # export
if spec[0] != '{':
spec = '{' + spec + '}'
try:
return json.loads(spec, object_pairs_hook=cls)
except:
return json.loads(spec, object_pairs_hook = cls)
except Exception:
pass
return None
@ -42,7 +44,7 @@ class Options: # export
r = cls()
try:
opt_strs = shlex.split(opts_str)
except Exception as e:
except Exception:
slog_m(ERR, traceback.format_exc())
slog(ERR, 'Failed to split options string >{}<'.format(opts_str))
raise
@ -52,7 +54,7 @@ class Options: # export
lhs = sides[0].strip()
if not len(lhs):
continue
if self.__allowed_keys and not lhs in self.__allowed_keys:
if self.__allowed_keys and lhs not in self.__allowed_keys:
raise Exception('Field "{}" not supported'.format(lhs))
rhs = ' '.join(sides[1:]).strip() if len(sides) > 1 else self.__true_val
if cls == OrderedDict:
@ -82,7 +84,7 @@ class Options: # export
self.__str = self.__str__()
def __getitem__(self, key):
if not key in self.__dict.keys():
if key not in self.__dict.keys():
return None
return self.__dict[key]
@ -99,35 +101,38 @@ class Options: # export
return len(self.__data.pairs)
def __contains__(self, keys):
if not type(keys) in [list, set]:
if type(keys) not in [list, set]:
return keys in self.__dict.keys()
for key in keys:
if not key in self.__dict.keys():
if key not in self.__dict.keys():
return False
return True
def __iter__(self):
return iter(self.__list)
def __next__(self):
return next(self.__list)
#def __next__(self):
# return next(self.__list)
def __init__(self, spec=None, delimiter=',', allowed_keys=None, true_val=True):
def __init__(
self, spec = None, delimiter = ',', allowed_keys = None, true_val = True
):
self.__true_val = true_val
self.__allowed_keys = None
self.__delimiter = delimiter
self.__data = self.OrderedData() if spec is None else self.__parse(spec, self.OrderedData)
self.__data = self.OrderedData(
) if spec is None else self.__parse(spec, self.OrderedData)
self.__dict = {}
#self.__dict = OrderedDict() if spec is None else self.__parse(spec, OrderedDict)
#self.__dict = OrderedDict() if spec is None else self.__parse(spec,OrderedDict)
self.__list = []
self.__str = None
self.__recache()
def dump(self, prio, caller=None):
def dump(self, prio, caller = None):
if caller is None:
caller = get_caller_pos()
for key, val in self.__data.pairs:
slog(prio, "{}=\"{}\"".format(key, val), caller=caller)
slog(prio, "{}=\"{}\"".format(key, val), caller = caller)
def keys(self):
return self.__dict.keys()
@ -136,22 +141,28 @@ class Options: # export
#return self.__dict.items()
return self.__data.pairs
def get(self, key, default=None, by_index=False):
def get(self, key, default = None, by_index = False):
if by_index:
if type(key) != int:
raise KeyError('Tried to get value from options string with ' +
'index {} of type "{}": {}'.format(key, type(key), str(self)))
if isinstance(key, int):
raise KeyError(
'Tried to get value from options string with ' +
'index {} of type "{}": {}'.format(key, type(key), str(self))
)
if key >= len(self.__data.pairs):
if default is not None:
return default
raise KeyError('Tried to get value from options string with ' +
'index {} of {}: {}'.format(key, len(self.__data.pairs), str(self)))
raise KeyError(
'Tried to get value from options string with ' +
'index {} of {}: {}'.format(key, len(self.__data.pairs), str(self))
)
return self.__list[key]
if key in self.__dict.keys():
return self.__dict[key]
if default is not None:
return default
raise KeyError('Key "{}" is not present in options string: {}'.format(key, str(self)))
raise KeyError(
'Key "{}" is not present in options string: {}'.format(key, str(self))
)
def update(self, rhs):
if hasattr(rhs, 'items'):
@ -159,9 +170,13 @@ class Options: # export
self.__dict[key] = val
return
if isinstance(rhs, str):
self.update(self.__parse(rhs))
self.update(self.__parse(rhs, self.OrderedData))
return
raise Exception('Tried to update options with object of incompatible type {}'.format(type(rhs)))
raise Exception(
'Tried to update options with object of incompatible type {}'.format(
type(rhs)
)
)
def append_to(self, obj):
for opt in self.__list:

47
src/python/jw/util/Process.py
View file

@ -1,41 +1,46 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import signal
from abc import ABC, abstractmethod
from enum import Enum, Flag, auto
from typing import List
from typing import TYPE_CHECKING
from .log import ERR, slog
if TYPE_CHECKING:
from .Signals import Signals
def _sigchld_handler(signum, process):
if not signum == signal.SIGCHLD:
return
Process.propagate_signal(signum)
class Process(ABC): # export
class Process(ABC): # export
__processes: List[Process] = []
__processes: set[Process] = set()
class State(Enum):
Running = auto()
Shutdown = auto()
Done = auto()
Running = auto()
Shutdown = auto()
Done = auto()
class Flags(Flag):
FailOnExitWithoutShutdown = auto()
def __init__(self):
self.__state = Running
self.__flags = Flags.FailOnExitWithoutShutdown
self.__state = Process.State.Running
self.__flags = self.Flags.FailOnExitWithoutShutdown
if len(self.__processes) == 0:
self._signals().add_handler(signals.SIGCHLD, _sigchld_handler)
self.signals().add_handler(signal.SIGCHLD, _sigchld_handler)
self.__processes.add(self)
@classmethod
def propagate_signal(cls, signum):
for p in cls.__processes:
p.__signal(signum)
cls.signals().propagate(signum)
def signal(self, signum):
if signum == signals.SIGCHLD:
if signum == signal.SIGCHLD:
self.exited()
@abstractmethod
@ -44,7 +49,7 @@ class Process(ABC): # export
@classmethod
@abstractmethod
def signals(cls):
def signals(cls) -> Signals:
pass
# to be reimplemented
@ -56,17 +61,15 @@ class Process(ABC): # export
return str(self._pid())
def request_shutdown(self):
if not self.__state == Shutdown:
self.__state = Shutdown
if not self.__state == Process.State.Shutdown:
self.__state = Process.State.Shutdown
self._request_shutdown()
def exited(self):
if self.__state == Process.State.Running:
slog(ERR, 'process "{}" exited unexpectedly'.format(process.name()))
if __flags & Process.Flags.FailOnExitWithoutShutdown:
slog(ERR, 'process exited unexpectedly')
if self.__flags & Process.Flags.FailOnExitWithoutShutdown:
slog(ERR, 'exiting')
exit(1)
self.__state = Process.State.Done
self.__processes.erase(self)
if len(self.__processes) == 0:
self._signals().remove_handler(signals.SIGCHLD) # FIXME: broken logic
self.__processes.remove(self)

28
src/python/jw/util/RedirectStdIO.py
View file

@ -1,13 +1,13 @@
# -*- coding: utf-8 -*-
from __future__ import print_function
import os, io, sys, traceback
import os
import io
import sys
from fcntl import fcntl, F_GETFL, F_SETFL
class RedirectStdIO: # export
class RedirectStdIO: # export
def __init__(self, stderr='on', stdout='off'):
def __init__(self, stderr = 'on', stdout = 'off'):
self.__stderr = stderr
self.__stdout = stdout
# TODO: arguments not fully implemented,
@ -30,12 +30,12 @@ class RedirectStdIO: # export
sys.stdout.flush()
os.dup2(self.real_stdout_fd, 1)
if type is not None:
#print("-------- Error while stdio was suppressed --------")
#traceback.print_stack()
#print(traceback)
print("-------- Captured output --------")
print(*self.rfile.readlines())
self.rfile.close()
#print('type = ' + str(type))
#print('value = ' + str(value))
raise type(value)
#print("-------- Error while stdio was suppressed --------")
#traceback.print_stack()
#print(traceback)
print("-------- Captured output --------")
print(*self.rfile.readlines())
self.rfile.close()
#print('type = ' + str(type))
#print('value = ' + str(value))
raise type(value)

35
src/python/jw/util/Signals.py
View file

@ -1,19 +1,10 @@
# -*- coding: utf-8 -*-
from typing import Dict, Callable
from abc import ABC, abstractmethod
_handled_signals: Dict[int, Callable] = {}
def _signal_handler(signal, frame):
if not signal in _handled_signals.keys():
return
for h in _handled_signals[signal]:
h.func(signal, *h.args)
from abc import abstractmethod
from typing import Dict
class Signals:
class Handler:
def __init__(self, func, args):
self.func = func
self.args = args
@ -23,15 +14,27 @@ class Signals:
@classmethod
@abstractmethod
def _add_handler(self, signal, handler):
def _add_handler(cls, signal, handler):
raise Exception("_add_handler() is not reimplemented")
@classmethod
def add_handler(cls, signals, handler, *args):
for signal in signals:
h = Signals.Handler(handler, args)
if not signal in _handled_signals.keys():
if signal not in _handled_signals.keys():
_handled_signals[signal] = [h]
cls._add_signal_handler(signal, _signal_handler)
cls._add_handler(signal, _signal_handler)
else:
_handled_signals[signal].add(h)
_handled_signals[signal].append(h)
@classmethod
def propagate(cls, signal):
_signal_handler(signal, None)
_handled_signals: Dict[int, list[Signals.Handler]] = {}
def _signal_handler(signal, frame):
if signal not in _handled_signals.keys():
return
for h in _handled_signals[signal]:
h.func(signal, *h.args)

14
src/python/jw/util/StopWatch.py
View file

@ -1,12 +1,10 @@
# -*- coding: utf-8 -*-
from datetime import datetime
from .log import *
from .log import get_caller_pos, slog
class StopWatch: # export
class StopWatch: # export
def __init__(self, name=''):
def __init__(self, name = ''):
self.__start = datetime.now()
self.__last = self.__start
self.name = name
@ -21,5 +19,9 @@ class StopWatch: # export
else:
msg = '------------------ '
caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
slog(prio, '{} {} {}'.format(self.name, str(now - self.__last), msg), caller=caller)
slog(
prio,
'{} {} {}'.format(self.name, str(now - self.__last), msg),
caller = caller
)
self.__last = now

3
src/python/jw/util/__init__.py
View file

@ -1,3 +0,0 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)

124
src/python/jw/util/algo/ShuntingYard.py
View file

@ -1,16 +1,15 @@
# -*- coding: utf-8 -*-
import re
import re, shlex
from collections import namedtuple
from ..log import *
from ..log import DEBUG, get_caller_pos, prio_gets_logged, slog
L, R = 'Left Right'.split()
ARG, KEYW, QUOTED, LPAREN, RPAREN = 'arg kw quoted ( )'.split()
class Operator: # export
class Operator: # export
def __init__(self, func=None, nargs=2, precedence=3, assoc=L):
def __init__(self, func = None, nargs = 2, precedence = 3, assoc = L):
self.func = func
self.nargs = nargs
self.prec = precedence
@ -18,7 +17,7 @@ class Operator: # export
class Stack:
def __init__(self, itemlist=[]):
def __init__(self, itemlist = []):
self.items = itemlist
def __repr__(self):
@ -39,7 +38,7 @@ class Stack:
self.items.append(item)
return 0
class ShuntingYard(object): # export
class ShuntingYard(object): # export
def __init__(self, operators = None):
self.do_debug = prio_gets_logged(DEBUG)
@ -54,7 +53,7 @@ class ShuntingYard(object): # export
for count, thing in enumerate(args):
msg += ' ' + str(thing)
if len(msg):
slog(DEBUG, msg[1:], caller=get_caller_pos())
slog(DEBUG, msg[1:], caller = get_caller_pos())
def operator(self, key: str) -> Operator:
return self.__ops[key]
@ -65,7 +64,7 @@ class ShuntingYard(object): # export
v = self.__ops[k]
buf = ", \"" + k
if v.nargs == 1:
if k[len(k)-1].isalnum():
if k[len(k) - 1].isalnum():
buf = buf + ' '
buf = buf + "xxx"
buf = buf + "\""
@ -83,16 +82,20 @@ class ShuntingYard(object): # export
regex = regex[1:]
scanner = re.Scanner([
(regex, lambda scanner,token:(KEYW, token)),
(r"\"[^\"]*\"|'[^']*'", lambda scanner,token:(QUOTED, token[1:-1])),
(r"[^\s()]+", lambda scanner,token:(ARG, token)),
(r"\s+", None), # None == skip token.
])
scanner = re.Scanner( # pyright: ignore[reportAttributeAccessIssue]
[
(regex, lambda scanner, token: (KEYW, token)),
(r"\"[^\"]*\"|'[^']*'", lambda scanner, token: (QUOTED, token[1:-1])),
(r"[^\s()]+", lambda scanner, token: (ARG, token)),
(r"\s+", None), # None == skip token.
]
)
tokens, remainder = scanner.scan(spec)
if len(remainder)>0:
raise Exception("Failed to tokenize " + spec + ", remaining bit is ", remainder)
if len(remainder) > 0:
raise Exception(
"Failed to tokenize " + spec + ", remaining bit is ", remainder
)
#self.debug(tokens)
return tokens
@ -112,14 +115,14 @@ class ShuntingYard(object): # export
tokenized = self.tokenize(infix)
self.debug("tokenized = ", tokenized)
outq, stack = [], []
table = ['TOKEN,ACTION,RPN OUTPUT,OP STACK,NOTES'.split(',')]
table = ['TOKEN', 'ACTION', 'RPN OUTPUT', ('OP STACK', ), 'NOTES']
for toktype, token in tokenized:
self.debug("Checking token", token)
note = action = ''
if toktype in [ ARG, QUOTED ]:
if toktype in [ARG, QUOTED]:
action = 'Add arg to output'
outq.append(token)
table.append( (token, action, outq, (s[0] for s in stack), note) )
table.append((token, action, outq, (s[0] for s in stack), note))
elif toktype == KEYW:
val = self.__ops[token]
t1, op1 = token, val
@ -127,7 +130,9 @@ class ShuntingYard(object): # export
note = 'Pop ops from stack to output'
while stack:
t2, op2 = stack[-1]
if (op1.assoc == L and op1.prec <= op2.prec) or (op1.assoc == R and op1.prec < op2.prec):
if (op1.assoc == L
and op1.prec <= op2.prec) or (op1.assoc == R
and op1.prec < op2.prec):
if t1 != RPAREN:
if t2 != LPAREN:
stack.pop()
@ -143,9 +148,11 @@ class ShuntingYard(object): # export
else:
stack.pop()
action = '(Pop & discard "(")'
table.append( (v, action, outq, (s[0] for s in stack), note) )
table.append(
(v, action, outq, (s[0] for s in stack), note)
)
break
table.append( (v, action, (outq), (s[0] for s in stack), note) )
table.append((v, action, (outq), (s[0] for s in stack), note))
v = note = ''
else:
note = ''
@ -157,7 +164,7 @@ class ShuntingYard(object): # export
action = 'Push op token to stack'
else:
action = 'Discard ")"'
table.append( (v, action, (outq), (s[0] for s in stack), note) )
table.append((v, action, (outq), (s[0] for s in stack), note))
note = 'Drain stack to output'
while stack:
v = ''
@ -165,15 +172,27 @@ class ShuntingYard(object): # export
action = '(Pop op)'
stack.pop()
outq.append(t2)
table.append( (v, action, outq, (s[0] for s in stack), note) )
table.append((v, action, outq, (s[0] for s in stack), note))
v = note = ''
if self.do_debug:
maxcolwidths = [len(max(x, key=len)) for x in [zip(*table)]]
caller = get_caller_pos()
maxcolwidths = [len(max(x, key = len)) for x in [zip(*table)]]
get_caller_pos()
row = table[0]
slog(DEBUG, ' '.join('{cell:^{width}}'.format(width=width, cell=cell) for (width, cell) in zip(maxcolwidths, row)))
slog(
DEBUG,
' '.join(
'{cell:^{width}}'.format(width = width, cell = cell)
for (width, cell) in zip(maxcolwidths, row)
)
)
for row in table[1:]:
slog(DEBUG, ' '.join('{cell:<{width}}'.format(width=width, cell=cell) for (width, cell) in zip(maxcolwidths, row)))
slog(
DEBUG,
' '.join(
'{cell:<{width}}'.format(width = width, cell = cell)
for (width, cell) in zip(maxcolwidths, row)
)
)
return table[-1][2]
def infix_to_postfix_orig(self, infix):
@ -185,7 +204,7 @@ class ShuntingYard(object): # export
for tokinfo in tokens:
self.debug(tokinfo)
toktype, token = tokinfo[0], tokinfo[1]
_toktype, token = tokinfo[0], tokinfo[1]
self.debug("Checking token ", token)
@ -204,7 +223,8 @@ class ShuntingYard(object): # export
topToken = s.pop()
continue
while (not s.isEmpty()) and (self.__ops[s.peek()].prec >= self.__ops[token].prec):
while (not s.isEmpty()) and (self.__ops[s.peek()].prec
>= self.__ops[token].prec):
#self.debug(token)
r.append(s.pop())
#self.debug(r)
@ -240,7 +260,9 @@ class ShuntingYard(object): # export
args.append(vals.pop())
#self.debug("running %s(%s)" % (token, ', '.join(reversed(args))))
val = op.func(*reversed(args))
self.debug("%s(%s) = %s" % (token, ', '.join(map(str, reversed(args))), val))
self.debug(
"%s(%s) = %s" % (token, ', '.join(map(str, reversed(args))), val)
)
vals.push(val)
return vals.pop()
@ -266,27 +288,27 @@ if __name__ == '__main__':
# return string.split()
def f_mult(self, a, b):
return str(atof(a) * atof(b));
return str(atof(a) * atof(b))
def f_div(self, a, b):
return str(atof(a) / atof(b));
return str(atof(a) / atof(b))
def f_add(self, a, b):
return str(atof(a) + atof(b));
return str(atof(a) + atof(b))
def f_sub(self, a, b):
return str(atof(a) - atof(b));
return str(atof(a) - atof(b))
def __init__(self):
Op = Operator
operators = {
'^': Op(None, 2, 4, R),
'^': Op(None, 2, 4, R),
'*': Op(self.f_mult, 2, 3, L),
'/': Op(self.f_div, 2, 3, L),
'+': Op(self.f_add, 2, 2, L),
'-': Op(self.f_sub, 2, 2, L),
'(': Op(None, 0, 9, L),
')': Op(None, 0, 0, L),
'/': Op(self.f_div, 2, 3, L),
'+': Op(self.f_add, 2, 2, L),
'-': Op(self.f_sub, 2, 2, L),
'(': Op(None, 0, 9, L),
')': Op(None, 0, 0, L),
}
super(Calculator, self).__init__(operators)
@ -295,7 +317,7 @@ if __name__ == '__main__':
# ------------- testbed match object
Object = namedtuple("Object", [ "Name", "Label" ])
Object = namedtuple("Object", ["Name", "Label"])
class Matcher(ShuntingYard):
@ -324,14 +346,14 @@ if __name__ == '__main__':
def __init__(self, obj):
Op = Operator
operators = {
'(': Op(None, 2, 9, L),
')': Op(None, 2, 0, L),
'name=': Op(self.f_is_name, 1, 3, R),
'and': Op(self.f_and, 2, 3, L),
'label~=': Op(self.f_matches_label, 1, 3, R),
'False': Op(None, 0, 3, L),
'True': Op(None, 0, 3, L),
'not': Op(self.f_is_not, 1, 3, R),
'(': Op(None, 2, 9, L),
')': Op(None, 2, 0, L),
'name=': Op(self.f_is_name, 1, 3, R),
'and': Op(self.f_and, 2, 3, L),
'label~=': Op(self.f_matches_label, 1, 3, R),
'False': Op(None, 0, 3, L),
'True': Op(None, 0, 3, L),
'not': Op(self.f_is_not, 1, 3, R),
}
super(Matcher, self).__init__(operators)

6
src/python/jw/util/asyncio/Process.py
View file

@ -1,16 +1,14 @@
# -*- coding: utf-8 -*-
from abc import abstractmethod
from ..Process import Process as ProcessBase
from .Signals import Signals
class Process(ProcessBase): # export
class Process(ProcessBase): # export
__signals = Signals()
def __init__(self, aio):
super().__init()
super().__init__()
self.aio = aio
@classmethod

51
src/python/jw/util/asyncio/ShellCmd.py
View file

@ -1,9 +1,11 @@
import asyncio
from ..log import *
import re
from ..log import DEBUG, ERR, INFO, WARNING, slog
# FIXME: Derive this from Process, or merge the classes entirely
class ShellCmd: # export
class ShellCmd: # export
class SubprocessProtocol(asyncio.SubprocessProtocol):
@ -26,12 +28,12 @@ class ShellCmd: # export
self.process.exited()
class ShutdownState:
Running = 1
Triggered = 2
Completed = 3
Running = 1
Triggered = 2
Completed = 3
Unnecessary = 4
def __init__(self, cmdline, eloop=None, name=None):
def __init__(self, cmdline, eloop = None, name = None):
if eloop is None:
eloop = asyncio.get_running_loop()
self.__eloop = eloop
@ -56,12 +58,19 @@ class ShellCmd: # export
return r[1:]
try:
slog(INFO, "Running shell command [{}]: {}".format(self.__name, format_cmdline(self.__cmdline)))
slog(
INFO,
"Running shell command [{}]: {}".format(
self.__name, format_cmdline(self.__cmdline)
)
)
self.__transport, self.__protocol = await self.__eloop.subprocess_exec(
lambda: self.SubprocessProtocol(self, self.__name),
*self.__cmdline,
)
self.__proc = self.__transport.get_extra_info('subprocess') # Popen instance
self.__proc = self.__transport.get_extra_info(
'subprocess'
) # Popen instance
except:
slog(ERR, "Failed to run process [{}]".format(self.__name))
raise
@ -69,7 +78,8 @@ class ShellCmd: # export
def __reap(self):
if self.__rc is None and self.__transport:
self.__transport = None
self.__rc = self.__proc.wait()
if self.__proc is not None:
self.__rc = self.__proc.wait()
# to be called from SubprocessProtocol / SIGCHLD handler
def exited(self):
@ -78,13 +88,24 @@ class ShellCmd: # export
async def __cleanup(self):
pid = self.__reap()
sd_fine = self.__shutdown in [ self.ShutdownState.Unnecessary, self.ShutdownState.Completed ]
sd_fine = self.__shutdown in [
self.ShutdownState.Unnecessary, self.ShutdownState.Completed
]
if self.__rc == 0 and sd_fine:
slog(INFO, "The shell command [{}], pid {}, has exited cleanly".format(self.__name, self.__proc.pid))
assert self.__proc is not None
slog(
INFO,
"The shell command [{}], pid {}, has exited cleanly".format(
self.__name, self.__proc.pid
)
)
self.monitor = self.console = self.__protocol = self.__task = None
return 0
slog(ERR, "The process ([{}], pid {}) has exited {}with status code {}, aborting".format(
self.__name, pid, "" if sd_fine else "prematurely ", self.__rc))
slog(
ERR,
"The process ([{}], pid {}) has exited {}with status code {}, aborting".
format(self.__name, pid, "" if sd_fine else "prematurely ", self.__rc)
)
exit(1)
async def init(self):
@ -100,9 +121,9 @@ class ShellCmd: # export
if __name__ == '__main__':
from .. import log
log.set_level('info')
async def run():
sp = ShellCmd([ 'echo', 'hello world!' ])
sp = ShellCmd(['echo', 'hello world!'])
await sp.run()
asyncio.run(run())

4
src/python/jw/util/asyncio/Signals.py
View file

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
import asyncio
from ..Signals import Signals as SignalsBase
@ -10,4 +8,4 @@ class Signals(SignalsBase):
@classmethod
def _add_handler(cls, signal, handler):
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal, handler, None) # None = *args
loop.add_signal_handler(signal, handler, None) # None = *args

44
src/python/jw/util/auth/Auth.py
View file

@ -1,27 +1,28 @@
# -*- coding: utf-8 -*-
from typing import Optional, Union, Self
from __future__ import annotations
import abc
from enum import Flag, Enum, auto
from enum import Enum, Flag, auto
from typing import TYPE_CHECKING, Optional, Self, Union
from ..log import *
from ..Config import Config
from ..log import ERR
from ..misc import load_object
class Access(Enum): # export
Read = auto()
if TYPE_CHECKING:
from ..Config import Config
class Access(Enum): # export
Read = auto()
Modify = auto()
Create = auto()
Delete = auto()
class ProjectFlags(Flag): # export
NoFlags = auto()
class ProjectFlags(Flag): # export
NoFlags = auto()
Contributing = auto()
Active = auto()
Active = auto()
class Group: # export
class Group: # export
def __repr__(self):
return f'Group({self.name})'
@ -34,7 +35,7 @@ class Group: # export
def name(self) -> str:
return self._name()
class User: # export
class User: # export
def __repr__(self):
return f'User({self.name})'
@ -70,14 +71,14 @@ class User: # export
def email(self) -> str:
return self._email()
class Auth(abc.ABC): # export
class Auth(abc.ABC): # export
@classmethod
def load(cls, conf: Config, tp: str='') -> Self:
def load(cls, conf: Config, tp: str = '') -> Self:
if tp == '':
val = conf.get('type')
if val is None:
msg = f'No type specified in auth configuration'
msg = 'No type specified in auth configuration'
conf.dump(ERR, msg)
raise Exception(msg)
tp = val
@ -92,10 +93,17 @@ class Auth(abc.ABC): # export
return self.__conf
@abc.abstractmethod
def _access(self, what: str, access_type: Optional[Access], who: User|Group|None) -> bool:
def _access(
self, what: str, access_type: Optional[Access], who: User | Group | None
) -> bool:
raise NotImplementedError
def access(self, what: str, access_type: Optional[Access]=None, who: Optional[Union[User|Group]]=None) -> bool:
def access(
self,
what: str,
access_type: Optional[Access] = None,
who: Optional[Union[User, Group]] = None
) -> bool:
return self._access(what, access_type, who)
@abc.abstractmethod

38
src/python/jw/util/auth/dummy/Auth.py
View file

@ -1,16 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Optional, Union
from typing import TYPE_CHECKING, Optional
from ...log import *
from ... import Config
from .. import Access
from .. import Auth as AuthBase
from .. import Group as GroupBase
from .. import User as UserBase
from .. import ProjectFlags
from ...log import WARNING, slog
from ..Auth import Access
from ..Auth import Auth as AuthBase
from ..Auth import Group as GroupBase
from ..Auth import ProjectFlags
from ..Auth import User as UserBase
class Group(GroupBase): # export
if TYPE_CHECKING:
from ...Config import Config
class Group(GroupBase): # export
def __init__(self, auth: AuthBase, name: str):
self.__name = name
@ -19,7 +21,7 @@ class Group(GroupBase): # export
def _name(self) -> str:
return self.__name
class User(UserBase): # export
class User(UserBase): # export
def __init__(self, auth: AuthBase, name: str, conf: Config):
self.__name = name
@ -47,13 +49,13 @@ class User(UserBase): # export
def _email(self) -> str:
return self.__email
class Auth(AuthBase): # export
class Auth(AuthBase): # export
def __init__(self, conf: Config):
super().__init__(conf)
self.___users: Optional[dict[str, UserBase]] = None
self.__groups = None
self.__current_user: UserBase|None = None
self.__current_user: UserBase | None = None
self.__user_by_email: Optional[dict[str, UserBase]] = None
@property
@ -62,12 +64,18 @@ class Auth(AuthBase): # export
ret: dict[str, UserBase] = {}
for name in self.conf.entries('user'):
conf = self.conf.branch('user.' + name)
assert conf is not None, 'Config is None'
ret[name] = User(self, name, conf)
self.___users = ret
return self.___users
def _access(self, what: str, access_type: Optional[Access], who: User|GroupBase|None) -> bool: # type: ignore
slog(WARNING, f'Returning False for {access_type} access to resource {what} by {who}')
def _access(
self, what: str, access_type: Access | None, who: UserBase | GroupBase | None
) -> bool: # type: ignore
slog(
WARNING,
f'Returning False for {access_type} access to resource {what} by {who}'
)
return False
def _user(self, name) -> UserBase:

66
src/python/jw/util/auth/ldap/Auth.py
View file

@ -1,19 +1,21 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Optional, Union
from typing import TYPE_CHECKING, Optional
import ldap
import ldap # type: ignore[import-untyped]
from ...log import *
from ...ldap import bind
from ...Config import Config
from .. import Access
from .. import Auth as AuthBase
from .. import Group as GroupBase
from .. import User as UserBase
from .. import ProjectFlags
from ...log import DEBUG, ERR, WARNING, slog
from ..Auth import Access
from ..Auth import Auth as AuthBase
from ..Auth import Group as GroupBase
from ..Auth import ProjectFlags
from ..Auth import User as UserBase
class Group(GroupBase): # export
if TYPE_CHECKING:
from ...Config import Config
class Group(GroupBase): # export
def __init__(self, auth: AuthBase, name: str):
self.__name = name
@ -24,13 +26,7 @@ class Group(GroupBase): # export
class User(UserBase):
def __init__(
self,
auth: AuthBase,
name: str,
cn: str,
email: str
):
def __init__(self, auth: AuthBase, name: str, cn: str, email: str):
self.__auth = auth
self.__name = name
@ -50,14 +46,14 @@ class User(UserBase):
def _display_name(self) -> str:
return self.__cn
class Auth(AuthBase): # export
class Auth(AuthBase): # export
def __init__(self, conf: Config):
super().__init__(conf)
self.___users: Optional[dict[str, UserBase]] = None
self.___user_by_email: Optional[dict[str, User]] = None
self.__groups = None
self.__current_user: User|None = None
self.__current_user: User | None = None
self.__user_base_dn = conf['user_base_dn']
self.__conn = self.__bind()
self.__dummy = self.load(conf, 'dummy')
@ -72,18 +68,16 @@ class Auth(AuthBase): # export
ret_by_email: dict[str, User] = {}
for res in self.__conn.find(
self.__user_base_dn,
ldap.SCOPE_SUBTREE,
ldap.SCOPE_SUBTREE, # pyright: ignore[reportAttributeAccessIssue]
"objectClass=inetOrgPerson",
('uid', 'cn', 'uidNumber', 'mail', 'maildrop')
):
('uid', 'cn', 'uidNumber', 'mail', 'maildrop')):
try:
display_name = None
if 'displayName' in res[1]:
cn = res[1]['displayName'][0].decode('utf-8')
else:
cn = res[1]['cn'][0].decode('utf-8')
uid = res[1]['uid'][0].decode('utf-8')
uidNumber = res[1]['uidNumber'][0].decode('utf-8')
res[1]['uidNumber'][0].decode('utf-8')
emails = []
#for attr in ['mail', 'maildrop']:
for attr in ['mail']:
@ -93,7 +87,7 @@ class Auth(AuthBase): # export
if not emails:
slog(DEBUG, f'No email for user "{uid}", skipping')
continue
user = User(self, name=uid, cn=cn, email=emails[0])
user = User(self, name = uid, cn = cn, email = emails[0])
ret[uid] = user
for email in emails:
ret_by_email[email] = user
@ -111,10 +105,18 @@ class Auth(AuthBase): # export
def __user_by_email(self) -> dict[str, UserBase]:
if self.___user_by_email is None:
self.__users
return self.___user_by_email # type: ignore # We are sure that ___user_by_email is not None at this point
return self.___user_by_email # type: ignore # We are sure that ___user_by_email is not None at this point
def _access(self, what: str, access_type: Optional[Access], who: User|GroupBase|None) -> bool: # type: ignore
slog(WARNING, f'Returning False for {access_type} access to resource {what} by {who}')
def _access(
self,
what: str,
access_type: Optional[Access],
who: UserBase | GroupBase | None
) -> bool: # type: ignore
slog(
WARNING,
f'Returning False for {access_type} access to resource {what} by {who}'
)
return False
def _user(self, name) -> UserBase:
@ -136,5 +138,9 @@ class Auth(AuthBase): # export
def _projects(self, name, flags: ProjectFlags) -> list[str]:
if flags & ProjectFlags.Contributing:
# TODO: Ask LDAP
slog(WARNING, f'Querying LDAP for projects a user contributes to is not implemented, ignoring')
slog(
WARNING,
'Querying LDAP for projects a user contributes to is not '
'implemented, ignoring'
)
return []

160
src/python/jw/util/cast.py
View file

@ -1,110 +1,130 @@
# -*- coding: utf-8 -*-
import os
import pytimeparse, os
from datetime import datetime, timedelta
from collections import OrderedDict
from datetime import timedelta
from .log import *
import pytimeparse # type: ignore[import-untyped]
from .log import DEBUG, WARNING, slog
_int_chars = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
def _strip(s_, throw=True, log_level=ERR):
def _strip(s_) -> str:
s = s_.strip()
if len(s) != 0:
return s
msg = f'Tried to strip empty string "{s_}" to int'
if throw:
raise Exception(msg)
slog(log_level, msg)
return None
raise Exception(f'Tried to strip empty string "{s_}"')
def cast_str_to_timedelta(s_: str, throw=True, log_level=DEBUG): # export
s = _strip(s_, throw=throw, log_level=log_level)
try:
return (True, timedelta(seconds=pytimeparse.parse(s_)))
except Exception as e:
msg = f'Could not convert string "{s_}" to time ({e})'
if throw:
raise Exception(msg)
slog(log_level, msg)
return (False, None)
def cast_str_to_timedelta(s_: str): # export
s = _strip(s_)
seconds = pytimeparse.parse(s)
if seconds is None:
raise Exception(f'Failed to convert {s} to timedelta')
return timedelta(seconds = seconds)
def cast_str_to_int(s_: str, throw=True, log_level=DEBUG): # export
s = _strip(s_, throw=throw, log_level=log_level)
def cast_str_to_int(s_: str): # export
s = _strip(s_)
if s[0] == '-':
s = s[1:]
for c in s:
if not c in _int_chars:
break
else:
return (True, int(s_))
msg = f'Could not convert string "{s_}" to int'
if throw:
raise Exception(msg)
slog(log_level, msg)
return (False, None)
if c not in _int_chars:
raise Exception(f'Could not convert string "{s}" to int')
return int(s)
def cast_str_to_bool(s_: str, throw=True, log_level=DEBUG): # export
s = _strip(s_, throw=throw, log_level=log_level).lower()
def cast_str_to_bool(s_: str): # export
s = _strip(s_).lower()
if s in ['true', 'yes', '1']:
return (True, True)
return True
if s in ['false', 'no', '0']:
return (True, False)
msg = f'Could not convert string "{s_}" to bool'
if throw:
raise Exception(msg)
slog(log_level, msg)
return (False, None)
return False
raise Exception(f'Could not convert string "{s_}" to bool')
_str_cast_functions = OrderedDict({
bool: cast_str_to_bool,
int: cast_str_to_int,
timedelta: cast_str_to_timedelta
_str_cast_functions = OrderedDict(
{
bool: cast_str_to_bool, int: cast_str_to_int, timedelta: cast_str_to_timedelta
}
)
})
def guess_type(s: str, default=None, log_level=DEBUG, throw=False): # export
def guess_type(s: str, default = None, log_level = DEBUG, throw = False): # export
if s is None:
raise Exception('None string passed to guess_type()')
for tp, func in _str_cast_functions.items():
try:
success, value = func(s, log_level=OFF, throw=False)
if success:
return tp
except:
func(s)
except Exception:
continue
return tp
msg = f'Failed to guess type of string "{s}"'
if throw:
raise Exception(msg)
slog(log_level, msg)
return default
def from_str(s: str, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None): # export
def from_str( # export
s: str,
target_type = None,
default_type = None,
throw = True,
log_level = WARNING,
caller = None
):
if target_type is None:
target_type = guess_type(s, default_type)
if target_type is None:
msg = f'Could not deduce type to cast to from string "{s}"'
if throw:
raise Exception(msg)
slog(log_level, msg)
return None
result = _str_cast_functions[target_type](s, throw=throw, log_level=log_level)
if result[0]:
return result[1]
msg = f'Failed to cast string "{s}" to type {target_type}'
if throw:
raise Exception(msg)
slog(log_level, msg)
for tp, func in _str_cast_functions.items():
try:
return func(s)
except Exception:
continue
msg = f'Could not deduce type to cast to from string "{s}"'
if throw:
raise Exception(msg)
slog(log_level, msg)
return None
try:
return _str_cast_functions[target_type](s)
except Exception as e:
msg = f'Failed to cast string "{s}" to type {target_type} ({str(e)})'
if throw:
raise Exception(msg)
slog(log_level, msg)
return None
def from_env(key: str, default=None, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None): # export
def from_env( # export
key: str,
default = None,
target_type = None,
default_type = None,
throw = True,
log_level = WARNING,
caller = None
):
val = os.getenv(key)
if val is None:
return default
if target_type is None and default is not None:
target_type = type(default)
return from_str(val, target_type=target_type, default_type=default_type, throw=throw, log_level=log_level, caller=caller)
return from_str(
val,
target_type = target_type,
default_type = default_type,
throw = throw,
log_level = log_level,
caller = caller
)
# deprecated name
def cast_str(s: str, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None):
return from_str(s, target_type=target_type, default_type=None, throw=True, log_level=WARNING, caller=None)
def cast_str(
s: str,
target_type = None,
default_type = None,
throw = True,
log_level = WARNING,
caller = None
):
return from_str(
s,
target_type = target_type,
default_type = None,
throw = True,
log_level = WARNING,
caller = None
)

24
src/python/jw/util/db/DataBase.py
View file

@ -1,15 +1,16 @@
# -*- coding: utf-8 -*-
from typing import Any
from __future__ import annotations
import abc
from contextlib import contextmanager
from ..Config import Config
from .schema.Schema import Schema
from ..Cmds import Cmds
from .Session import Session
from ..log import *
from contextlib import contextmanager
from typing import TYPE_CHECKING
from ..log import NOTICE
if TYPE_CHECKING:
from ..Config import Config
from .schema.Schema import Schema
from .Session import Session
class DataBase(abc.ABC):
@ -37,6 +38,7 @@ class DataBase(abc.ABC):
def session(self):
ret = self._create_session()
try:
yield ret
yield ret
finally:
self._delete_session(ret)
if ret is not None:
self._delete_session(ret)

4
src/python/jw/util/db/Session.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
import abc
class Session(abc.ABC): # export
class Session(abc.ABC): # export
def __init__(self, db):
self.__db = db

39
src/python/jw/util/db/TableIoHandler.py
View file

@ -1,18 +1,17 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Any, List, Union, Optional, Dict
from abc import ABC, abstractmethod
import re, csv, json
from typing import TYPE_CHECKING, Any, Dict, Union
from ..log import *
from ..cast import cast_str
from .schema.Schema import Schema
from ..log import ERR, INFO, OFF, slog, slog_m
from .rows import rows_check_not_null, rows_dump, rows_duplicates
from .rows import *
if TYPE_CHECKING:
from .schema.Schema import Schema
TType = Union[Any, Dict[str, Any]]
class TableIoHandler(ABC): # export
class TableIoHandler(ABC): # export
def __init__(self, schema: Schema):
self.__table_meta = None
@ -22,7 +21,8 @@ class TableIoHandler(ABC): # export
def _table_meta(self):
if self.__table_meta is None:
self.__table_meta = self.__schema.table_by_model_name(
self.__class__.__name__, throw=True)
self.__class__.__name__, throw = True
)
return self.__table_meta
@property
@ -35,24 +35,29 @@ class TableIoHandler(ABC): # export
def _check_non_nullable(self, rows):
buf = []
non_nullable = self.__table_meta.not_null_insertible_columns
non_nullable = self._table_meta.not_null_insertible_columns
try:
rows_check_not_null(rows, non_nullable, buf=buf)
rows_check_not_null(rows, non_nullable, buf = buf)
except:
cn = self.__class__.__name__
tn = self._table_name
d = '========================================================='
slog_m(ERR, f'{d} Null values in {cn}\n')
for key in non_nullable:
buf = rows_check_not_null(rows, key, log_prio=OFF, throw=False)
buf = rows_check_not_null(rows, key, log_prio = OFF, throw = False)
if not buf:
continue
slog_m(ERR, f'\n{d} Null values in {cn} / {tn}: "{key}"\n')
use_cols=self.log_columns
use_cols = list(self.log_columns)
if key not in use_cols:
use_cols.append(key)
rows_dump(buf, use_cols=use_cols, log_prio=ERR)
rows_dump(buf, use_cols=use_cols, out_path=f'/tmp/missing_{key}_in_{tn}.html', heading=f'Missing "{key}" in table {tn}')
rows_dump(buf, use_cols = use_cols, log_prio = ERR)
rows_dump(
buf,
use_cols = use_cols,
out_path = f'/tmp/missing_{key}_in_{tn}.html',
heading = f'Missing "{key}" in table {tn}'
)
raise
@property
@ -67,7 +72,9 @@ class TableIoHandler(ABC): # export
def _store(self, uri: str, data: TType):
pass
def load(self, uri: str, reference, check_duplicates=False, write_csv=None) -> TType:
def load(
self, uri: str, reference, check_duplicates = False, write_csv = None
) -> TType:
slog(INFO, f'Reading table "{self._table_name}" from "{uri}"')
ret = self._load(uri, reference)
if check_duplicates:

24
src/python/jw/util/db/query/Queries.py
View file

@ -1,18 +1,18 @@
# -*- coding: utf-8 -*-
from typing import Any
from __future__ import annotations
from typing import Any, TYPE_CHECKING
import abc
from ...log import *
from ...log import slog, slog_m, ERR, INFO
from ...misc import load_classes
from ...Cmds import Cmds
from ..DataBase import DataBase
from ..schema.Schema import Schema
from .Query import Query as QueryBase
from .QueryResult import QueryResult
class Queries(abc.ABC): # export
if TYPE_CHECKING:
from ..schema.Schema import Schema
from .QueryResult import QueryResult
class Queries(abc.ABC): # export
class Query(QueryBase):
@ -38,7 +38,7 @@ class Queries(abc.ABC): # export
return self.__name
def __init__(self, db: DataBase) -> None:
assert(isinstance(db, DataBase))
assert (isinstance(db, DataBase))
self.__db = db
self.__queries: dict[str, Any] = dict()
@ -57,7 +57,7 @@ class Queries(abc.ABC): # export
def db(self) -> DataBase:
return self.__db
def load(self, modules: list[str], cls=QueryBase):
def load(self, modules: list[str], cls = QueryBase):
for path in modules:
slog(INFO, f'Loading modules from {path}')
for c in load_classes(path, cls):
@ -69,8 +69,8 @@ class Queries(abc.ABC): # export
def add(self, query: QueryBase, query_name: str, location: str, func: Any):
slog(INFO, f'Adding query "{query_name}" on location "{location}"')
assert(isinstance(query_name, str))
assert(isinstance(location, str))
assert (isinstance(query_name, str))
assert (isinstance(location, str))
#ret = self.Query(query, func)
ret = self.Query(query, query_name, location, func)
#setattr(ret, 'name', name)

19
src/python/jw/util/db/query/Query.py
View file

@ -1,18 +1,15 @@
# -*- coding: utf-8 -*-
from typing import Any
from __future__ import annotations
import abc
from ...log import *
from ...misc import load_classes
from ...Cmds import Cmds
from ..DataBase import DataBase
from ..Session import Session
from .QueryResult import QueryResult
#from .Queries import Queries
from typing import TYPE_CHECKING, Any
class Query(abc.ABC): # export
if TYPE_CHECKING:
from ..DataBase import DataBase
from ..Session import Session
from .QueryResult import QueryResult
class Query(abc.ABC): # export
def __init__(self, parent: Any) -> None:
self.__parent = parent

33
src/python/jw/util/db/query/QueryResult.py
View file

@ -1,23 +1,22 @@
# -*- coding: utf-8 -*-
from typing import Any, Union
from __future__ import annotations
import abc
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Union
from ...log import *
from ...Cmds import Cmds
from ..DataBase import DataBase
from ..Session import Session
if TYPE_CHECKING:
from ..DataBase import DataBase
from ..Session import Session
class ResType(Enum): # export
Statement = auto()
Scalars = auto()
One = auto()
First = auto()
Pages = auto()
class ResType(Enum): # export
Statement = auto()
Scalars = auto()
One = auto()
First = auto()
Pages = auto()
class QueryResult(abc.ABC): # export
class QueryResult(abc.ABC): # export
def __init__(self, session: Session, query: Any) -> None:
self.__query = query
@ -42,8 +41,8 @@ class QueryResult(abc.ABC): # export
def rows(self) -> list[Any]:
return self._cast(ResType.Scalars)
def pages(self, per_page=20, page=1) -> Any:
return self._cast(ResType.Pages, per_page=per_page, page=page)
def pages(self, per_page = 20, page = 1) -> Any:
return self._cast(ResType.Pages, per_page = per_page, page = page)
def one(self) -> Any:
return self._cast(ResType.One)
@ -58,5 +57,5 @@ class QueryResult(abc.ABC): # export
# -- pure virtuals
@abc.abstractmethod
def _cast(self, res_type: ResType, **kwargs) -> Union[Any|list[Any]]:
def _cast(self, res_type: ResType, **kwargs) -> Union[Any, list[Any]]:
pass

164
src/python/jw/util/db/rows.py
View file

@ -1,19 +1,24 @@
# -*- coding: utf-8 -*-
import csv
import io
import json
import os
import re
import textwrap
import io, os, re, textwrap, json, csv
from tabulate import tabulate # type: ignore
from tabulate import TableFormat, tabulate # type: ignore
from ..log import *
from ..log import (ERR, INFO, WARNING, get_caller_pos, prio_gets_logged, slog, slog_m)
def rows_pretty(rows): # export
if type(rows) == dict:
def rows_pretty(rows): # export
if isinstance(rows, dict):
rows = [rows]
out = []
for row in rows:
out.append(json.dumps(row, sort_keys=True, indent=4, default=str))
out.append(json.dumps(row, sort_keys = True, indent = 4, default = str))
return '\n'.join(out)
def rows_duplicates(rows, log_prio=INFO, caller=None): # export
def rows_duplicates(rows, log_prio = INFO, caller = None): # export
def __equal(r1, r2):
for col in set(r1.keys()) | set(r2.keys()):
if col in r1:
@ -25,11 +30,12 @@ def rows_duplicates(rows, log_prio=INFO, caller=None): # export
if r1[col] != r2[col]:
return False
return True
ret = []
last = len(rows) - 1
i = last
while last > 0:
for i in reversed(range(0, last-1)):
for i in reversed(range(0, last - 1)):
if __equal(rows[last], rows[i]):
ret.append(last)
last -= 1
@ -37,12 +43,15 @@ def rows_duplicates(rows, log_prio=INFO, caller=None): # export
last -= 1
return ret
def rows_remove(rows, callback=None, candidates=None, log_prio=INFO, caller=None): # export
def rows_remove( # export
rows, callback = None, candidates = None, log_prio = INFO, caller = None
):
def __is_remove_candidate(row):
assert candidates is not None, 'Candidates is None'
for remove_row in candidates:
for col, val in row.items():
if not col in remove_row.keys():
if col not in remove_row.keys():
break
if val != remove_row[col]:
break
@ -65,14 +74,14 @@ def rows_remove(rows, callback=None, candidates=None, log_prio=INFO, caller=None
remove.append(index)
continue
for index in reversed(remove):
slog(log_prio, f'Removing row {rows[index]}', caller=caller)
slog(log_prio, f'Removing row {rows[index]}', caller = caller)
del rows[index]
def rows_select(rows, rules): # export
def rows_select(rows, rules): # export
ret = []
for row in rows:
for rule in rules:
if type(rule) == tuple():
if isinstance(rule, tuple):
search_rule = rule[0]
else:
search_rule = rule
@ -84,7 +93,7 @@ def rows_select(rows, rules): # export
break
return ret
def rows_rewrite_regex(rows, rules): # export
def rows_rewrite_regex(rows, rules): # export
for row in rows:
for rule in rules:
try:
@ -93,14 +102,25 @@ def rows_rewrite_regex(rows, rules): # export
break
else:
for exec_col_name, exec_val in rule[1].items():
slog(INFO, f'Rewriting {row} {row.get(exec_col_name)} -> {exec_val}')
slog(
INFO,
f'Rewriting {row} {row.get(exec_col_name)} -> {exec_val}'
)
row[exec_col_name] = exec_val
except Exception as e:
slog(ERR, f'Failed to run rule {rule} against {row} ({e})')
raise
def rows_check_not_null(rows, keys, log_prio=WARNING, buf=None, stat_key=None, throw=True, caller=None): # export
if type(keys) == str:
def rows_check_not_null( # export
rows,
keys,
log_prio = WARNING,
buf = None,
stat_key = None,
throw = True,
caller = None
):
if isinstance(keys, str):
keys = [keys]
if caller is None:
caller = get_caller_pos()
@ -113,11 +133,11 @@ def rows_check_not_null(rows, keys, log_prio=WARNING, buf=None, stat_key=None, t
for row in rows:
for key in keys:
if row.get(key) is None:
slog(log_prio, f'{key} is missing in row {row}', caller=caller)
slog(log_prio, f'{key} is missing in row {row}', caller = caller)
buf.append(row)
if stat_key is not None:
stat_val = row[stat_key]
if not stat_val in stats.keys():
if stat_val not in stats.keys():
stats[stat_val] = 0
stats[stat_val] += 1
count += 1
@ -125,14 +145,27 @@ def rows_check_not_null(rows, keys, log_prio=WARNING, buf=None, stat_key=None, t
if count > 0:
if stat_key is not None:
i = 0
for k, v in reversed(sorted(stats.items(), key=lambda item: item[1])):
for k, v in reversed(sorted(stats.items(), key = lambda item: item[1])):
i += 1
slog(ERR, f'{i:>3}. {k:<23}: {v}', caller=caller)
slog(ERR, f'{i:>3}. {k:<23}: {v}', caller = caller)
if throw:
raise Exception(f'Found {count} rows violating null-constraint for keys {keys}')
raise Exception(
f'Found {count} rows violating null-constraint for keys {keys}'
)
return buf
def rows_dumps(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None, table_name=None, out_path='log', heading=None, lead=None, tablefmt=None): # export
def rows_dumps( # export
rows,
log_prio = INFO,
caller = None,
use_cols = None,
skip_cols = None,
table_name = None,
out_path = 'log',
heading = None,
lead = None,
tablefmt = None
):
headers = 'keys'
dump_rows = rows
@ -152,20 +185,21 @@ def rows_dumps(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None,
new_row[col] = val
new_dump_rows.append(new_row)
dump_rows = new_dump_rows
out = header = footer = ""
header = footer = ""
match tablefmt:
case 'html':
if heading is not None:
heading = f'<h1>{heading}</h1>\n'
if type(lead) == str:
if isinstance(lead, str):
lead = f'<div class="lead">\n {lead}\n</div>\n'
elif type(lead) == list:
l = '<ul>\n'
elif isinstance(lead, list):
lst = '<ul>\n'
for li in lead:
l += f'<li>{li}</li>\n'
l += '</ul>\n'
lead = l
header=textwrap.dedent('''\
lst += f'<li>{li}</li>\n'
lst += '</ul>\n'
lead = lst
header = textwrap.dedent(
'''\
<html>
<head>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
@ -185,30 +219,47 @@ def rows_dumps(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None,
</style>
</head>
<body>
''')
footer = textwrap.dedent('''
'''
)
footer = textwrap.dedent(
'''
</body>
</html>
''')
'''
)
case _:
if type(heading) == str:
if isinstance(heading, str):
heading = '\n' + heading
if type(lead) == str:
if isinstance(lead, str):
pass
elif type(lead) == list:
l =''
elif isinstance(lead, list):
lst = ''
for li in lead:
l += f' - {li}\n'
lead = '\n\n' + l + '\n'
lst += f' - {li}\n'
lead = '\n\n' + lst + '\n'
if heading is None:
heading = ''
if lead is None:
lead = ''
return header + heading + lead + tabulate(dump_rows, headers=headers, tablefmt=tablefmt) + footer
assert isinstance(tablefmt, str) or isinstance(tablefmt, TableFormat), 'tablefmt'
return header + heading + lead + tabulate(
dump_rows, headers = headers, tablefmt = tablefmt
) + footer
def rows_dump(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None, table_name=None, out_path='log', heading=None, lead=None, tablefmt=None): # export
def rows_dump( # export
rows,
log_prio = INFO,
caller = None,
use_cols = None,
skip_cols = None,
table_name = None,
out_path = 'log',
heading = None,
lead = None,
tablefmt = None
):
if not prio_gets_logged(log_prio):
return
@ -218,18 +269,35 @@ def rows_dump(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None, t
if tablefmt is None and out_path:
tablefmt = os.path.splitext(out_path)[1][1:]
out = rows_dumps(rows, log_prio=log_prio, caller=caller, use_cols=use_cols, skip_cols=skip_cols, table_name=table_name, heading=heading, lead=lead, tablefmt=tablefmt)
out = rows_dumps(
rows,
log_prio = log_prio,
caller = caller,
use_cols = use_cols,
skip_cols = skip_cols,
table_name = table_name,
heading = heading,
lead = lead,
tablefmt = tablefmt
)
match out_path:
case 'log':
slog_m(log_prio, out, caller=caller)
slog_m(log_prio, out, caller = caller)
case _:
with open(out_path, 'w') as fp:
fp.write(out)
def rows_to_csv(rows, use_tmpfile=False): # export
def rows_to_csv(rows, use_tmpfile = False): # export
def __write(rows, out):
writer = csv.DictWriter(out, fieldnames=field_names, delimiter=';', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
writer = csv.DictWriter(
out,
fieldnames = field_names,
delimiter = ';',
quotechar = '"',
quoting = csv.QUOTE_NONNUMERIC
)
writer.writeheader()
for row in rows:
writer.writerow(row)
@ -244,7 +312,7 @@ def rows_to_csv(rows, use_tmpfile=False): # export
__write(rows, out)
return out.getvalue()
import tempfile
with tempfile.TemporaryFile(mode='w', newline='') as out:
with tempfile.TemporaryFile(mode = 'w', newline = '') as out:
__write(rows, out)
out.seek(0)
return out.read()

35
src/python/jw/util/db/schema/Column.py
View file

@ -1,17 +1,20 @@
# -*- coding: utf-8 -*-
from typing import Optional, Any
from __future__ import annotations
import abc
from .DataType import DataType
from ...log import *
from typing import TYPE_CHECKING, Any, Optional
class Column(abc.ABC): # export
from ...log import ERR, throw
def __init__(self, table, name, data_type: DataType):
if TYPE_CHECKING:
from .DataType import DataType
from .Table import Table
class Column(abc.ABC): # export
def __init__(self, table: Table, name: str, data_type: DataType) -> None:
self.__name: str = name
self.__table: Any = table
self.__table: Table = table
self.__is_nullable: Optional[bool] = None
self.__is_null_insertible: Optional[bool] = None
self.__is_primary_key: Optional[bool] = None
@ -39,18 +42,18 @@ class Column(abc.ABC): # export
return False
return True
throw(ERR, f'Tried to compare column {self} to type {type(rhs)}: {rhs}')
return False # Unreachable but requested by mypy
return False # Unreachable but requested by mypy
@property
def name(self) -> str:
return self.__name
@property
def data_type(self):
def data_type(self) -> DataType:
return self.__data_type
@property
def table(self) -> str:
def table(self) -> Table:
return self.__table
@property
@ -60,7 +63,7 @@ class Column(abc.ABC): # export
return self.__is_nullable
@property
def is_null_insertible(self):
def is_null_insertible(self) -> bool:
if self.__is_null_insertible is None:
ret = False
if self.is_nullable:
@ -81,7 +84,9 @@ class Column(abc.ABC): # export
@property
def is_auto_increment(self) -> bool:
if self.__is_auto_increment is None:
self.__is_auto_increment = self.__name in self.__table.auto_increment_columns
self.__is_auto_increment = (
self.__name in self.__table.auto_increment_columns
)
return self.__is_auto_increment
@property
@ -113,8 +118,8 @@ class Column(abc.ABC): # export
def foreign_key(self, table) -> Optional[Any]:
if self.__foreign_keys_by_table is None:
self.__foreign_keys_by_table = dict()
for col in self.foreign_keys: # type: ignore # Any not iterable
assert(col.table.name not in self.__foreign_keys_by_table)
for col in self.foreign_keys: # type: ignore # Any not iterable
assert (col.table.name not in self.__foreign_keys_by_table)
self.__foreign_keys_by_table[col.table.name] = col
table_name = table if isinstance(table, str) else table.name
return self.__foreign_keys_by_table.get(table_name)

18
src/python/jw/util/db/schema/ColumnSet.py
View file

@ -1,20 +1,24 @@
# -*- coding: utf-8 -*-
from typing import Optional, Any
from typing import Optional, Iterable, Any
class ColumnSet: # export
class ColumnSet: # export
def __init__(self, *args: list[Any], columns: list[Any]=[], table: Optional[Any]=None, names: Optional[list[str]]=None):
def __init__(
self,
*args: list[Any],
columns: list[Any] = [],
table: Optional[Any] = None,
names: Optional[list[str]] = None
):
self.__columns: list[Any] = [*args]
self.__columns.extend(columns)
self.__table = table
if names is not None:
assert(table is not None)
assert (table is not None)
for name in names:
self.__columns.append(table.column(name))
if self.__table is not None:
for col in columns:
assert(col.table == self.__table)
assert (col.table == self.__table)
def __len__(self):
return len(self.__columns)

48
src/python/jw/util/db/schema/CompositeForeignKey.py
View file

@ -1,15 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Optional, Any
from typing import TYPE_CHECKING, Any, Optional
from ...log import *
from .ColumnSet import ColumnSet
from ...log import WARNING, slog
from .SingleForeignKey import SingleForeignKey
class CompositeForeignKey: # export
if TYPE_CHECKING:
from .ColumnSet import ColumnSet
def __init__(self, child_col_set: ColumnSet, parent_col_set: ColumnSet): # TODO: Implement alternative ways to construct
class CompositeForeignKey: # export
def __init__(
self, child_col_set: ColumnSet, parent_col_set: ColumnSet
): # TODO: Implement alternative ways to construct
def __table(s):
ret = None
@ -17,8 +20,8 @@ class CompositeForeignKey: # export
if ret is None:
ret = c.table
else:
assert(ret == c.table)
assert(ret is not None)
assert (ret == c.table)
assert (ret is not None)
return ret
self.__child_col_set = child_col_set
@ -26,7 +29,7 @@ class CompositeForeignKey: # export
self.__child_table = __table(child_col_set)
self.__parent_table = __table(parent_col_set)
assert(len(self.__child_col_set) == len(self.__parent_col_set))
assert (len(self.__child_col_set) == len(self.__parent_col_set))
self.__len = len(self.__child_col_set)
self.__column_relations: Optional[list[SingleForeignKey]] = None
self.__parent_columns_by_child_column: Optional[dict[str, Any]] = None
@ -46,7 +49,12 @@ class CompositeForeignKey: # export
def __repr__(self):
ret = self.__table_rel_str()
ret += ': ' + ', '.join([self.__cols_rel_str(rel.child_column, rel.parent_column) for rel in self.column_relations])
ret += ': ' + ', '.join(
[
self.__cols_rel_str(rel.child_column, rel.parent_column)
for rel in self.column_relations
]
)
return ret
def __eq__(self, rhs):
@ -73,21 +81,25 @@ class CompositeForeignKey: # export
return self.__parent_col_set
def parent_column(self, child_column) -> Any:
child_column_name = child_column if isinstance(child_column, str) else child_column.name
child_column if isinstance(child_column, str) else child_column.name
if self.__parent_columns_by_child_column is None:
d: dict[str, Any] = {}
assert(len(self.__child_col_set) == len(self.__parent_col_set))
assert (len(self.__child_col_set) == len(self.__parent_col_set))
for i in range(0, len(self.__child_col_set)):
d[self.__child_col_set[i].name] = self.__parent_col_set[i]
self.__parent_columns_by_child_column = d
return self.__parent_columns_by_child_column[child_column]
def child_column(self, parent_column) -> Any:
slog(WARNING, f'{self}: Looking for child column belonging to parent column "{parent_column}"')
parent_column_name = parent_column if isinstance(parent_column, str) else parent_column.name
slog(
WARNING,
f'{self}: Looking for child column belonging to parent column '
f'"{parent_column}"'
)
parent_column if isinstance(parent_column, str) else parent_column.name
if self.__child_columns_by_parent_column is None:
d: dict[str, Any] = {}
assert(len(self.__parent_col_set) == len(self.__child_col_set))
assert (len(self.__parent_col_set) == len(self.__child_col_set))
for i in range(0, len(self.__parent_col_set)):
d[self.__parent_col_set[i].name] = self.__child_col_set[i]
self.__child_columns_by_parent_column = d
@ -98,6 +110,8 @@ class CompositeForeignKey: # export
ret = []
if self.__column_relations is None:
for i in range(0, self.__len):
ret.append(SingleForeignKey(self.__child_col_set[i], self.__parent_col_set[i]))
ret.append(
SingleForeignKey(self.__child_col_set[i], self.__parent_col_set[i])
)
self.__column_relations = ret
return self.__column_relations

25
src/python/jw/util/db/schema/DataType.py
View file

@ -1,10 +1,8 @@
# -*- coding: utf-8 -*-
from typing import Optional
from enum import Enum, auto
from datetime import datetime
from enum import Enum, auto
from typing import Optional
from ...log import *
from ...log import ERR, throw
class Id(Enum):
Integer = auto()
@ -16,7 +14,7 @@ class Id(Enum):
Text = auto()
Invalid = auto()
def py_type(type_id: Id) -> type: # export
def py_type(type_id: Id) -> type: # export
match type_id:
case Id.Integer:
@ -38,14 +36,17 @@ def py_type(type_id: Id) -> type: # export
raise Exception(f'Unknown column type-id "{type_id}"')
class DataType: # export
class DataType: # export
def __init__(self, type_id: Id, size: Optional[int]=None):
def __init__(self, type_id: Id, size: Optional[int] = None):
if not isinstance(type_id, Id):
throw(ERR, f'Passed type id "{type_id}" with unsupported data type {type(type_id)}')
throw(
ERR,
f'Passed type id "{type_id}" with unsupported data type {type(type_id)}'
)
if size is not None:
assert(isinstance(size, int))
assert(size > 0)
assert (isinstance(size, int))
assert (size > 0)
self.__id = type_id
self.__size = size
@ -80,4 +81,4 @@ class DataType: # export
@property
def py_type_annotation(self) -> str:
return self.py_type_str # FIXME: This is not always correct
return self.py_type_str # FIXME: This is not always correct

34
src/python/jw/util/db/schema/Schema.py
View file

@ -1,20 +1,20 @@
# -*- coding: utf-8 -*-
from typing import Optional, Iterable
from __future__ import annotations
import abc
from ...log import *
from typing import TYPE_CHECKING, Iterable, Optional
from .Table import Table
from .Column import Column
from .DataType import DataType
from .CompositeForeignKey import CompositeForeignKey
from ...log import DEBUG, ERR, slog, throw
class Schema(abc.ABC): # export
if TYPE_CHECKING:
from .Column import Column
from .CompositeForeignKey import CompositeForeignKey
from .Table import Table
class Schema(abc.ABC): # export
def __init__(self) -> None:
self.___tables: Optional[list[Table]] = None
self.___tables: Optional[dict[str, Table]] = None
self.__foreign_keys: Optional[list[CompositeForeignKey]] = None
self.__access_defining_columns: Optional[list[str]] = None
@ -24,7 +24,7 @@ class Schema(abc.ABC): # export
ret = dict()
for name in self._table_names():
slog(DEBUG, f'Caching metadata for table "{name}"')
assert(isinstance(name, str))
assert (isinstance(name, str))
ret[name] = self._table(name)
self.___tables = ret
return self.___tables
@ -39,7 +39,7 @@ class Schema(abc.ABC): # export
@abc.abstractmethod
def _table(self, name: str) -> Table:
throw(ERR, "Called pure virtual base class method")
return None # type: ignore
return None # type: ignore
@abc.abstractmethod
def _foreign_keys(self) -> list[CompositeForeignKey]:
@ -50,7 +50,7 @@ class Schema(abc.ABC): # export
pass
@abc.abstractmethod
def _model_module_search_paths(self) -> list[tuple[str, type]]:
def _model_module_search_paths(self) -> list[tuple[str, type]]:
pass
# ------ API to be called
@ -62,7 +62,7 @@ class Schema(abc.ABC): # export
yield from self.__tables.values()
def __repr__(self):
return '|'.join([table.name for table in self.__tables])
return '|'.join([table.name for table in self.__tables.values()])
def __getitem__(self, index):
return self.__tables[index]
@ -90,13 +90,13 @@ class Schema(abc.ABC): # export
def table(self, name: str) -> Table:
return self.__tables[name]
def table_by_model_name(self, name: str, throw=False) -> Table:
def table_by_model_name(self, name: str, throw = False) -> Table:
for table in self.__tables.values():
if table.model_name == name:
return table
if throw:
raise Exception(f'Table "{name}" not found in database metadata')
return None # type: ignore
return None # type: ignore
def primary_keys(self, table_name: str) -> Iterable[str]:
return self.__tables[table_name].primary_keys
@ -105,5 +105,5 @@ class Schema(abc.ABC): # export
return self.__tables[table_name].columns
@property
def model_module_search_paths(self) -> list[tuple[str, type]]:
def model_module_search_paths(self) -> list[tuple[str, type]]:
return self._model_module_search_paths()

13
src/python/jw/util/db/schema/SingleForeignKey.py
View file

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Optional, Any
from typing import TYPE_CHECKING
from .Column import Column
from .ColumnSet import ColumnSet
if TYPE_CHECKING:
from .Column import Column
class SingleForeignKey:
@ -19,7 +19,10 @@ class SingleForeignKey:
yield from self.__iterable
def __repr__(self):
return f'{self.__child_col.table.name}.{self.__child_col.name} -> {self.__parent_col.table.name}.{self.__parent_col.name}'
return (
f'{self.__child_col.table.name}.{self.__child_col.name} -> '
'{self.__parent_col.table.name}.{self.__parent_col.name}'
)
def __getitem__(self, index):
return self.__iterable[index]

117
src/python/jw/util/db/schema/Table.py
View file

@ -1,47 +1,51 @@
# -*- coding: utf-8 -*-
from typing import Optional, Union, Iterable, Self, Any # TODO: Need any for many things, as I can't figure out how to avoid circular imports from here
from __future__ import annotations
import abc
import re
from collections import OrderedDict
from typing import TYPE_CHECKING
from urllib.parse import quote_plus
from ...log import *
from ...log import ERR, WARNING, slog, throw
from ...misc import load_class
from .ColumnSet import ColumnSet
from .DataType import DataType
from .CompositeForeignKey import CompositeForeignKey
from .Column import Column
from .ColumnSet import ColumnSet
class Table(abc.ABC): # export
if TYPE_CHECKING:
from typing import Any, Iterable, Optional, Self, Union
from .CompositeForeignKey import CompositeForeignKey
from .DataType import DataType
class Table(abc.ABC): # export
def __init__(self, schema, name: str):
assert(isinstance(name, str))
assert (isinstance(name, str))
self.__schema = schema
self.__name = name
self.___columns: Optional[OrderedDict[str, Any]] = None
self.___foreign_key_parent_tables: Optional[OrderedDict[str, Any]] = None
self.___foreign_key_parent_tables: Optional[OrderedDict[str, Any]] = None
self.__primary_keys: Optional[Iterable[str]] = None
self.__unique_constraints: Optional[list[ColumnSet]] = None
self.__foreign_key_constraints: Optional[list[CompositeForeignKey]] = None
self.__nullable_columns: Optional[Iterable[str]] = None
self.__non_nullable_columns: Optional[Iterable[str]] = None
self.__null_insertible_columns: Optional[Iterable[str]] = None
self.__not_null_insertible_columns: Optional[Iterable[str]] = None
self.__log_columns: Optional[Iterable[str]] = None
self.__edit_columns: Optional[Iterable[str]] = None
self.__translate_columns: Optional[Iterable[str]] = None
self.__display_columns: Optional[Iterable[str]] = None
self.__default_sort_columns: Optional[Iterable[str]] = None
self.__column_default: Optional[dict[str, Any]] = None
self.__base_location_rule: Optional[Iterable[str]] = None
self.__location_rule: Optional[Iterable[str]] = None
self.__row_location_rule: Optional[Iterable[str]] = None
self.__add_row_location_rule: Optional[Iterable[str]] = None
self.___add_child_row_location_rules: Optional[dict[str, str]] = None
self.__nullable_columns: Optional[Iterable[str]] = None
self.__non_nullable_columns: Optional[Iterable[str]] = None
self.__null_insertible_columns: Optional[Iterable[str]] = None
self.__not_null_insertible_columns: Optional[Iterable[str]] = None
self.__log_columns: Optional[Iterable[str]] = None
self.__edit_columns: Optional[Iterable[str]] = None
self.__translate_columns: Optional[Iterable[str]] = None
self.__display_columns: Optional[Iterable[str]] = None
self.__default_sort_columns: Optional[Iterable[str]] = None
self.__column_default: Optional[dict[str, Any]] = None
self.__base_location_rule: Optional[Iterable[str]] = None
self.__location_rule: Optional[Iterable[str]] = None
self.__row_location_rule: Optional[Iterable[str]] = None
self.__add_row_location_rule: Optional[Iterable[str]] = None
self.___add_child_row_location_rules: Optional[dict[str, str]] = None
self.__foreign_keys_to_parent_table: Optional[OrderedDict[str, Any]] = None
self.__relationships: Optional[list[tuple[str, Self]]] = None
self.__model_class: Optional[Any] = None
@ -61,7 +65,8 @@ class Table(abc.ABC): # export
if self.___foreign_key_parent_tables is None:
self.___foreign_key_parent_tables = OrderedDict()
for cfk in self.foreign_key_constraints:
self.___foreign_key_parent_tables[cfk.parent_table.name] = cfk.parent_table
self.___foreign_key_parent_tables[cfk.parent_table.name
] = cfk.parent_table
return self.___foreign_key_parent_tables
@property
@ -77,12 +82,12 @@ class Table(abc.ABC): # export
def __add_child_row_location_rules(self) -> dict[str, str]:
if self.___add_child_row_location_rules is None:
ret: dict[str, str] = {}
for foreign_table_name, foreign_table in self.__relationship_by_foreign_table.items():
if len([self.foreign_keys_to_parent_table(foreign_table)]):
rule = self._add_child_row_location_rule(foreign_table_name)
for table_name, table in self.__relationship_by_foreign_table.items():
if len([self.foreign_keys_to_parent_table(table)]):
rule = self._add_child_row_location_rule(table_name)
if rule is None:
continue
ret[foreign_table_name] = rule
ret[table_name] = rule
self.___add_child_row_location_rules = ret
return self.___add_child_row_location_rules
@ -108,7 +113,7 @@ class Table(abc.ABC): # export
return False
return True
throw(ERR, f'Tried to compare table {self} to type {type(rhs)}: {rhs}')
return False # Unreachable but requested by mypy
return False # Unreachable but requested by mypy
def __hash__(self) -> int:
return hash(self.name)
@ -167,8 +172,9 @@ class Table(abc.ABC): # export
slog(WARNING, f'Returning None model name for table {self.name}')
return None
def _model_module_search_paths(self) -> list[tuple[str, type]]:
return self.schema.model_module_search_paths # Fall back to Schema-global default
def _model_module_search_paths(self) -> list[tuple[str, type]]:
# Fall back to Schema-global default
return self.schema.model_module_search_paths
@abc.abstractmethod
def _query_name(self) -> str:
@ -190,7 +196,9 @@ class Table(abc.ABC): # export
for col in self.__schema.access_defining_columns:
if col in self.primary_keys:
ret += f'/<{col}>'
ret += self.base_location_rule
base = self.base_location_rule
if base is not None:
ret += base if isinstance(base, str) else 'what-goes-here?'.join(base)
return ret
def _row_location_rule(self) -> Optional[str]:
@ -261,7 +269,7 @@ class Table(abc.ABC): # export
return None
pattern = r'^' + model_name + '$'
for module_path, base_class in self._model_module_search_paths():
ret = load_class(module_path, base_class, class_name_filter=pattern)
ret = load_class(module_path, base_class, class_name_filter = pattern)
if ret is not None:
self.__model_class = ret
break
@ -288,8 +296,8 @@ class Table(abc.ABC): # export
return self.__location_rule
def location(self, *args, **kwargs):
ret = self.location_rule
for token, val in kwargs.items(): # FIXME: Poor man's row location assembly
ret = str(self.location_rule)
for token, val in kwargs.items(): # FIXME: Poor man's row location assembly
ret = re.sub(f'<{token}>', quote_plus(quote_plus(str(val))), ret)
return ret
@ -300,9 +308,9 @@ class Table(abc.ABC): # export
return self.__row_location_rule
def row_location(self, *args, **kwargs):
ret = self.row_location_rule
ret = str(self.row_location_rule)
for col in self.primary_keys:
if col in kwargs: # FIXME: Poor man's row location assembly
if col in kwargs: # FIXME: Poor man's row location assembly
ret = re.sub(f'<{col}>', quote_plus(quote_plus(str(kwargs[col]))), ret)
return ret
@ -313,9 +321,9 @@ class Table(abc.ABC): # export
return self.__add_row_location_rule
def add_row_location(self, *args, **kwargs) -> Optional[str]:
ret = self.add_row_location_rule
ret = str(self.add_row_location_rule)
for col in self.primary_keys:
if col in kwargs: # FIXME: Poor man's row location assembly
if col in kwargs: # FIXME: Poor man's row location assembly
ret = re.sub(f'<{col}>', quote_plus(quote_plus(str(kwargs[col]))), ret)
return ret
@ -323,12 +331,14 @@ class Table(abc.ABC): # export
def add_child_row_location_rules(self) -> Iterable[str]:
return self.__add_child_row_location_rules.values()
def add_child_row_location_rule(self, child_table: Union[Self, str]) -> Optional[str]:
def add_child_row_location_rule(self, child_table: Union[Self,
str]) -> Optional[str]:
if isinstance(child_table, Table):
child_table = child_table.name
return self.__add_child_row_location_rules.get(child_table)
def add_child_row_location(self, parent_table: Union[Self, str], **kwargs) -> Optional[str]:
def add_child_row_location(self, parent_table: Union[Self, str],
**kwargs) -> Optional[str]:
ret = self.add_child_row_location_rule(parent_table)
if isinstance(parent_table, str):
parent_table = self.schema[parent_table]
@ -337,7 +347,11 @@ class Table(abc.ABC): # export
for cfk in self.foreign_keys_to_parent_table(parent_table):
for fk in cfk:
if fk.parent_column.name in kwargs:
ret = re.sub(f'<{fk.child_column.name}>', quote_plus(quote_plus(str(kwargs[fk.parent_column.name]))), ret)
ret = re.sub(
f'<{fk.child_column.name}>',
quote_plus(quote_plus(str(kwargs[fk.parent_column.name]))),
ret
)
return ret
@property
@ -425,7 +439,7 @@ class Table(abc.ABC): # export
impl = self._unique_constraints()
if impl is not None:
for columns in impl:
ret.append(ColumnSet(columns=columns))
ret.append(ColumnSet(columns = columns))
self.__unique_constraints = ret
return self.__unique_constraints
@ -443,7 +457,8 @@ class Table(abc.ABC): # export
def foreign_key_parent_tables(self):
return self.__foreign_key_parent_tables.values()
def foreign_keys_to_parent_table(self, parent_table) -> Iterable[CompositeForeignKey]:
def foreign_keys_to_parent_table(self,
parent_table) -> Iterable[CompositeForeignKey]:
if self.__foreign_keys_to_parent_table is None:
self.__foreign_keys_to_parent_table = OrderedDict()
for cfk in self.foreign_key_constraints:
@ -451,8 +466,12 @@ class Table(abc.ABC): # export
if pt not in self.__foreign_keys_to_parent_table:
self.__foreign_keys_to_parent_table[pt] = []
self.__foreign_keys_to_parent_table[pt].append(cfk)
parent_table_name = parent_table if isinstance(parent_table, str) else parent_table.name
return self.__foreign_keys_to_parent_table[parent_table_name] if parent_table_name in self.__foreign_keys_to_parent_table else []
parent_table_name = parent_table if isinstance(
parent_table, str
) else parent_table.name
return self.__foreign_keys_to_parent_table[
parent_table_name
] if parent_table_name in self.__foreign_keys_to_parent_table else []
@property
def relationships(self) -> list[tuple[str, Self]]:

16
src/python/jw/util/db/schema/utils.py
View file

@ -1,12 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from .Schema import Schema
from typing import TYPE_CHECKING
from ...log import *
from ...log import NOTICE, slog
def check_schema(schema: Schema): # export
if TYPE_CHECKING:
from .Schema import Schema
def check_schema(schema: Schema): # export
slog(NOTICE, f'There are {len(schema)} tables in the database')
for cfk in schema.foreign_key_constraints:
for fk in cfk:
if fk.child_column.data_type != fk.parent_column.data_type:
raise Exception(f'Type mismatch in foreign key {fk}: {fk.child_column.data_type} != {fk.parent_column.data_type}')
raise Exception(
f'Type mismatch in foreign key {fk}: {fk.child_column.data_type} '
f'!= {fk.parent_column.data_type}'
)

173
src/python/jw/util/graph/yed/MapAttr2Shape.py
View file

@ -1,29 +1,35 @@
# -*- coding: utf-8 -*-
from collections.abc import Callable
from __future__ import annotations
import xml.etree.ElementTree as ET
from ...log import *
from typing import TYPE_CHECKING
class MapAttr2Shape: # export
if TYPE_CHECKING:
from collections.abc import Callable
from typing import Any
def __init__(self, mappings: dict[str, str|Callable[[dict[str, str]], str]]|None=None) -> None:
class MapAttr2Shape: # export
def __init__(
self,
mappings: dict[str, str | Callable[[dict[str, str]], str]] | None = None
) -> None:
self.__mappings = mappings if mappings is not None else {}
self.__shape_node_key = 'd25'
self.__ns_gml = "http://graphml.graphdrawing.org/xmlns"
self.__ns = {
# -- Standard GraphML
"": self.__ns_gml,
"xsi": "http://www.w3.org/2001/XMLSchema-instance",
"xsi:schemaLocation": "http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd",
"": self.__ns_gml,
"xsi": "http://www.w3.org/2001/XMLSchema-instance",
"xsi:schemaLocation":
"http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd",
# -- YWorks GraphML
"java": "http://www.yworks.com/xml/yfiles-common/1.0/java",
"sys": "http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0",
"x": "http://www.yworks.com/xml/yfiles-common/markup/2.0",
"y": "http://www.yworks.com/xml/graphml",
"yed": "http://www.yworks.com/xml/yed/3",
"java": "http://www.yworks.com/xml/yfiles-common/1.0/java",
"sys": "http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0",
"x": "http://www.yworks.com/xml/yfiles-common/markup/2.0",
"y": "http://www.yworks.com/xml/graphml",
"yed": "http://www.yworks.com/xml/yed/3",
}
# https://stackoverflow.com/questions/4997848/
for name, url in self.__ns.items():
@ -72,7 +78,7 @@ class MapAttr2Shape: # export
ns, tag = tag.split(':')
tag = '{' + self.__ns[ns] + '}' + tag
attrib = content.get('a') or {}
el = ET.Element(tag, attrib=attrib)
el = ET.Element(tag, attrib = attrib)
text = content.get('t')
if text is not None:
el.text = text
@ -81,10 +87,7 @@ class MapAttr2Shape: # export
if children is not None:
__add(el, children)
default_values = {
'color': '#FFCC00',
'text': ''
}
default_values = {'color': '#FFCC00', 'text': ''}
values = {}
for key, default in default_values.items():
@ -98,11 +101,11 @@ class MapAttr2Shape: # export
continue
mapped = mapping(self.__attribs(node, keys))
values[key] = mapped or default
except:
except Exception:
pass
color = values['color']
text = values['text']
text = values['text']
has_text = 'true' if text else 'false'
width_text = round(len(text) * 5.75, 5) if text else 0
@ -110,61 +113,89 @@ class MapAttr2Shape: # export
shape = {
'data': {
'a': {'key': self.__shape_node_key},
'a': {
'key': self.__shape_node_key
},
'c': {
'y:ShapeNode': {
'a': {},
'c': {
'y:Geometry': {'a': {'height': '30.0', 'width': str(width_box), 'x': str(-(width_box / 2)), 'y':' -15.0'}},
'y:Fill': {'a': {'color': color, 'transparent': 'false'}},
'y:BorderStyle': {'a': {'color': '#000000', 'raised': 'false', 'type': 'line', 'width': '1.0'}},
'y:Geometry': {
'a': {
'height': '30.0',
'width': str(width_box),
'x': str(-(width_box / 2)),
'y': ' -15.0'
}
},
'y:Fill': {
'a': {
'color': color, 'transparent': 'false'
}
},
'y:BorderStyle': {
'a': {
'color': '#000000',
'raised': 'false',
'type': 'line',
'width': '1.0'
}
},
'y:NodeLabel': {
'a': {
'alignment': 'center',
'autoSizePolicy': 'content',
'fontFamily': 'Dialog',
'fontSize': '12',
'fontStyle': 'plain',
'hasBackgroundColor': 'false',
'hasLineColor': 'false',
'hasText': has_text,
'height': '18',
'horizontalTextPosition': 'center',
'iconTextGap': '4',
'modelName': 'custom',
'textColor': '#000000',
'verticalTextPosition': 'bottom',
'visible': 'true',
'width': str(width_text),
'x': '13.0',
'y': '13.0',
'alignment': 'center',
'autoSizePolicy': 'content',
'fontFamily': 'Dialog',
'fontSize': '12',
'fontStyle': 'plain',
'hasBackgroundColor': 'false',
'hasLineColor': 'false',
'hasText': has_text,
'height': '18',
'horizontalTextPosition': 'center',
'iconTextGap': '4',
'modelName': 'custom',
'textColor': '#000000',
'verticalTextPosition': 'bottom',
'visible': 'true',
'width': str(width_text),
'x': '13.0',
'y': '13.0',
},
'c': {
'y:LabelModel': {
'c': {
'y:SmartNodeLabelModel': {'a': {'distance': '4.0'}}
},
},
'y:ModelParameter': {
'c': {
'y:SmartNodeLabelModelParameter': {
'a': {
'labelRatioX':'0.0',
'labelRatioY': '0.0',
'nodeRatioX': '0.0',
'nodeRatioY': '0.0',
'offsetX': '0.0',
'offsetY': '0.0',
'upX': '0.0',
'upY': '-1.0',
}
}
}
}
'y:LabelModel': {
'c': {
'y:SmartNodeLabelModel': {
'a': {
'distance': '4.0'
}
}
},
},
'y:ModelParameter': {
'c': {
'y:SmartNodeLabelModelParameter': {
'a': {
'labelRatioX': '0.0',
'labelRatioY': '0.0',
'nodeRatioX': '0.0',
'nodeRatioY': '0.0',
'offsetX': '0.0',
'offsetY': '0.0',
'upX': '0.0',
'upY': '-1.0',
}
}
}
}
},
't': text
},
'y:Shape': {'a': {'type': 'rectangle'}}
},
'y:Shape': {
'a': {
'type': 'rectangle'
}
}
}
}
}
@ -175,17 +206,17 @@ class MapAttr2Shape: # export
def __massage_nodes(self, root) -> None:
keys = self.__keys(root)
graph = root.find(f'graph', self.__ns)
graph = root.find('graph', self.__ns)
for node in graph:
self.__massage_node(node, keys)
def run(self, path_in: str, path_out: str) -> None:
parser = ET.XMLParser(encoding="utf-8")
tree = ET.parse(path_in, parser=parser)
parser = ET.XMLParser(encoding = "utf-8")
tree = ET.parse(path_in, parser = parser)
root = tree.getroot()
self.__add_key_nodegraphics(root)
self.__massage_nodes(root)
ET.indent(tree, space=' ', level=0)
tree.write(path_out, xml_declaration=True, encoding='utf-8')
ET.indent(tree, space = ' ', level = 0)
tree.write(path_out, xml_declaration = True, encoding = 'utf-8')

280
src/python/jw/util/ldap.py
View file

@ -1,36 +1,47 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import copy
import getpass
import pathlib
import ldap, getpass, pathlib, copy
from ldap.schema.models import ObjectClass
from enum import Flag, auto
import networkx as nx
from typing import Any, Self
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Self
from .Config import Config as BaseConfig
from .log import *
import ldap # type: ignore[import-untyped]
import networkx as nx # type: ignore[import-untyped]
from ldap.schema.models import ObjectClass # type: ignore[import-untyped]
from .log import ERR, INFO, WARNING, slog
if TYPE_CHECKING:
from collections.abc import Callable
from .Config import Config as BaseConfig
class Config:
def __init__(self, external: BaseConfig|None=None):
def __init__(self, external: BaseConfig | None = None):
self.__external = external
for attr in ['ldap_uri', 'bind_dn', 'bind_pw', 'base_dn']:
setattr(self, '_Config__' + attr, None)
def __get(self, key: str, default: str):
def __get(self, key: str, default: str | None):
if not self.__external:
return default
return self.__external.value(key, default=default)
return self.__external.value(key, default = default)
@property
def ldap_uri(self):
if self.__ldap_uri is None:
for key in ['ldap_uri', 'uri']:
self.__ldap_uri = self.__get(key, default=None)
self.__ldap_uri = self.__get(key, default = None)
if self.__ldap_uri is not None:
break
else:
self.__ldap_uri = 'ldap://ldap.janware.com'
return self.__ldap_uri
@ldap_uri.setter
def ldap_uri(self, rhs):
self.__ldap_uri = rhs
@ -38,8 +49,12 @@ class Config:
@property
def bind_dn(self):
if self.__bind_dn is None:
self.__bind_dn = self.__get('bind_dn', default=f'uid={getpass.getuser()},ou=users,dc=jannet,dc=de')
self.__bind_dn = self.__get(
'bind_dn',
default = f'uid={getpass.getuser()},ou=users,dc=jannet,dc=de'
)
return self.__bind_dn
@bind_dn.setter
def bind_dn(self, rhs):
self.__bind_dn = rhs
@ -48,17 +63,21 @@ class Config:
def bind_pw(self):
if self.__bind_pw is None:
for key in ['bind_pw', 'password']:
ret = self.__get(key, default=None)
ret = self.__get(key, default = None)
if ret is not None:
break
if ret is None:
ldap_secret_file = self.__get('secret_file', f'{pathlib.Path.home()}/.ldap.secret')
ldap_secret_file = self.__get(
'secret_file', f'{pathlib.Path.home()}/.ldap.secret'
)
assert ldap_secret_file is not None, 'ldap_secret_file'
with open(ldap_secret_file, 'r') as file:
ret = file.read()
file.closed
ret = ret.strip()
self.__bind_pw = ret
return self.__bind_pw
@bind_pw.setter
def bind_pw(self, rhs):
self.__bind_pw = rhs
@ -66,25 +85,28 @@ class Config:
@property
def base_dn(self):
if self.__base_dn is None:
self.__base_dn = self.__get('base_dn', default=f'dc=jannet,dc=de')
self.__base_dn = self.__get('base_dn', default = 'dc=jannet,dc=de')
return self.__base_dn
@base_dn.setter
def base_dn(self, rhs):
self.__base_dn = rhs
class Connection: # export
class Connection: # export
class AttrType(Flag):
Must = auto()
May = auto()
May = auto()
def __init__(self, conf: Config|BaseConfig|None=None, backtrace=False):
uri: str|None = None
def __init__(self, conf: Config | BaseConfig | None = None, backtrace = False):
uri: str | None = None
c = conf if isinstance(conf, Config) else Config(conf)
try:
uri = c.ldap_uri
except:
uri = c.uri
except Exception:
# mypy says: E: "Config" has no attribute "uri" [attr-defined]
# FIXME: Who adds .uri?
uri = c.uri # type: ignore
try:
ret = ldap.initialize(uri)
ret.start_tls_s()
@ -92,46 +114,60 @@ class Connection: # export
slog(ERR, f'Failed to initialize LDAP connection to "{uri}" ({str(e)})')
raise
try:
rr = ret.bind_s(c.bind_dn, c.bind_pw) # method)
ret.bind_s(c.bind_dn, c.bind_pw) # method)
except Exception as e:
slog(ERR, f'Failed to bind to "{uri}" with dn "{c.bind_dn}" ({str(e)})')
raise
self.__ldap = ret
self.__backtrace = backtrace
self.__object_classes_by_oid: dict[str, ObjectClass]|None = None
self.__object_class_tree: nx.Graph|None = None
self.__object_classes_by_name: dict[str, ObjectClass]|None = None
self.__object_classes_by_oid: dict[str, ObjectClass] | None = None
self.__object_class_tree: nx.Graph | None = None
self.__object_classes_by_name: dict[str, ObjectClass] | None = None
@property
def ldap(self):
return self.__ldap
def add(self, attrs: dict[str, bytes], dn: str|None=None):
def add(self, attrs: dict[str, bytes], dn: str | None = None):
if dn is None:
if not 'dn' in attrs:
if 'dn' not in attrs:
raise Exception('No DN to add an LDAP entry to')
attrs = copy.deepcopy(attrs)
del attrs['dn']
try:
slog(INFO, f'LDAP: Add [{dn}] -> {attrs}')
self.__ldap.add_s(dn, ldap.modlist.addModlist(attrs))
ml =ldap.modlist.addModlist( # pyright: ignore[reportAttributeAccessIssue]
attrs
)
self.__ldap.add_s(dn, ml)
except Exception as e:
slog(ERR, f'{dn}: Failed to add entry {attrs} ({e})')
raise
def delete(self, dn: str, recursive=False, force_existence: bool=False):
def delete(self, dn: str, recursive = False, force_existence: bool = False):
def __walk_cb_delete(conn: Connection, entry, context):
self.walk(__walk_cb_delete, base=entry[0], scope=ldap.SCOPE_ONELEVEL, context=context)
self.walk(
__walk_cb_delete,
base = entry[0],
scope = ldap.
SCOPE_ONELEVEL, # pyright: ignore[reportAttributeAccessIssue]
context = context
)
self.__ldap.delete_s(entry[0])
try:
if recursive:
self.walk(__walk_cb_delete, dn, scope=ldap.SCOPE_ONELEVEL)
self.walk(
__walk_cb_delete,
dn,
scope = ldap.
SCOPE_ONELEVEL # pyright: ignore[reportAttributeAccessIssue]
)
self.__ldap.delete_s(dn)
else:
self.__ldap.delete_s(dn)
except ldap.NO_SUCH_OBJECT as e:
except ldap.NO_SUCH_OBJECT: # pyright: ignore[reportAttributeAccessIssue]
if force_existence:
raise
except Exception as e:
@ -139,37 +175,42 @@ class Connection: # export
raise
def walk(
self,
callback: Callable[[Self, Any, Any], None],
base: str,
scope,
context=None,
filterstr=None,
attrlist=None,
attrsonly=0,
serverctrls=None,
clientctrls=None,
timeout=-1,
sizelimit=0,
decode: bool=False,
unroll: bool=False
):
self,
callback: Callable[[Self, Any, Any], None],
base: str,
scope,
context = None,
filterstr = None,
attrlist = None,
attrsonly = 0,
serverctrls = None,
clientctrls = None,
timeout = -1,
sizelimit = 0,
decode: bool = False,
unroll: bool = False
):
# TODO: Support ignored arguments
search_return = self.__ldap.search(base=base,
scope=scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly)
search_return = self.__ldap.search(
base = base,
scope = scope,
filterstr = filterstr,
attrlist = attrlist,
attrsonly = attrsonly
)
while True:
result_type, result_data = self.__ldap.result(search_return, 0)
if (result_data == []):
if (not result_data):
break
if result_type != ldap.RES_SEARCH_ENTRY:
if result_type != ldap.RES_SEARCH_ENTRY: # pyright: ignore[reportAttributeAccessIssue]
continue
for entry in result_data:
if decode:
entry = entry[0], {key: [val.decode() for val in vals] for key, vals in entry[1].items()}
entry = entry[0], {
key: [val.decode() for val in vals]
for key, vals in entry[1].items()
}
if unroll and False:
entry = entry[0], {key: val[0] for key, val in entry[1].items()}
try:
@ -182,19 +223,20 @@ class Connection: # export
slog(WARNING, msg)
continue
def find(self,
base: str,
scope,
filterstr=None,
attrlist=None,
attrsonly=0,
serverctrls=None,
clientctrls=None,
timeout=-1,
sizelimit=0,
assert_unique=False,
assert_not_empty=False,
):
def find(
self,
base: str,
scope,
filterstr = None,
attrlist = None,
attrsonly = 0,
serverctrls = None,
clientctrls = None,
timeout = -1,
sizelimit = 0,
assert_unique = False,
assert_not_empty = False,
):
def __walk_cb_find(conn: Connection, entry: Any, context: Any):
result.append(entry)
@ -204,7 +246,13 @@ class Connection: # export
try:
result: list[Any] = []
self.walk(__walk_cb_find, base, scope=scope, filterstr=filterstr, attrlist=attrlist)
self.walk(
__walk_cb_find,
base,
scope = scope,
filterstr = filterstr,
attrlist = attrlist
)
except Exception as e:
slog(ERR, f'Failed search {__search()} ({e})')
raise
@ -216,17 +264,34 @@ class Connection: # export
@property
def object_classes(self) -> dict[str, ObjectClass]:
#def object_classes(self):
#def object_classes(self):
if self.__object_classes_by_oid is None:
res = self.find(base='', scope=ldap.SCOPE_BASE, filterstr='(objectClass=*)', attrlist=['subschemaSubentry'])
dn = res[0][1]['subschemaSubentry'][0].decode('utf-8') # Usually yields cn=Subschema
res = self.find(base=dn, scope=ldap.SCOPE_BASE, filterstr='(objectClass=*)', attrlist=['*', '+'])
res = self.find(
base = '',
scope = ldap.SCOPE_BASE, # pyright: ignore[reportAttributeAccessIssue]
filterstr = '(objectClass=*)',
attrlist = ['subschemaSubentry']
)
dn = res[0][1]['subschemaSubentry'][0].decode(
'utf-8'
) # Usually yields cn=Subschema
res = self.find(
base = dn,
scope = ldap.SCOPE_BASE, # pyright: ignore[reportAttributeAccessIssue]
filterstr = '(objectClass=*)',
attrlist = ['*', '+']
)
subschema_entry = res[0]
subschema_subentry = ldap.cidict.cidict(subschema_entry[1])
subschema = ldap.schema.SubSchema(subschema_subentry)
subschema_subentry = ldap.cidict.cidict( # pyright: ignore[reportAttributeAccessIssue]
subschema_entry[1]
)
subschema = ldap.schema.SubSchema( # pyright: ignore[reportAttributeAccessIssue]
subschema_subentry
)
object_class_oids = subschema.listall(ObjectClass)
self.__object_classes_by_oid = {
oid: subschema.get_obj(ObjectClass, oid) for oid in object_class_oids
oid: subschema.get_obj(ObjectClass, oid)
for oid in object_class_oids
}
return self.__object_classes_by_oid
@ -242,15 +307,18 @@ class Connection: # export
ret[name.lower()] = oc
return self.__object_classes_by_name
def __oc_recurse_to_top(self, cur: str|ObjectClass, cb, context):
cur_oc = cur if isinstance(cur, ObjectClass) else self.object_class_by_name[cur.lower()]
def __oc_recurse_to_top(self, cur: str | ObjectClass, cb, context):
cur_oc = cur if isinstance(cur, ObjectClass) else self.object_class_by_name[
cur.lower()]
for s in cur_oc.sup:
self.__oc_recurse_to_top(s, cb, context)
cb(cur_oc, context)
def object_class_path(self, leaf: str|ObjectClass):
def object_class_path(self, leaf: str | ObjectClass):
def cb(oc, context):
ret.append(oc)
ret: list[str] = []
self.__oc_recurse_to_top(leaf, cb, None)
return reversed(ret)
@ -262,47 +330,55 @@ class Connection: # export
def collect(root, attr):
ret = set()
def cb(oc, attr):
vals = getattr(oc, attr)
if vals is None:
return
for val in vals:
ret.add(val)
self.__oc_recurse_to_top(root, cb, attr)
return ret
kind = {
0: 'STRUCTURAL',
1: 'ABSTRACT',
2: 'AUXILIARY'
}
kind = {0: 'STRUCTURAL', 1: 'ABSTRACT', 2: 'AUXILIARY'}
ret = nx.DiGraph()
for oid, oc in self.object_classes.items():
ret.add_node(
oid,
oid=oid,
name=oc.names[0],
kind=kind[oc.kind],
must=', '.join(collect(oc, 'must')),
may=', '.join(collect(oc, 'may'))
)
oid,
oid = oid,
name = oc.names[0],
kind = kind[oc.kind],
must = ', '.join(collect(oc, 'must')),
may = ', '.join(collect(oc, 'may'))
)
for base_class in oc.sup:
try:
ret.add_edge(oid, self.object_class_by_name[base_class.lower()].oid)
ret.add_edge(
oid, self.object_class_by_name[base_class.lower()].oid
)
except Exception as e:
slog(WARNING, f'Failed to add edge {oid}:{oc.names} -> {base_class} ({e})')
slog(
WARNING,
f'Failed to add edge {oid}:{oc.names} -> {base_class} ({e})'
)
self.__object_class_tree = ret
return self.__object_class_tree
def object_class_attrs(self, oc: str|ObjectClass, required: AttrType = AttrType.Must, origins: bool=False) -> dict[str, set[str]]|set[str]:
def object_class_attrs(
self,
oc: str | ObjectClass,
required: AttrType = AttrType.Must,
origins: bool = False
) -> dict[str, set[str]] | set[str]:
all_attrs: set[str] = set()
attrs_by_origin: dict[str, set[str]] = {}
for oc in self.object_class_path(oc):
cur = set()
if required & self.AttrType.Must:
cur |= set(oc.must)
cur |= set(oc.must) # pyright: ignore[reportAttributeAccessIssue]
if required & self.AttrType.May:
cur |= set(oc.may)
cur |= set(oc.may) # pyright: ignore[reportAttributeAccessIssue]
if cur:
all_attrs |= cur
attrs_by_origin[oc] = cur
@ -313,10 +389,14 @@ class Connection: # export
#base_oid = self.object_class_by_name[base_candidate].oid
#if base_oid in [oc.oid for oc in self.object_class_path(name)]:
# return True
return nx.has_path(self.object_class_tree, self.object_class_by_name[name.lower()].oid, self.object_class_by_name[base_candidate.lower()].oid)
return nx.has_path(
self.object_class_tree,
self.object_class_by_name[name.lower()].oid,
self.object_class_by_name[base_candidate.lower()].oid
)
def default_config() -> Config: # export
def default_config() -> Config: # export
return Config()
def bind(conf: Config|BaseConfig|None=None) -> Connection:
def bind(conf: Config | BaseConfig | None = None) -> Connection:
return Connection(conf)

109
src/python/jw/util/log.py
View file

@ -1,22 +1,28 @@
# -*- coding: utf-8 -*-
from __future__ import annotations, print_function
from __future__ import print_function
import inspect
import re
import sys
import syslog
import unicodedata
from typing import List, Tuple, Optional, Any
import sys, re, io, syslog, inspect, unicodedata
from os.path import basename
from datetime import datetime
from os.path import basename
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from . import misc
if TYPE_CHECKING:
import io
# --- python 2 / 3 compatibility stuff
try:
basestring # type: ignore
basestring # type: ignore
except NameError:
basestring = str
# fmt: disable # don't conflate
# yapf: disable # don't conflate
_special_chars = {
'\a' : '\\a',
'\b' : '\\b',
@ -26,12 +32,20 @@ _special_chars = {
'\f' : '\\f',
'\r' : '\\r',
}
# yapf: enable
# fmt: enable
_special_char_regex = re.compile("(%s)" % "|".join(map(re.escape, _special_chars.keys())))
_special_char_regex = re.compile(
"(%s)" % "|".join(map(re.escape, _special_chars.keys()))
)
_all_control_chars = ''.join(chr(c) for c in range(sys.maxunicode) if unicodedata.category(chr(c)) in {'Cc'})
_clean_str_regex = re.compile(r'(\033\[[0-9]*m|[%s])' % re.escape(_all_control_chars))
_all_control_chars = ''.join(
chr(c) for c in range(sys.maxunicode) if unicodedata.category(chr(c)) in {'Cc'}
)
_clean_str_regex = re.compile(r'(\033\[[0-9]*m|[%s])' % re.escape(_all_control_chars))
# fmt: disable # don't conflate
# yapf: disable # don't conflate
EMERG = int(syslog.LOG_EMERG)
ALERT = int(syslog.LOG_ALERT)
CRIT = int(syslog.LOG_CRIT)
@ -98,44 +112,49 @@ _prio_colors = {
EMERG : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_MAGENTA, CONSOLE_FONT_OFF ],
}
# yapf: enable
# fmt: enable
class Stream:
def __init__(self, stream, flags):
self.stream = stream
self.flags = flags
_streams: dict[int, Stream] = dict()
_stream_descriptors = [reversed(range(1, 16))]
_stream_descriptors = list(reversed(range(1, 16)))
def add_capture_stream(stream, flags=0x0):
def add_capture_stream(stream, flags = 0x0):
ret = _stream_descriptors.pop()
_streams[ret] = Stream(stream=stream, flags=flags)
_streams[ret] = Stream(stream = stream, flags = flags)
return ret
def rm_capture_stream(sd):
del _streams[sd]
_stream_descriptors.append(sd)
def prio_gets_logged(prio: int) -> bool: # export
def prio_gets_logged(prio: int) -> bool: # export
if prio > _level:
return False
return True
def log_level(s: Optional[str]=None) -> int: # export
def log_level(s: Optional[str] = None) -> int: # export
if s is None:
return _level
return parse_log_prio_str(s)
def get_caller_pos(up: int = 1, kwargs: Optional[dict[str, Any]] = None) -> Tuple[str, str, int]:
def get_caller_pos(up: int = 1,
kwargs: Optional[dict[str, Any]] = None) -> Tuple[str, str, int]:
if kwargs and 'caller' in kwargs:
r = kwargs['caller']
del kwargs['caller']
return r
caller = inspect.stack()[up+1]
caller = inspect.stack()[up + 1]
mod = inspect.getmodule(caller[0])
mod_name = '' if mod is None else mod.__name__
return (mod_name, basename(caller.filename), caller.lineno)
def slog_m(prio: int, *args, **kwargs) -> None: # export
def slog_m(prio: int, *args, **kwargs) -> None: # export
if prio > _level:
return
if len(args):
@ -151,9 +170,9 @@ def slog_m(prio: int, *args, **kwargs) -> None: # export
caller = kwargs['caller']
del kwargs['caller']
for line in margs[1:].split('\n'):
slog(prio, line, **kwargs, caller=caller)
slog(prio, line, **kwargs, caller = caller)
def slog(prio: int, *args, only_printable: bool=False, **kwargs) -> None: # export
def slog(prio: int, *args, only_printable: bool = False, **kwargs) -> None: # export
if prio > _level:
return
@ -188,11 +207,13 @@ def slog(prio: int, *args, only_printable: bool=False, **kwargs) -> None: # expo
for a in args:
margs += ' ' + str(a)
if only_printable:
margs = _special_char_regex.sub(lambda mo: _special_chars[mo.string[mo.start():mo.end()]], margs)
margs = _special_char_regex.sub(
lambda mo: _special_chars[mo.string[mo.start():mo.end()]], margs
)
margs = re.sub('[\x01-\x1f]', '.', margs)
for file in _log_file_streams:
print(msg + _clean_log_prefix + margs, file=file)
print(msg + _clean_log_prefix + margs, file = file)
msg += _log_prefix
@ -215,24 +236,26 @@ def slog(prio: int, *args, only_printable: bool=False, **kwargs) -> None: # expo
files.append(sys.stderr)
if not len(files):
files = [ sys.stdout ]
files = [sys.stdout]
for file in files:
print(msg, file=file)
print(msg, file = file)
def throw(*args, prio=ERR, caller=None, **kwargs) -> None:
def throw(*args, prio = ERR, caller = None, **kwargs) -> None:
if caller is None:
caller = get_caller_pos(1)
msg = ' '.join([str(arg) for arg in args])
slog(prio, msg, caller=caller)
slog(prio, msg, caller = caller)
raise Exception(msg)
def parse_log_prio_str(prio: str) -> int: # export
def parse_log_prio_str(prio: str) -> int: # export
try:
r = int(prio)
if r < 0 or r > DEVEL:
raise Exception("Invalid log priority ", prio)
except ValueError:
# fmt: disable # don't conflate
# yapf: disable # don't conflate
map_prio_str_to_val = {
"EMERG" : EMERG,
"emerg" : EMERG,
@ -255,23 +278,25 @@ def parse_log_prio_str(prio: str) -> int: # export
"OFF" : OFF,
"off" : OFF,
}
# yapf: enable
# fmt: enable
if prio in map_prio_str_to_val:
return map_prio_str_to_val[prio]
raise Exception("Unknown priority string \"", prio, "\"")
def console_color_chars(prio: int) -> List[str]: # export
def console_color_chars(prio: int) -> List[str]: # export
if not sys.stdout.isatty():
return [ '', '' ]
return ['', '']
return _prio_colors[prio]
def set_level(level_: str) -> None: # export
def set_level(level_: str) -> None: # export
global _level
if isinstance(level_, basestring):
_level = parse_log_prio_str(level_)
return
_level = level_
def set_flags(flags: str|None) -> str: # export
def set_flags(flags: str | None) -> str: # export
global _flags
ret = ','.join(_flags)
if flags is not None:
@ -293,7 +318,7 @@ def set_flags(flags: str|None) -> str: # export
#pid
#highlight_first_error
def append_to_prefix(prefix: str) -> str: # export
def append_to_prefix(prefix: str) -> str: # export
global _log_prefix
global _clean_log_prefix
r = _log_prefix
@ -302,7 +327,7 @@ def append_to_prefix(prefix: str) -> str: # export
_clean_log_prefix = _clean_str_regex.sub('', _log_prefix)
return r
def remove_from_prefix(count) -> str: # export
def remove_from_prefix(count) -> str: # export
if isinstance(count, str):
count = len(count)
global _log_prefix
@ -312,21 +337,21 @@ def remove_from_prefix(count) -> str: # export
_clean_log_prefix = _clean_str_regex.sub('', _log_prefix)
return r
def set_filename_length(l: int) -> int: # export
def set_filename_length(length: int) -> int: # export
global _file_name_len
r = _file_name_len
if l:
_file_name_len = l
if length:
_file_name_len = length
return r
def set_module_name_length(l: int) -> int: # export
def set_module_name_length(length: int) -> int: # export
global _module_name_len
r = _module_name_len
if l:
_module_name_len = l
if length:
_module_name_len = length
return r
def add_log_file(path: str) -> None: # export
def add_log_file(path: str) -> None: # export
global _log_file_streams
fd = open(path, 'w', buffering=1)
fd = open(path, 'w', buffering = 1)
_log_file_streams.append(fd)

101
src/python/jw/util/misc.py
View file

@ -1,6 +1,11 @@
# -*- coding: utf-8 -*-
import os, errno, atexit, tempfile, filecmp, inspect, importlib, re
import atexit
import errno
import filecmp
import importlib
import inspect
import os
import re
import tempfile
from typing import Iterable
@ -12,12 +17,12 @@ def _cleanup():
for f in _tmpfiles:
silentremove(f)
def silentremove(filename): #export
def silentremove(filename): #export
try:
os.remove(filename)
except OSError as e:
if e.errno != errno.ENOENT:
raise # re-raise exception if a different error occurred
raise # re-raise exception if a different error occurred
def update_symlink(target, link_name):
try:
@ -38,12 +43,14 @@ def pad(token: str, total_size: int, right_align: bool = False) -> str:
return space + token
return token + space
def atomic_store(contents, path): # export
def atomic_store(contents, path): # export
if path[0:3] == '/dev':
with open(path, 'w') as outfile:
outfile.write(contents)
return
outfile = tempfile.NamedTemporaryFile(prefix=os.path.basename(path), delete=False, dir=os.path.dirname(path))
outfile = tempfile.NamedTemporaryFile(
prefix = os.path.basename(path), delete = False, dir = os.path.dirname(path)
)
name = outfile.name
_tmpfiles.add(name)
outfile.write(contents)
@ -52,7 +59,7 @@ def atomic_store(contents, path): # export
_tmpfiles.remove(name)
# see https://stackoverflow.com/questions/2020014
def object_builtin_name(o, full=True): # export
def object_builtin_name(o, full = True): # export
#if not full:
# return o.__class__.__name__
module = o.__class__.__module__
@ -60,7 +67,7 @@ def object_builtin_name(o, full=True): # export
return o.__class__.__name__ # Avoid reporting __builtin__
return module + '.' + o.__class__.__name__
def get_derived_classes(mod, base, flt=None): # export
def get_derived_classes(mod, base, flt = None): # export
members = inspect.getmembers(mod, inspect.isclass)
r = []
for name, c in members:
@ -68,8 +75,10 @@ def get_derived_classes(mod, base, flt=None): # export
if inspect.isabstract(c):
log.slog(log.DEBUG, " is abstract")
continue
if not base in inspect.getmro(c):
log.slog(log.DEBUG, " is not derived from", base, "only", inspect.getmro(c))
if base not in inspect.getmro(c):
log.slog(
log.DEBUG, " is not derived from", base, "only", inspect.getmro(c)
)
continue
if flt and not re.match(flt, name):
log.slog(log.DEBUG, ' "{}.{}" has wrong name'.format(mod, name))
@ -77,7 +86,7 @@ def get_derived_classes(mod, base, flt=None): # export
r.append(c)
return r
def load_classes(path, baseclass, flt=None): # export
def load_classes(path, baseclass, flt = None): # export
r = []
for p in path.split(':'):
mod = importlib.import_module(path)
@ -85,22 +94,28 @@ def load_classes(path, baseclass, flt=None): # export
r.extend(get_derived_classes(mod, baseclass, flt))
return r
def load_class(module_path, baseclass, class_name_filter=None): # export
def load_class(module_path, baseclass, class_name_filter = None): # export
mod = importlib.import_module(module_path)
classes = get_derived_classes(mod, baseclass, flt=class_name_filter)
classes = get_derived_classes(mod, baseclass, flt = class_name_filter)
if len(classes) == 0:
raise Exception(f'no class matching "{class_name_filter}" of type "{baseclass}" found in module "{module_path}"')
raise Exception(
f'no class matching "{class_name_filter}" of type "{baseclass}" '
f'found in module "{module_path}"'
)
if len(classes) > 1:
raise Exception(f'{len(classes)} classes matching "{class_name_filter}" of type "{baseclass}" found in module "{module_path}"')
raise Exception(
f'{len(classes)} classes matching "{class_name_filter}" of type '
f'"{baseclass}" found in module "{module_path}"'
)
return classes[0]
def load_class_names(path, baseclass, flt=None, remove_flt=False): # export
def load_class_names(path, baseclass, flt = None, remove_flt = False): # export
classes = load_classes(path, baseclass, flt)
r = []
for c in classes:
name = c.__name__
if flt and remove_flt:
name = re.subst(flt, "", name)
name = re.sub(flt, '', name)
if name not in r:
r.append(name)
else:
@ -108,64 +123,72 @@ def load_class_names(path, baseclass, flt=None, remove_flt=False): # export
#log.slog(log.WARNING, "{} is already in in {}".format(name, r))
return r
def load_object(module_path, baseclass, class_name_filter=None, *args, **kwargs): # export
return load_class(module_path, baseclass, class_name_filter=class_name_filter)(*args, **kwargs)
def load_object( # export
module_path, baseclass, class_name_filter = None, *args, **kwargs
):
return load_class(
module_path, baseclass, class_name_filter = class_name_filter
)(*args, **kwargs)
def load_function(module_path, name): # export
def load_function(module_path, name): # export
mod = importlib.import_module(module_path)
return getattr(mod, name)
def commit_tmpfile(tmp: str, path: str) -> None: # export
def commit_tmpfile(tmp: str, path: str) -> None: # export
caller = log.get_caller_pos()
if os.path.isfile(path) and filecmp.cmp(tmp, path):
log.slog(log.INFO, "{} is up to date".format(path), caller=caller)
log.slog(log.INFO, "{} is up to date".format(path), caller = caller)
os.unlink(tmp)
else:
log.slog(log.NOTICE, "saving {}".format(path), caller=caller)
log.slog(log.NOTICE, "saving {}".format(path), caller = caller)
os.rename(path + '.tmp', path)
def multi_regex_edit(spec, strings): # export
def multi_regex_edit(spec, strings): # export
for cmd in spec:
if len(cmd) < 2:
raise Exception('Invalid command in multi_regex_edit(): {}'.format(str(cmd)))
raise Exception(
'Invalid command in multi_regex_edit(): {}'.format(str(cmd))
)
if cmd[0] == 'sub':
rx = re.compile(cmd[1])
replacement = cmd[2]
r = []
for l in strings:
r.append(re.sub(rx, replacement, l))
for string in strings:
r.append(re.sub(rx, replacement, string))
strings = r
continue
if cmd[0] == 'del':
rx = re.compile(cmd[1])
r = []
for l in strings:
if rx.search(l) is not None:
for string in strings:
if rx.search(string) is not None:
continue
r.append(l)
r.append(string)
strings = r
continue
if cmd[0] == 'match':
rx = re.compile(cmd[1])
r = []
for l in strings:
if rx.search(l) is not None:
r.append(l)
for string in strings:
if rx.search(string) is not None:
r.append(string)
strings = r
continue
raise Exception('Invalid command in multi_regex_edit(): {}'.format(str(cmd)))
return strings
def dump(prio: int, objects: Iterable, *args, **kwargs) -> None: # export
caller = log.get_caller_pos(kwargs=kwargs)
log.slog(prio, ",---------- {}".format(' '.join(args)), caller=caller)
def dump(prio: int, objects: Iterable, *args, **kwargs) -> None: # export
caller = log.get_caller_pos(kwargs = kwargs)
log.slog(prio, ",---------- {}".format(' '.join(args)), caller = caller)
prefix = " | "
log.append_to_prefix(prefix)
i = 1
for o in objects:
o.dump(prio, "{} ({})".format(i, o.__class__.__name__), caller=caller, **kwargs)
o.dump(
prio, "{} ({})".format(i, o.__class__.__name__), caller = caller, **kwargs
)
i += 1
log.remove_from_prefix(prefix)
log.slog(prio, "`---------- {}".format(' '.join(args)), caller=caller)
log.slog(prio, "`---------- {}".format(' '.join(args)), caller = caller)
atexit.register(_cleanup)

457
src/python/jw/util/multi_key_dict.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
'''
Created on 26 May 2013
@ -14,38 +13,42 @@ ___________________________________
Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify, merge,
publish, distribute, sub-license, and/or sell copies of the Software, and to permit persons
to whom the Software is furnished to do so, subject to the following conditions:
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sub-license, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to the following
conditions:
- The above copyright notice and this permission notice shall be included in all copies
or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
'''
import platform
_python3 = int(platform.python_version_tuple()[0]) >= 3
class multi_key_dict(object):
""" The purpose of this type is to provide a multi-key dictionary.
This kind of dictionary has a similar interface to the standard dictionary, and indeed if used
"""
The purpose of this type is to provide a multi-key dictionary. This kind of
dictionary has a similar interface to the standard dictionary, and indeed if used
with single key key elements - it's behaviour is the same as for a standard dict().
However it also allows for creation of elements using multiple keys (using tuples/lists).
Such elements can be accessed using either of those keys (e.g read/updated/deleted).
Dictionary provides also an extended interface for iterating over items and keys by the key type.
This can be useful e.g.: when creating dictionaries with (index,name) allowing one to iterate over
items using either: names or indexes. It can be useful for many many other similar use-cases,
and there is no limit to the number of keys used to map to the value.
However it also allows for creation of elements using multiple keys (using
tuples/lists). Such elements can be accessed using either of those keys (e.g
read/updated/deleted). Dictionary provides also an extended interface for iterating
over items and keys by the key type. This can be useful e.g.: when creating
dictionaries with (index,name) allowing one to iterate over items using either:
names or indexes. It can be useful for many many other similar use-cases, and there
is no limit to the number of keys used to map to the value.
There are also methods to find other keys mapping to the same value as the specified keys etc.
Refer to examples and test code to see it in action.
There are also methods to find other keys mapping to the same value as the specified
keys etc. Refer to examples and test code to see it in action.
simple example:
k = multi_key_dict()
@ -58,21 +61,26 @@ class multi_key_dict(object):
print k[1000] # will print 'kilo (x1000)'
print k['k'] # will also print 'kilo (x1000)'
# the same way objects can be updated, and if an object is updated using one key, the new value will
# be accessible using any other key, e.g. for example above:
# the same way objects can be updated, and if an object is updated using one
# key, the new value will be accessible using any other key, e.g. for example
# above:
k['kilo'] = 'kilo'
print k[1000] # will print 'kilo' as value was updated
"""
def __init__(self, mapping_or_iterable=None, **kwargs):
""" Initializes dictionary from an optional positional argument and a possibly empty set of keyword arguments."""
def __init__(self, mapping_or_iterable = None, **kwargs):
""" Initializes dictionary from an optional positional argument and a possibly
empty set of keyword arguments."""
self.items_dict = {}
if mapping_or_iterable is not None:
if type(mapping_or_iterable) is dict:
mapping_or_iterable = mapping_or_iterable.items()
for kv in mapping_or_iterable:
if len(kv) != 2:
raise Exception('Iterable should contain tuples with exactly two values but specified: {0}.'.format(kv))
raise Exception(
'Iterable should contain tuples with exactly two values '
'but specified: {0}.'.format(kv)
)
self[kv[0]] = kv[1]
for keys, value in kwargs.items():
self[keys] = value
@ -88,15 +96,14 @@ class multi_key_dict(object):
- all of keys should map to exactly the same item (as previously created)
(item update)
If this is not the case - KeyError is raised. """
if(type(keys) in [tuple, list]):
at_least_one_key_exists = False
if (type(keys) in [tuple, list]):
num_of_keys_we_have = 0
for x in keys:
try:
self.__getitem__(x)
num_of_keys_we_have += 1
except Exception as err:
except Exception:
continue
if num_of_keys_we_have:
@ -112,33 +119,34 @@ class multi_key_dict(object):
if new != direct_key:
all_select_same_item = False
break
except Exception as err:
except Exception:
all_select_same_item = False
break;
break
if not all_select_same_item:
raise KeyError(', '.join(str(key) for key in keys))
first_key = keys[0] # combination if keys is allowed, simply use the first one
first_key = keys[
0] # combination if keys is allowed, simply use the first one
else:
first_key = keys
key_type = str(type(first_key)) # find the intermediate dictionary..
key_type = str(type(first_key)) # find the intermediate dictionary..
if first_key in self:
self.items_dict[self.__dict__[key_type][first_key]] = value # .. and update the object if it exists..
self.items_dict[self.__dict__[key_type][first_key]
] = value # .. and update the object if it exists..
else:
if(type(keys) not in [tuple, list]):
if (type(keys) not in [tuple, list]):
key = keys
keys = [keys]
self.__add_item(value, keys) # .. or create it - if it doesn't
self.__add_item(value, keys) # .. or create it - if it doesn't
def __delitem__(self, key):
""" Called to implement deletion of self[key]."""
key_type = str(type(key))
if (key in self and
self.items_dict and
(self.__dict__[key_type][key] in self.items_dict) ):
if (key in self and self.items_dict
and (self.__dict__[key_type][key] in self.items_dict)):
intermediate_key = self.__dict__[key_type][key]
# remove the item in main dictionary
@ -166,10 +174,10 @@ class multi_key_dict(object):
""" Returns True if this object contains an item referenced by the key."""
return key in self
def get_other_keys(self, key, including_current=False):
""" Returns list of other keys that are mapped to the same value as specified key.
@param key - key for which other keys should be returned.
@param including_current if set to True - key will also appear on this list."""
def get_other_keys(self, key, including_current = False):
""" Returns list of other keys that are mapped to the same value as specified
key. @param key - key for which other keys should be returned. @param
including_current if set to True - key will also appear on this list."""
other_keys = []
if key in self:
other_keys.extend(self.__dict__[str(type(key))][key])
@ -177,12 +185,17 @@ class multi_key_dict(object):
other_keys.remove(key)
return other_keys
def iteritems(self, key_type=None, return_all_keys=False):
def iteritems(self, key_type = None, return_all_keys = False):
""" Returns an iterator over the dictionary's (key, value) pairs.
@param key_type if specified, iterator will be returning only (key,value) pairs for this type of key.
Otherwise (if not specified) ((keys,...), value)
i.e. (tuple of keys, values) pairs for all items in this dictionary will be generated.
@param return_all_keys if set to True - tuple of keys is retuned instead of a key of this type."""
@param key_type if specified, iterator will be returning only (key,value)
pairs for this type of key.
Otherwise (if not specified) ((keys,...), value) i.e. (tuple of keys,
values) pairs for all items in this dictionary will be generated.
@param return_all_keys if set to True - tuple of keys is retuned instead of
a key of this type."""
if key_type is None:
for item in self.items_dict.items():
@ -200,13 +213,15 @@ class multi_key_dict(object):
keys = tuple(k for k in keys if isinstance(k, key_type))
yield keys, value
def iterkeys(self, key_type=None, return_all_keys=False):
def iterkeys(self, key_type = None, return_all_keys = False):
""" Returns an iterator over the dictionary's keys.
@param key_type if specified, iterator for a dictionary of this type will be used.
@param key_type if specified, iterator for a dictionary of this type will
be used.
Otherwise (if not specified) tuples containing all (multiple) keys
for this dictionary will be generated.
@param return_all_keys if set to True - tuple of keys is retuned instead of a key of this type."""
if(key_type is not None):
@param return_all_keys if set to True - tuple of keys is retuned instead of
a key of this type."""
if (key_type is not None):
the_key = str(key_type)
if the_key in self.__dict__:
for key in self.__dict__[the_key].keys():
@ -218,11 +233,14 @@ class multi_key_dict(object):
for keys in self.items_dict.keys():
yield keys
def itervalues(self, key_type=None):
def itervalues(self, key_type = None):
""" Returns an iterator over the dictionary's values.
@param key_type if specified, iterator will be returning only values pointed by keys of this type.
Otherwise (if not specified) all values in this dictinary will be generated."""
if(key_type is not None):
@param key_type if specified, iterator will be returning only values pointed
by keys of this type.
Otherwise (if not specified) all values in this dictinary will be
generated."""
if (key_type is not None):
intermediate_key = str(key_type)
if intermediate_key in self.__dict__:
for direct_key in self.__dict__[intermediate_key].values():
@ -232,37 +250,42 @@ class multi_key_dict(object):
yield value
if _python3:
items = iteritems
items = iteritems # type: ignore
else:
def items(self, key_type=None, return_all_keys=False):
def items(self, key_type = None, return_all_keys = False):
return list(self.iteritems(key_type, return_all_keys))
items.__doc__ = iteritems.__doc__
def keys(self, key_type=None):
def keys(self, key_type = None):
""" Returns a copy of the dictionary's keys.
@param key_type if specified, only keys for this type will be returned.
Otherwise list of tuples containing all (multiple) keys will be returned."""
Otherwise list of tuples containing all (multiple) keys will be
returned."""
if key_type is not None:
intermediate_key = str(key_type)
if intermediate_key in self.__dict__:
return self.__dict__[intermediate_key].keys()
else:
all_keys = {} # in order to preserve keys() type (dict_keys for python3)
all_keys = {} # in order to preserve keys() type (dict_keys for python3)
for keys in self.items_dict.keys():
all_keys[keys] = None
return all_keys.keys()
def values(self, key_type=None):
def values(self, key_type = None):
""" Returns a copy of the dictionary's values.
@param key_type if specified, only values pointed by keys of this type will be returned.
Otherwise list of all values contained in this dictionary will be returned."""
if(key_type is not None):
all_items = {} # in order to preserve keys() type (dict_values for python3)
@param key_type if specified, only values pointed by keys of this type
will be returned
Otherwise list of all values contained in this dictionary will be
returned."""
if (key_type is not None):
all_items = {} # in order to preserve keys() type (dict_values for python3)
keys_used = set()
direct_key = str(key_type)
if direct_key in self.__dict__:
for intermediate_key in self.__dict__[direct_key].values():
if not intermediate_key in keys_used:
if intermediate_key not in keys_used:
all_items[intermediate_key] = self.items_dict[intermediate_key]
keys_used.add(intermediate_key)
return all_items.values()
@ -276,26 +299,29 @@ class multi_key_dict(object):
length = len(self.items_dict)
return length
def __add_item(self, item, keys=None):
def __add_item(self, item, keys = None):
""" Internal method to add an item to the multi-key dictionary"""
if(not keys or not len(keys)):
raise Exception('Error in %s.__add_item(%s, keys=tuple/list of items): need to specify a tuple/list containing at least one key!'
% (self.__class__.__name__, str(item)))
direct_key = tuple(keys) # put all keys in a tuple, and use it as a key
if (not keys or not len(keys)):
raise Exception(
'Error in %s.__add_item(%s, keys=tuple/list of items): need to specify'
'a tuple/list containing at least one key!' %
(self.__class__.__name__, str(item))
)
direct_key = tuple(keys) # put all keys in a tuple, and use it as a key
for key in keys:
key_type = str(type(key))
# store direct key as a value in an intermediate dictionary
if(not key_type in self.__dict__):
if (key_type not in self.__dict__):
self.__setattr__(key_type, dict())
self.__dict__[key_type][key] = direct_key
# store the value in the actual dictionary
if(not 'items_dict' in self.__dict__):
if ('items_dict' not in self.__dict__):
self.items_dict = dict()
self.items_dict[direct_key] = item
def get(self, key, default=None):
def get(self, key, default = None):
""" Return the value at index specified as key."""
if key in self:
return self.items_dict[self.__dict__[str(type(key))][key]]
@ -304,74 +330,91 @@ class multi_key_dict(object):
def __str__(self):
items = []
str_repr = lambda x: '\'%s\'' % x if type(x) == str else str(x)
def str_repr(x):
return '\'%s\'' % x if isinstance(x, str) else str(x)
if hasattr(self, 'items_dict'):
for (keys, value) in self.items():
keys_str = [str_repr(k) for k in keys]
items.append('(%s): %s' % (', '.join(keys_str),
str_repr(value)))
dict_str = '{%s}' % ( ', '.join(items))
items.append('(%s): %s' % (', '.join(keys_str), str_repr(value)))
dict_str = '{%s}' % (', '.join(items))
return dict_str
def test_multi_key_dict():
contains_all = lambda cont, in_items: not (False in [c in cont for c in in_items])
def contains_all(cont, in_items):
return False not in [c in cont for c in in_items]
m = multi_key_dict()
assert( len(m) == 0 ), 'expected len(m) == 0'
assert (len(m) == 0), 'expected len(m) == 0'
all_keys = list()
m['aa', 12, 32, 'mmm'] = 123 # create a value with multiple keys..
assert( len(m) == 1 ), 'expected len(m) == 1'
all_keys.append(('aa', 'mmm', 32, 12)) # store it for later
assert (len(m) == 1), 'expected len(m) == 1'
all_keys.append(('aa', 'mmm', 32, 12)) # store it for later
# try retrieving other keys mapped to the same value using one of them
res = m.get_other_keys('aa')
expected = ['mmm', 32, 12]
assert(set(res) == set(expected)), 'get_other_keys(\'aa\'): {0} other than expected: {1} '.format(res, expected)
# try retrieving other keys mapped to the same value using one of them: also include this key
assert (set(res) == set(expected)), (
'get_other_keys(\'aa\'): {0} other '
'than expected: {1} '.format(res, expected)
)
# try retrieving other keys mapped to the same value using one of them: also include
# this key
res = m.get_other_keys(32, True)
expected = ['aa', 'mmm', 32, 12]
assert(set(res) == set(expected)), 'get_other_keys(32): {0} other than expected: {1} '.format(res, expected)
assert (set(res) == set(expected)), (
'get_other_keys(32): {0} other than expected: '
'{1} '.format(res, expected)
)
assert( m.has_key('aa') == True ), 'expected m.has_key(\'aa\') == True'
assert( m.has_key('aab') == False ), 'expected m.has_key(\'aab\') == False'
assert (m.has_key('aa')), 'expected m.has_key(\'aa\') == True'
assert (not m.has_key('aab')), 'expected m.has_key(\'aab\') == False'
assert( m.has_key(12) == True ), 'expected m.has_key(12) == True'
assert( m.has_key(13) == False ), 'expected m.has_key(13) == False'
assert( m.has_key(32) == True ), 'expected m.has_key(32) == True'
assert (m.has_key(12)), 'expected m.has_key(12) == True'
assert (not m.has_key(13)), 'expected m.has_key(13) == False'
assert (m.has_key(32)), 'expected m.has_key(32) == True'
m['something else'] = 'abcd'
assert( len(m) == 2 ), 'expected len(m) == 2'
all_keys.append(('something else',)) # store for later
assert (len(m) == 2), 'expected len(m) == 2'
all_keys.append(('something else', )) # store for later
m[23] = 0
assert( len(m) == 3 ), 'expected len(m) == 3'
all_keys.append((23,)) # store for later
assert (len(m) == 3), 'expected len(m) == 3'
all_keys.append((23, )) # store for later
# check if it's possible to read this value back using either of keys
assert( m['aa'] == 123 ), 'expected m[\'aa\'] == 123'
assert( m[12] == 123 ), 'expected m[12] == 123'
assert( m[32] == 123 ), 'expected m[32] == 123'
assert( m['mmm'] == 123 ), 'expected m[\'mmm\'] == 123'
assert (m['aa'] == 123), 'expected m[\'aa\'] == 123'
assert (m[12] == 123), 'expected m[12] == 123'
assert (m[32] == 123), 'expected m[32] == 123'
assert (m['mmm'] == 123), 'expected m[\'mmm\'] == 123'
# now update value and again - confirm it back - using different keys..
m['aa'] = 45
assert( m['aa'] == 45 ), 'expected m[\'aa\'] == 45'
assert( m[12] == 45 ), 'expected m[12] == 45'
assert( m[32] == 45 ), 'expected m[32] == 45'
assert( m['mmm'] == 45 ), 'expected m[\'mmm\'] == 45'
assert (m['aa'] == 45), 'expected m[\'aa\'] == 45'
assert (m[12] == 45), 'expected m[12] == 45'
assert (m[32] == 45), 'expected m[32] == 45'
assert (m['mmm'] == 45), 'expected m[\'mmm\'] == 45'
m[12] = '4'
assert( m['aa'] == '4' ), 'expected m[\'aa\'] == \'4\''
assert( m[12] == '4' ), 'expected m[12] == \'4\''
assert (m['aa'] == '4'), 'expected m[\'aa\'] == \'4\''
assert (m[12] == '4'), 'expected m[12] == \'4\''
# test __str__
m_str_exp = '{(23): 0, (\'aa\', \'mmm\', 32, 12): \'4\', (\'something else\'): \'abcd\'}'
m_str_exp = (
'{(23): 0, (\'aa\', \'mmm\', 32, 12): \'4\', '
'(\'something else\'): \'abcd\'}'
)
m_str = str(m)
assert (len(m_str) > 0), 'str(m) should not be empty!'
assert (m_str[0] == '{'), 'str(m) should start with \'{\', but does with \'%c\'' % m_str[0]
assert (m_str[-1] == '}'), 'str(m) should end with \'}\', but does with \'%c\'' % m_str[-1]
assert (m_str[0] == '{'
), ('str(m) should start with \'{\', but does with \'%c\'' % m_str[0])
assert (m_str[-1] == '}'
), ('str(m) should end with \'}\', but does with \'%c\'' % m_str[-1])
# check if all key-values are there as expected. They might be sorted differently
def get_values_from_str(dict_str):
@ -381,41 +424,52 @@ def test_multi_key_dict():
keys = tuple(sorted([k.strip() for k in keys.split(',')]))
sorted_keys_and_values.append((keys, val))
return sorted_keys_and_values
exp = get_values_from_str(m_str_exp)
act = get_values_from_str(m_str)
assert (set(act) == set(exp)), 'str(m) values: \'{0}\' are not {1} '.format(act, exp)
assert (set(act) == set(exp)
), ('str(m) values: \'{0}\' are not {1} '.format(act, exp))
# try accessing / creating new (keys)-> value mapping whilst one of these
# keys already maps to a value in this dictionaries
try:
m['aa', 'bb'] = 'something new'
assert(False), 'Should not allow adding multiple-keys when one of keys (\'aa\') already exists!'
except KeyError as err:
assert(False), (
'Should not allow adding multiple-keys when one of keys '
'(\'aa\') already exists!'
)
except KeyError:
pass
# now check if we can get all possible keys (formed in a list of tuples)
# each tuple containing all keys)
res = sorted([sorted([str(x) for x in k]) for k in m.keys()])
res = sorted([sorted([str(x) for x in k]) for k in m.keys()]) # type: ignore
expected = sorted([sorted([str(x) for x in k]) for k in all_keys])
assert(res == expected), 'unexpected values from m.keys(), got:\n%s\n expected:\n%s' %(res, expected)
assert (res == expected), (
'unexpected values from m.keys(), got:\n%s\n expected:\n%s' % (res, expected)
)
# check default items (which will unpack tupe with key(s) and value)
num_of_elements = 0
for keys, value in m.items():
sorted_keys = sorted([str(k) for k in keys])
num_of_elements += 1
assert(sorted_keys in expected), 'm.items(): unexpected keys: %s' % (sorted_keys)
assert(m[keys[0]] == value), 'm.items(): unexpected value: %s (keys: %s)' % (value, keys)
assert(num_of_elements > 0), 'm.items() returned generator that did not produce anything'
assert (sorted_keys
in expected), ('m.items(): unexpected keys: %s' % (sorted_keys))
assert (m[keys[0]] == value
), ('m.items(): unexpected value: %s (keys: %s)' % (value, keys))
assert (num_of_elements
> 0), ('m.items() returned generator that did not produce anything')
# test default iterkeys()
num_of_elements = 0
for keys in m.keys():
for keys in m.keys(): # type: ignore
num_of_elements += 1
keys_s = sorted([str(k) for k in keys])
assert(keys_s in expected), 'm.keys(): unexpected keys: {0}'.format(keys_s)
assert (keys_s in expected), 'm.keys(): unexpected keys: {0}'.format(keys_s)
assert(num_of_elements > 0), 'm.iterkeys() returned generator that did not produce anything'
assert (num_of_elements
> 0), ('m.iterkeys() returned generator that did not produce anything')
# test iterkeys(int, True): useful to get all info from the dictionary
# dictionary is iterated over the type specified, but all keys are returned.
@ -423,75 +477,93 @@ def test_multi_key_dict():
for keys in m.iterkeys(int, True):
keys_s = sorted([str(k) for k in keys])
num_of_elements += 1
assert(keys_s in expected), 'm.iterkeys(int, True): unexpected keys: {0}'.format(keys_s)
assert(num_of_elements > 0), 'm.iterkeys(int, True) returned generator that did not produce anything'
assert (keys_s in expected
), ('m.iterkeys(int, True): unexpected keys: {0}'.format(keys_s))
assert (num_of_elements > 0), (
'm.iterkeys(int, True) returned generator that did not produce anything'
)
# test values for different types of keys()
expected = set([0, '4'])
res = set(m.values(int))
assert (res == expected), 'm.values(int) are {0}, but expected: {1}.'.format(res, expected)
assert (res == expected
), ('m.values(int) are {0}, but expected: {1}.'.format(res, expected))
expected = sorted(['4', 'abcd'])
res = sorted(m.values(str))
assert (res == expected), 'm.values(str) are {0}, but expected: {1}.'.format(res, expected)
assert (res == expected
), ('m.values(str) are {0}, but expected: {1}.'.format(res, expected))
current_values = set([0, '4', 'abcd']) # default (should give all values)
current_values = set([0, '4', 'abcd']) # default (should give all values)
res = set(m.values())
assert (res == current_values), 'm.values() are {0}, but expected: {1}.'.format(res, current_values)
assert (res == current_values
), ('m.values() are {0}, but expected: {1}.'.format(res, current_values))
#test itervalues() (default) - should return all values. (Itervalues for other types
# are tested below)
#test itervalues() (default) - should return all values. (Itervalues for other types are tested below)
vals = set()
for value in m.itervalues():
vals.add(value)
assert (current_values == vals), 'itervalues(): expected {0}, but collected {1}'.format(current_values, vals)
assert (current_values == vals), (
'itervalues(): expected {0}, but collected {1}'.format(current_values, vals)
)
#test items(int)
items_for_int = sorted([((12, 32), '4'), ((23,), 0)])
assert (items_for_int == sorted(m.items(int))), 'items(int): expected {0}, but collected {1}'.format(items_for_int,
sorted(m.items(int)))
items_for_int = sorted([((12, 32), '4'), ((23, ), 0)])
assert (items_for_int == sorted(m.items(int))), (
'items(int): expected {0}, but collected {1}'.format(
items_for_int, sorted(m.items(int))
)
)
# test items(str)
items_for_str = set([(('aa','mmm'), '4'), (('something else',), 'abcd')])
items_for_str = set([(('aa', 'mmm'), '4'), (('something else', ), 'abcd')])
res = set(m.items(str))
assert (set(res) == items_for_str), 'items(str): expected {0}, but collected {1}'.format(items_for_str, res)
assert (set(res) == items_for_str), (
'items(str): expected {0}, but collected {1}'.format(items_for_str, res)
)
# test items() (default - all items)
# we tested keys(), values(), and __get_item__ above so here we'll re-create all_items using that
# we tested keys(), values(), and __get_item__ above so here we'll re-create
# all_items using that
all_items = set()
keys = m.keys()
values = m.values()
for k in keys:
all_items.add( (tuple(k), m[k[0]]) )
m.values()
for k in keys: # type: ignore
all_items.add((tuple(k), m[k[0]]))
res = set(m.items())
assert (all_items == res), 'items() (all items): expected {0},\n\t\t\t\tbut collected {1}'.format(all_items, res)
assert (all_items == res), (
'items() (all items): expected {0},\n\t\t\t\tbut '
'collected {1}'.format(all_items, res)
)
# now test deletion..
curr_len = len(m)
del m[12]
assert( len(m) == curr_len - 1 ), 'expected len(m) == %d' % (curr_len - 1)
assert(not m.has_key(12)), 'expected deleted key to no longer be found!'
assert (len(m) == curr_len - 1), 'expected len(m) == %d' % (curr_len - 1)
assert (not m.has_key(12)), 'expected deleted key to no longer be found!'
# try again
try:
del m['aa']
assert(False), 'cant remove again: item m[\'aa\'] should not exist!'
except KeyError as err:
assert (False), 'cant remove again: item m[\'aa\'] should not exist!'
except KeyError:
pass
# try to access non-existing
try:
k = m['aa']
assert(False), 'removed item m[\'aa\'] should not exist!'
except KeyError as err:
k = m['aa']
assert (False), 'removed item m[\'aa\'] should not exist!'
except KeyError:
pass
# try to access non-existing with a different key
try:
k = m[12]
assert(False), 'removed item m[12] should not exist!'
except KeyError as err:
k = m[12]
assert (False), 'removed item m[12] should not exist!'
except KeyError:
pass
# prepare for other tests (also testing creation of new items)
@ -499,11 +571,12 @@ def test_multi_key_dict():
m = multi_key_dict()
tst_range = list(range(10, 40)) + list(range(50, 70))
for i in tst_range:
m[i] = i # will create a dictionary, where keys are same as items
m[i] = i # will create a dictionary, where keys are same as items
# test items()
for key, value in m.items(int):
assert(key == (value,)), 'items(int): expected {0}, but received {1}'.format(key, value)
assert (key == (value, )
), ('items(int): expected {0}, but received {1}'.format(key, value))
# test iterkeys()
num_of_elements = 0
@ -511,88 +584,103 @@ def test_multi_key_dict():
for key in m.iterkeys(int):
returned_keys.add(key)
num_of_elements += 1
assert(num_of_elements > 0), 'm.iteritems(int) returned generator that did not produce anything'
assert (returned_keys == set(tst_range)), 'iterkeys(int): expected {0}, but received {1}'.format(expected, key)
assert (num_of_elements
> 0), ('m.iteritems(int) returned generator that did not produce anything')
assert (returned_keys == set(tst_range)
), ('iterkeys(int): expected {0}, but received {1}'.format(expected, key))
#test itervalues(int)
num_of_elements = 0
returned_values = set()
returned_values = set()
for value in m.itervalues(int):
returned_values.add(value)
num_of_elements += 1
assert (num_of_elements > 0), 'm.itervalues(int) returned generator that did not produce anything'
assert (returned_values == set(tst_range)), 'itervalues(int): expected {0}, but received {1}'.format(expected, value)
assert (num_of_elements > 0
), ('m.itervalues(int) returned generator that did not produce anything')
assert (returned_values == set(tst_range)), (
'itervalues(int): expected {0}, '
'but received {1}'.format(expected, value)
)
# test values(int)
res = sorted([x for x in m.values(int)])
assert (res == tst_range), 'm.values(int) is not as expected.'
# test keys()
assert (set(m.keys(int)) == set(tst_range)), 'm.keys(int) is not as expected.'
assert (set(m.keys(int)) == set(tst_range)), 'm.keys(int) is not as expected.' # type: ignore
# test setitem with multiple keys
m['xy', 999, 'abcd'] = 'teststr'
try:
m['xy', 998] = 'otherstr'
assert(False), 'creating / updating m[\'xy\', 998] should fail!'
except KeyError as err:
assert (False), 'creating / updating m[\'xy\', 998] should fail!'
except KeyError:
pass
# test setitem with multiple keys
m['cd'] = 'somethingelse'
try:
m['cd', 999] = 'otherstr'
assert(False), 'creating / updating m[\'cd\', 999] should fail!'
except KeyError as err:
assert (False), 'creating / updating m[\'cd\', 999] should fail!'
except KeyError:
pass
m['xy', 999] = 'otherstr'
assert (m['xy'] == 'otherstr'), 'm[\'xy\'] is not as expected.'
assert (m[999] == 'otherstr'), 'm[999] is not as expected.'
assert (m['xy'] == 'otherstr'), 'm[\'xy\'] is not as expected.'
assert (m[999] == 'otherstr'), 'm[999] is not as expected.'
assert (m['abcd'] == 'otherstr'), 'm[\'abcd\'] is not as expected.'
m['abcd', 'xy'] = 'another'
assert (m['xy'] == 'another'), 'm[\'xy\'] is not == \'another\'.'
assert (m[999] == 'another'), 'm[999] is not == \'another\''
m['abcd', 'xy'] = 'another'
assert (m['xy'] == 'another'), 'm[\'xy\'] is not == \'another\'.'
assert (m[999] == 'another'), 'm[999] is not == \'another\''
assert (m['abcd'] == 'another'), 'm[\'abcd\'] is not == \'another\'.'
# test get functionality of basic dictionaries
m['CanIGet'] = 'yes'
assert (m.get('CanIGet') == 'yes')
assert (m.get('ICantGet') == None)
assert (m.get('ICantGet') is None)
assert (m.get('ICantGet', "Ok") == "Ok")
k = multi_key_dict()
k['1:12', 1] = 'key_has_:'
k.items() # should not cause any problems to have : in key
k.items() # should not cause any problems to have : in key
assert (k[1] == 'key_has_:'), 'k[1] is not equal to \'abc:def:ghi\''
import datetime
n = datetime.datetime.now()
l = multi_key_dict()
l[n] = 'now' # use datetime obj as a key
d = multi_key_dict()
d[n] = 'now' # use datetime obj as a key
#test keys..
res = [x for x in l.keys()][0] # for python3 keys() returns dict_keys dictionarly
res = [
x for x in d.keys(
) # type: ignore # for python3 keys() returns dict_keys dictionarly
][0]
expected = n,
assert(expected == res), 'Expected \"{0}\", but got: \"{1}\"'.format(expected, res)
assert (expected == res), 'Expected \"{0}\", but got: \"{1}\"'.format(expected, res)
res = [x for x in l.keys(datetime.datetime)][0]
assert(n == res), 'Expected {0} as a key, but got: {1}'.format(n, res)
res = [x for x in d.keys(datetime.datetime)][0] # type: ignore
assert (n == res), 'Expected {0} as a key, but got: {1}'.format(n, res)
res = [x for x in l.values()] # for python3 keys() returns dict_values dictionarly
res = [x for x in d.values()] # for python3 keys() returns dict_values dictionarly
expected = ['now']
assert(res == expected), 'Expected values: {0}, but got: {1}'.format(expected, res)
assert (res == expected), 'Expected values: {0}, but got: {1}'.format(expected, res)
# test items..
exp_items = [((n,), 'now')]
r = list(l.items())
assert(r == exp_items), 'Expected for items(): tuple of keys: {0}, but got: {1}'.format(r, exp_items)
assert(exp_items[0][1] == 'now'), 'Expected for items(): value: {0}, but got: {1}'.format('now',
exp_items[0][1])
exp_items = [((n, ), 'now')]
r = list(d.items())
assert (r == exp_items), (
'Expected for items(): tuple of keys: {0}, but got: {1}'.format(r, exp_items)
)
assert (exp_items[0][1] == 'now'), (
'Expected for items(): value: {0}, but got: {1}'.format('now', exp_items[0][1])
)
x = multi_key_dict({('k', 'kilo'):1000, ('M', 'MEGA', 1000000):1000000}, milli=0.01)
x = multi_key_dict(
{
('k', 'kilo'): 1000, ('M', 'MEGA', 1000000): 1000000
}, milli = 0.01
)
assert (x['k'] == 1000), 'x[\'k\'] is not equal to 1000'
x['kilo'] = 'kilo'
assert (x['kilo'] == 'kilo'), 'x[\'kilo\'] is not equal to \'kilo\''
@ -605,11 +693,13 @@ def test_multi_key_dict():
try:
y = multi_key_dict([(('two', 'duo'), 2), ('one', 'uno', 1), ('three', 3)])
assert(False), 'creating dictionary using iterable with tuples of size > 2 should fail!'
except:
assert (False), (
'creating dictionary using iterable with tuples of size > 2 should fail!'
)
except Exception:
pass
print ('All test passed OK!')
print('All test passed OK!')
__all__ = ["multi_key_dict"]
@ -617,5 +707,4 @@ if __name__ == '__main__':
try:
test_multi_key_dict()
except KeyboardInterrupt:
print ('\n(interrupted by user)')
print('\n(interrupted by user)')

198
src/python/jw/util/stree/StringTree.py
View file

@ -1,14 +1,16 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Any, List, Optional, Union
import fnmatch
import re
import re, fnmatch
from collections import OrderedDict
from enum import Enum, auto
from typing import TYPE_CHECKING
from ..log import *
if TYPE_CHECKING:
from typing import List, Optional, Union, Any
from ..log import DEBUG, get_caller_pos, slog
def quote(s):
if is_quoted(s):
@ -26,7 +28,7 @@ def is_quoted(s: str) -> bool:
if len(s) < 2:
return False
d = s[0]
if d == s[-1] and d in [ '"', "'" ]:
if d == s[-1] and d in ['"', "'"]:
return True
return False
@ -38,16 +40,18 @@ def cleanup_string(s: str) -> str:
return s[1:-1].replace('\\' + s[0], s[0])
return s
class StringTree: # export
class StringTree: # export
def __init__(self, path: str, content: str, parent: StringTree|None=None) -> None:
def __init__(
self, path: str, content: str, parent: StringTree | None = None
) -> None:
slog(DEBUG, f'Constructing StringTree(path="{path}", content="{content}")')
self.__parent = parent
self.children: OrderedDict[str, StringTree] = OrderedDict()
self.content: Optional[str] = None
self.__set(path, content)
assert(hasattr(self, "content"))
assert (hasattr(self, "content"))
#assert self.content is not None
# root (content = [ symbols ])
@ -65,60 +69,89 @@ class StringTree: # export
#parent.dump(INFO, "These children are added")
self.content = parent.content
for name, c in parent.children.items():
if not name in self.children.keys():
if name not in self.children.keys():
slog(DEBUG, f'At {self.content}: Adding new child {c}')
self.children[name] = c
else:
self.children[name].__adopt_children(c)
def __set(self, path_, content, split=True):
slog(DEBUG, ('At "{}": '.format(str(self.content)) if hasattr(self, "content") else "") + f'Setting "{path_}" -> "{content}"')
#assert self.content != str(content) # Not sure what the idea behind this was. It often goes off, and all works fine without.
if content is not None and not type(content) in [str, StringTree]:
raise Exception("Tried to add content of unsupported type {}".format(type(content).__name__))
def __set(self, path_, content, split = True):
slog(
DEBUG,
('At "{}": '.format(str(self.content)) if hasattr(self, "content") else "")
+ f'Setting "{path_}" -> "{content}"'
)
# Not sure what the idea behind this was. It often goes off, and all
# works fine without.
#assert self.content != str(content)
if content is not None and type(content) not in [str, StringTree]:
raise Exception(
"Tried to add content of unsupported type {}".format(
type(content).__name__
)
)
if path_ is None:
if isinstance(content, str):
self.content = cleanup_string(content)
elif isinstance(content, StringTree):
self.__adopt_children(content)
else:
raise Exception("Tried to add content of unsupported type {}".format(type(content).__name__))
slog(DEBUG, " -- content = >" + str(content) + "<, self.content = >" + str(self.content) + "<")
raise Exception(
"Tried to add content of unsupported type {}".format(
type(content).__name__
)
)
slog(
DEBUG,
" -- content = >" + str(content) + "<, self.content = >" +
str(self.content) + "<"
)
return self
path = cleanup_string(path_)
components = path.split('.') if split else [ path ]
l = len(components)
if len(path) == 0 or l == 0:
#assert self.content is None or (isinstance(content, StringTree) and content.content == self.content)
components = path.split('.') if split else [path]
length = len(components)
if len(path) == 0 or length == 0:
#assert self.content is None or (
# isinstance(content, StringTree) and content.content == self.content
#)
if isinstance(content, StringTree):
#assert isinstance(content, StringTree), "Type: " + type(content).__name__
#assert isinstance(content, StringTree), (
# f'Type: {type(content).__name__ }'
#)
self.__adopt_children(content)
else:
if self.content != content:
#self.content = cleanup_string(content)
slog(DEBUG, f'Changing content: "{self.content}" ->"{content}"')
assert(content != '"[a-zA-Z0-9+_*/-]"')
assert (content != '"[a-zA-Z0-9+_*/-]"')
self.content = content
#assert(content != "'antlr_doesnt_understand_vertical_tab'")
#self.children[content] = StringTree(None, content)
return self
#assert self.content is not None, "tried to set empty content to {}".format(path_)
#assert self.content is not None, f'Tried to set empty content to "{path_}"'
nibble = components[0]
rest = '.'.join(components[1:])
if nibble not in self.children:
self.children[nibble] = StringTree('', content=nibble, parent=self)
if l > 1:
self.children[nibble] = StringTree('', content = nibble, parent = self)
if length > 1:
assert len(rest) > 0
return self.children[nibble].__set(rest, content=content)
return self.children[nibble].__set(rest, content = content)
# last component, a.k.a. leaf
if content is not None:
gc = content if isinstance(content, StringTree) else StringTree('', content=content, parent=self.children[nibble])
gc = content if isinstance(content, StringTree) else StringTree(
'', content = content, parent = self.children[nibble]
)
# Make sure no existing grand child is updated. It would reside too
# far up in the grand child OrderedDict, we need it last
if gc.content in self.children[nibble].children:
del self.children[nibble].children[gc.content]
assert gc.content is not None, 'Grand-child content is None'
self.children[nibble].children[gc.content] = gc
return self.children[nibble]
@ -129,17 +162,17 @@ class StringTree: # export
r = self.get(path)
if r is None:
raise KeyError(path)
return r.value() # type: ignore
return r.value() # type: ignore
def __setitem__(self, key, value):
return self.__set(key, value)
def __dump(self, prio, indent=0, **kwargs):
def __dump(self, prio, indent = 0, **kwargs):
caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
slog(prio, '|' + (' ' * indent) + str(self.content), caller=caller)
slog(prio, '|' + (' ' * indent) + str(self.content), caller = caller)
indent += 2
for name, child in self.children.items():
child.__dump(prio, indent=indent, caller=caller)
child.__dump(prio, indent = indent, caller = caller)
@property
def path(self):
@ -164,7 +197,12 @@ class StringTree: # export
raise Exception("Tried to set empty content")
self.content = content
def add(self, path: str, content: Optional[Union[str, StringTree]] = None, split: bool = True) -> StringTree:
def add(
self,
path: str,
content: Optional[Union[str, StringTree]] = None,
split: bool = True
) -> StringTree:
slog(DEBUG, f'-- At "{self.content}": Adding "{path}" -> "{content}"')
return self.__set(path, content, split)
@ -176,7 +214,7 @@ class StringTree: # export
slog(DEBUG, "returning myself")
return self
if is_quoted(path_):
if not path in self.children.keys():
if path not in self.children.keys():
return None
return self.children[path]
components = path.split('.')
@ -185,7 +223,7 @@ class StringTree: # export
name = cleanup_string(components[0])
if not hasattr(self, "children"):
return None
if not name in self.children.keys():
if name not in self.children.keys():
slog(DEBUG, "Name \"" + name + "\" is not in children of", self.content)
for child in self.children:
slog(DEBUG, "child = ", child)
@ -193,7 +231,7 @@ class StringTree: # export
relpath = '.'.join(components[1:])
return self.children[name].get(relpath)
def value(self, path = None, default=None) -> Optional[str]:
def value(self, path = None, default = None) -> Optional[str]:
if path:
child = self.get(path)
if child is None:
@ -204,7 +242,7 @@ class StringTree: # export
if len(self.children) == 0:
raise Exception('tried to get value from leaf "{}"'.format(self.content))
slog(DEBUG, f'Returning value from children {self.children}')
return self.children[next(reversed(self.children))].content # type: ignore
return self.children[next(reversed(self.children))].content # type: ignore
@property
def parent(self):
@ -216,9 +254,12 @@ class StringTree: # export
return self
return self.__parent.root
def child_list(self, depth_first: bool=True) -> List[StringTree]:
if depth_first == False:
raise Exception("tried to retrieve child list with breadth-first search, not yet implemented")
def child_list(self, depth_first: bool = True) -> List[StringTree]:
if not depth_first:
raise Exception(
'Tried to retrieve child list with breadth-first '
'search, not yet implemented'
)
r = []
for name, c in self.children.items():
r.append(c)
@ -230,32 +271,30 @@ class StringTree: # export
msg = ''
if args is not None:
msg = ' ' + ' '.join(args) + ' '
slog(prio, ",------------" + msg + "----------- >", caller=caller)
self.__dump(prio, indent=0, caller=caller)
slog(prio, "`------------" + msg + "----------- <", caller=caller)
slog(prio, ",------------" + msg + "----------- >", caller = caller)
self.__dump(prio, indent = 0, caller = caller)
slog(prio, "`------------" + msg + "----------- <", caller = caller)
class Match(Enum):
Equal = auto()
RegExArg = auto()
RegExConf = auto()
GlobArg = auto()
GlobConf = auto()
Equal = auto()
RegExArg = auto()
RegExConf = auto()
GlobArg = auto()
GlobConf = auto()
def __find(self, key: str|None, val: str|None, match: Match, depth_first: bool):
def __find(self, key: str | None, val: str | None, m: Match, depth_first: bool):
def __children():
for name, child in self.children.items():
ret.extend(child.__find(key, val, match, depth_first))
ret.extend(child.__find(key, val, m, depth_first))
def __self():
_val = self.value()
_content = self.content
try:
if (
(key == _content and matcher(val, _val))
or (key is None and matcher(val, _val))
or (key == _content and val is None)
):
if ((key == _content and matcher(val, _val))
or (key is None and matcher(val, _val))
or (key == _content and val is None)):
ret.append(self)
except Exception as e:
if isinstance(e, re.PatternError):
@ -263,29 +302,33 @@ class StringTree: # export
else:
raise
def __debug_matcher(matcher, log_level=DEBUG):
def __select_matcher(m: StringTree.Match) -> Any:
match m:
case self.Match.Equal:
return lambda x, y: x == y
case self.Match.RegExArg:
return lambda x, y: re.search(x, y) is not None
case self.Match.RegExConf:
return lambda x, y: re.search(y, x) is not None
case self.Match.GlobArg:
return lambda x, y: fnmatch.fnmatch(y, x)
case self.Match.GlobConf:
return lambda x, y: fnmatch.fnmatch(x, y)
case _:
raise NotImplementedError(f'Matcher {m} is not yet implemented')
def __debug_matcher(matcher, log_level = DEBUG):
def __matcher(x, y):
slog(log_level, f'Comparing "{x}" ~ "{y}"')
return matcher(x, y)
return __matcher
if not self.children:
return []
matcher = lambda x, y: x == y
match match:
case self.Match.Equal:
pass
case self.Match.RegExArg:
matcher = lambda x, y: re.search(x, y) is not None
case self.Match.RegExConf:
matcher = lambda x, y: re.search(y, x) is not None
case self.Match.GlobArg:
matcher = lambda x, y: fnmatch.fnmatch(y, x)
case self.Match.GlobConf:
matcher = lambda x, y: fnmatch.fnmatch(x, y)
case _:
raise NotImplementedError(f'Matcher {match} is not yet implemented')
matcher = __select_matcher(m)
ret: list[StringTree] = []
@ -298,5 +341,16 @@ class StringTree: # export
return ret
def find(self, key: str|None=None, val: str|None=None, match: Match=Match.Equal, depth_first: bool=False):
return [ node.parent.path for node in self.__find(key, val, match=match, depth_first=depth_first)]
def find(
self,
key: str | None = None,
val: str | None = None,
match: Match = Match.Equal,
depth_first: bool = False
):
ret: list[str] = []
for node in self.__find(key, val, m = match, depth_first = depth_first):
if node.parent is None:
break
ret.append(node.parent.path)
return ret

58
src/python/jw/util/stree/serdes.py
View file

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
import glob
import os
import re
import os, glob
from .StringTree import *
from ..log import *
from ..log import DEBUG, ERR, INFO, slog, slog_m
from .StringTree import StringTree, cleanup_string
def _cleanup_line(line: str) -> str:
line = line.strip()
@ -15,18 +15,22 @@ def _cleanup_line(line: str) -> str:
if c == in_quote:
in_quote = None
else:
if c in [ '"', "'" ]:
if c in ['"', "'"]:
in_quote = c
elif in_quote is None and c == '#':
return r.strip()
r += c
if len(r) >= 2 and r[0] in [ '"', "'" ] and r[-1] == r[0]:
if len(r) >= 2 and r[0] in ['"', "'"] and r[-1] == r[0]:
return r[1:-1]
return r
def parse(s: str, allow_full_lines: bool=True, root_content: str='root') -> StringTree: # export
def parse( # export
s: str,
allow_full_lines: bool = True,
root_content: str = 'root'
) -> StringTree:
slog_m(DEBUG, "--->--- parsing --->---\n" + s + "\n---<--- parsing ---<---\n")
root = StringTree('', content=root_content)
root = StringTree('', content = root_content)
sec = ''
for line in s.splitlines():
slog(DEBUG, f'Parsing: "{line}"')
@ -47,7 +51,7 @@ def parse(s: str, allow_full_lines: bool=True, root_content: str='root') -> Stri
root.add(sec)
continue
elif line[0] == ']':
assert(len(sec) > 0)
assert (len(sec) > 0)
sec = '.'.join(sec.split('.')[0:-1])
continue
lhs = ''
@ -67,17 +71,19 @@ def parse(s: str, allow_full_lines: bool=True, root_content: str='root') -> Stri
raise Exception("failed to parse assignment", line)
rhs = 'empty'
split = False
root.add(sec + '.' + cleanup_string(lhs), cleanup_string(rhs), split=split)
root.add(sec + '.' + cleanup_string(lhs), cleanup_string(rhs), split = split)
return root
def _read_lines_from_one_path(path: str, throw=True, level=0, log_prio=INFO, paths_buf=None):
def _read_lines_from_one_path(
path: str, throw = True, level = 0, log_prio = INFO, paths_buf = None
):
try:
with open(path, 'r') as infile:
slog(log_prio, 'Reading {}"{}"'.format(' ' * level * 2, path))
if paths_buf is not None:
paths_buf.append(path)
ret = []
for line in infile: # lines are all trailed by \n
for line in infile: # lines are all trailed by \n
m = re.search(r'^\s*(-)*include\s+(\S+)', line)
if m:
optional = m.group(1) == '-'
@ -86,7 +92,12 @@ def _read_lines_from_one_path(path: str, throw=True, level=0, log_prio=INFO, pat
dir_name = os.path.dirname(path)
if len(dir_name):
include_path = dir_name + '/' + include_path
include_lines = _read_lines(include_path, throw=(not optional), level=level+1, paths_buf=paths_buf)
include_lines = _read_lines(
include_path,
throw = (not optional),
level = level + 1,
paths_buf = paths_buf
)
if include_lines is None:
slog(DEBUG, f'{path}: Failed to process "{line}"')
continue
@ -100,17 +111,26 @@ def _read_lines_from_one_path(path: str, throw=True, level=0, log_prio=INFO, pat
raise
return None
def _read_lines(path: str, throw=True, level=0, log_prio=INFO, paths_buf=None):
def _read_lines(path: str, throw = True, level = 0, log_prio = INFO, paths_buf = None):
paths = glob.glob(path)
ret = []
for p in paths:
rr = _read_lines_from_one_path(p, throw=throw, level=level, log_prio=log_prio, paths_buf=paths_buf)
rr = _read_lines_from_one_path(
p, throw = throw, level = level, log_prio = log_prio, paths_buf = paths_buf
)
if rr is None:
return None
ret.extend(rr)
return ret
def read(path: str, root_content: str='root', log_prio=INFO, paths_buf=None) -> StringTree: # export
lines = _read_lines_from_one_path(path, log_prio=log_prio, paths_buf=paths_buf)
def read( # export
path: str,
root_content: str = 'root',
log_prio = INFO,
paths_buf = None
) -> StringTree:
lines = _read_lines_from_one_path(path, log_prio = log_prio, paths_buf = paths_buf)
if lines is None:
raise Exception(f'Could not read ini file from "{path}"')
s = ''.join(lines)
return parse(s, root_content=root_content)
return parse(s, root_content = root_content)