build.cmds: Move build.App.cmd_xxx() here

For every cmd_xxx() method in build.App, create a class that's
instatiated for running the respective command. This has the
advantage of making App.py smaller (and faster), and having smaller,
more maintainable command modules adhering to a common interface.

Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/jw/build/App.py

@@ -337,10 +337,7 @@ class App(object):
                            r.append(m)
         return r
 
-    def pkg_relations(self, rel_type, args_):
+    def pkg_relations_add_arguments(self, parser: argparse.ArgumentParser) -> None:
-        parser = argparse.ArgumentParser(description='pkg-' + rel_type)
-        # TODO: implement Vendor evaluation
-
         parser.add_argument('-S', '--subsections', nargs='?', default=None, help='Subsections to consider, comma-separated')
         parser.add_argument('-d', '--delimiter', nargs='?', default=', ', help='Output words delimiter')
         parser.add_argument('flavour', help='Flavour')
@@ -357,7 +354,8 @@ class App(object):
             default=False, help='Don\'t expand VERSION and REVISION macros')
         parser.add_argument('--ignore', nargs='?', default='', help='Packages that '
             'should be ignored together with their dependencies')
-        args = parser.parse_args(args_)
+
+    def pkg_relations(self, rel_type, args):
         version_pattern=re.compile("[0-9-.]*")
         if args.subsections is None:
             subsecs = self.os_cascade()
@@ -479,409 +477,8 @@ class App(object):
             return r[1::]
         return ''
 
-    def commands(self):
-        f = open(sys.argv[0])
-        cmds = []
-        for line in f:
-            self.debug("checking line ", line)
-            rr = re.findall('^    def  *cmd_([a-z0-9_]+).*', line)
-            if len(rr):
-                cmds.append(rr[0].replace('_', '-'))
-        f.close()
-        return ' '.join(cmds)
-
     # ----------------------------------------------------------------- commands
 
-    def cmd_commands(self, args_):
-        print(self.commands())
-
-    def cmd_modules(self, args_):
-        proj_root = self.projs_root
-        self.debug("proj_root = " + proj_root)
-        path = pathlib.Path(self.projs_root)
-
-        parser = argparse.ArgumentParser(description='Query existing janware packages')
-
-        parser.add_argument('-F', '--filter', nargs='?', default=None, help='Key-value pairs, seperated by commas, to be searched for in project.conf')
-        args = parser.parse_args(args_)
-        modules = [p.parents[1].name for p in path.glob('*/make/project.conf')]
-        self.debug("modules = ", modules)
-        out = []
-        filters = None if args.filter is None else [re.split("=", f) for f in re.split(",", args.filter)]
-        for m in modules:
-            if filters:
-                for f in filters:
-                    path = f[0].rsplit('.')
-                    if len(path) > 1:
-                        sec = path[0]
-                        key = path[1]
-                    else:
-                        sec = None
-                        key = path[0]
-                    val = self.get_value(m, sec, key)
-                    self.debug('Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val))
-                    if val and val == f[1]:
-                        out.append(m)
-                        break
-            else:
-                out.append(m)
-        print(' '.join(out))
-
-    def cmd_build(self, args_):
-
-        def read_deps(cur, prereq_type):
-            # dep cache doesn't make a difference at all
-            if prereq_type in self.dep_cache:
-                if cur in self.dep_cache[prereq_type]:
-                    return self.dep_cache[prereq_type][cur]
-            else:
-                self.dep_cache[prereq_type] = {}
-
-            r = self.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'],
-                prereq_type, scope = 2, add_self=False, names_only=True)
-            self.debug('prerequisites = ' + ' '.join(r))
-            if cur in r:
-                r.remove(cur)
-            self.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
-            self.dep_cache[prereq_type][cur] = r
-            return r
-
-        def read_deps_cached(cur, prereq_type):
-            return self.res_cache.run(read_deps, [ cur, prereq_type ])
-
-        def add_dep_tree(cur, prereq_types, tree, all_deps):
-            self.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
-            if cur in all_deps:
-                self.debug('already handled module ' + cur)
-                return 0
-            deps = set()
-            all_deps.add(cur)
-            for t in prereq_types:
-                self.debug("checking prereqisites of type " + t)
-                deps.update(read_deps_cached(cur, t))
-            for d in deps:
-                add_dep_tree(d, prereq_types, tree, all_deps)
-            tree[cur] = deps
-            return len(deps)
-
-        def calculate_order(order, modules, prereq_types):
-            all_deps = set()
-            dep_tree = {}
-            for m in modules:
-                self.debug("--- adding dependency tree of module " + m)
-                add_dep_tree(m, prereq_types, dep_tree, all_deps)
-            while len(all_deps):
-                # Find any leaf
-                for d in all_deps:
-                    if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself
-                        break # found
-                else: # no Leaf found
-                    print(all_deps)
-                    raise Exception("fatal: the dependencies between these modules are unresolvable")
-                order.append(d) # do it
-                # bookkeep it
-                all_deps.remove(d)
-                for k in dep_tree.keys():
-                    if d in dep_tree[k]:
-                        dep_tree[k].remove(d)
-            return 1
-
-        def run_make(module, target, cur_project, num_projects):
-            #make_cmd = "make " + target + " 2>&1"
-            make_cmd = [ "make", target ]
-            path = self.proj_dir(module)
-            delim_len = 120
-            delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
-            delim = delim + '-' * (delim_len - len(delim))
-
-            print(',' + delim + ' >')
-
-            patt = self.is_excluded_from_build(module)
-            if patt is not None:
-                    print('| Configured to skip build on platform >' + patt + '<')
-                    print('`' + delim + ' <')
-                    return
-
-            os.chdir(path)
-            p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
-            for line in iter(p.stdout.readline, b''):
-                line = line.decode(sys.stdout.encoding)
-                sys.stdout.write('| ' + line) # avoid extra newlines from print()
-                sys.stdout.flush()
-            p.wait()
-            print('`' + delim + ' <')
-            if p.returncode:
-                print(' '.join(make_cmd) + ' failed')
-                raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projs_root)
-
-        def run_make_on_modules(modules, order, target):
-            cur_project = 0
-            num_projects = len(order)
-            if target in ["clean", "distclean"]:
-                for m in reversed(order):
-                    cur_project += 1
-                    run_make(m, target, cur_project, num_projects)
-                    if m in modules:
-                        modules.remove(m)
-                    if not len(modules):
-                        print("all modules cleaned")
-                        return
-            else:
-                for m in order:
-                    cur_project += 1
-                    run_make(m, target, cur_project, num_projects)
-
-        def run(args_):
-
-            # -- parse command line
-            parser = argparse.ArgumentParser(description='janware software project build tool')
-            parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
-            parser.add_argument('-n', '--dry-run', action='store_true',
-                default=False, help='Don\'t build anything, just print what would be done.')
-            parser.add_argument('-O', '--build-order', action='store_true',
-                default=False, help='Don\'t build anything, just print the build order.')
-            parser.add_argument('-I', '--ignore-deps', action='store_true',
-                default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
-            parser.add_argument('target', default='all', help='Build target')
-            parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
-
-            args = parser.parse_args(args_)
-
-            self.debug("----------------------------------------- running ", ' '.join(args_))
-
-            modules = args.modules
-            exclude = args.exclude.split()
-            target = args.target
-
-            env_exclude = os.getenv('BUILD_EXCLUDE', '')
-            if len(env_exclude):
-                print("exluding modules from environment: " + env_exclude)
-                exclude += " " + env_exclude
-
-            # -- build
-            order = []
-
-            glob_prereq_types = [ "build" ]
-            if re.match("pkg-.*", target) is not None:
-                glob_prereq_types = [ "build", "run", "release", "devel" ]
-
-            if target != 'order' and not args.build_order:
-                print("using prerequisite types " + ' '.join(glob_prereq_types))
-                print("calculating order for modules ... ")
-
-            calculate_order(order, modules, glob_prereq_types)
-            if args.ignore_deps:
-                order = [m for m in order if m in args.modules]
-            order = [m for m in order if m not in exclude]
-            if target == 'order' or args.build_order:
-                print(' '.join(order))
-                exit(0)
-
-            cur_project = 0
-            print("Building target %s in %d projects:" % (target, len(order)))
-            for m in order:
-                cur_project += 1
-                print("  %3d %s" % (cur_project, m))
-
-            if args.dry_run:
-                exit(0)
-
-            run_make_on_modules(modules, order, target)
-
-            print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
-
-        run(args_)
-
-    def cmd_test(self, args_):
-        parser = argparse.ArgumentParser(description='Test')
-        parser.add_argument('blah', default='', help='The blah argument')
-        args = parser.parse_args(args_)
-        print("blah = " + args.blah)
-
-    def cmd_os_cascade(self, args_):
-        print(' '.join(self.os_cascade()))
-
-    def cmd_ldlibpath(self, args_):
-        parser = argparse.ArgumentParser(description='ldlibpath')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
-                scope = 2, add_self=True, names_only=True)
-        r = ''
-        for m in deps:
-            pd = self.proj_dir(m)
-            if pd is None:
-                continue
-            r = r + ':' + pd + '/lib'
-        print(r[1:])
-
-    def cmd_pythonpath_orig(self, args_):
-        parser = argparse.ArgumentParser(description='pythonpath')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ],
-                scope = 2, add_self=True, names_only=True)
-        r = ''
-        for m in deps:
-            pd = self.proj_dir(m)
-            if pd is None:
-                continue
-            for subdir in [ 'src/python', 'tools/python' ]:
-                cand = pd + "/" + subdir
-                if isdir(cand):
-                    r = r + ':' + cand
-        print(r[1:])
-
-    def cmd_exepath(self, args_):
-        parser = argparse.ArgumentParser(description='exepath')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
-                scope = 2, add_self=True, names_only=True)
-        self.debug('deps = ', deps)
-        r = ''
-        for m in deps:
-            pd = self.proj_dir(m)
-            if pd is None:
-                continue
-            r = r + ':' + pd + '/bin'
-        print(r[1:])
-
-    def cmd_libname(self, args_):
-        parser = argparse.ArgumentParser(description='libname')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        print(self.get_libname(args.module))
-
-    def cmd_ldflags(self, args_):
-        parser = argparse.ArgumentParser(description='ldflags')
-        parser.add_argument('module', nargs='*', help='Modules')
-        parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[])
-        parser.add_argument('-s', '--add-self', action='store_true',
-            default=False, help='Include libflags of specified modules, too, not only their dependencies')
-        args = parser.parse_args(args_)
-        print(self.get_ldflags(args.module, args.exclude, args.add_self))
-
-    def cmd_cflags(self, args_):
-        parser = argparse.ArgumentParser(description='cflags')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build',
-                scope = 2, add_self=True, names_only=True)
-        r = ''
-        for m in reversed(deps):
-            try:
-                pd = self.proj_dir(m)
-                if pd is None:
-                    continue
-                r = r + ' -I' + pd + '/include'
-            except Exception as e:
-                self.warn(f'No include path for module "{m}", ignoring: {e}')
-        print(r[1:])
-
-    def cmd_path(self, args_):
-        parser = argparse.ArgumentParser(description='path')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'run',
-                scope = 2, add_self=True, names_only=True)
-        r = ''
-        for m in deps:
-            pd = self.proj_dir(m)
-            if pd is None:
-                continue
-            r = r + ':' + pd + '/bin'
-        print(r[1:])
-
-    # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations
-    def cmd_prereq(self, args_):
-        parser = argparse.ArgumentParser(description='path')
-        parser.add_argument('flavour', help='Flavour')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'],
-            args.flavour, scope = 2, add_self=False, names_only=True)
-        print(' '.join(deps))
-
-    # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations
-    def cmd_required_os_pkg(self, args_):
-        parser = argparse.ArgumentParser(description='required-os-pkg')
-        parser.add_argument('module', nargs='*', help='Modules')
-        parser.add_argument('--flavours', help='Dependency flavours', default='build')
-        parser.add_argument('--skip-excluded', action='store_true', default=False,
-            help='Output empty prerequisite list if module is excluded')
-        args = parser.parse_args(args_)
-        modules = args.module
-        flavours = args.flavours.split()
-        if 'build' in flavours and not 'run' in flavours:
-            # TODO: This adds too much. Only the run dependencies of the build dependencies would be needed.
-            flavours.append('run')
-        self.debug("flavours = " + args.flavours)
-        deps = self.get_modules_from_project_txt(modules, ['pkg.requires.jw'], flavours,
-                scope = 2, add_self=True, names_only=True)
-        if args.skip_excluded:
-            for d in deps:
-                if self.is_excluded_from_build(d) is not None:
-                    deps.remove(d)
-        subsecs = self.os_cascade()
-        self.debug("subsecs = ", subsecs)
-        requires = []
-        for s in subsecs:
-            for f in flavours:
-                vals = self.collect_values(deps, 'pkg.requires.' + s, f)
-                if vals:
-                    requires = requires + vals
-        # TODO: add all not in build tree as -devel
-        r = ''
-        for m in requires:
-            r = r + ' ' + m
-        print(r[1:])
-
-    def cmd_pkg_requires(self, args_):
-        return self.print_pkg_relations("requires", args_)
-
-    def cmd_pkg_conflicts(self, args_):
-        return self.print_pkg_relations("conflicts", args_)
-
-    def cmd_pkg_provides(self, args_):
-        return self.print_pkg_relations("provides", args_)
-
-    def cmd_proj_dir(self, args_):
-        parser = argparse.ArgumentParser(description='proj-dir')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        r = []
-        for m in args.module:
-            try:
-                pd = self.proj_dir(m)
-                if pd is None:
-                    continue
-                r.append(pd)
-            except Exception as e:
-                self.warn(f'No project directory for module "{m}: {e}')
-                continue
-        print(' '.join(r))
-
-    def cmd_htdocs_dir(self, args_):
-        parser = argparse.ArgumentParser(description='htdocs-dir')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        r = []
-        for m in args.module:
-            r.append(self.htdocs_dir(m))
-        print(' '.join(r))
-
-    def cmd_summary(self, args_):
-        parser = argparse.ArgumentParser(description='summary')
-        parser.add_argument('module', nargs='*', help='Modules')
-        args = parser.parse_args(args_)
-        r = []
-        for m in args.module:
-            summary = self.get_value(m, "summary", None)
-            if summary is not None:
-                r.append(summary)
-        print(' '.join(r))
-
     def contains(self, small, big):
         for i in xrange(len(big)-len(small)+1):
             for j in xrange(len(small)):
@@ -927,38 +524,6 @@ class App(object):
         unvisited.remove(module)
         temp.remove(module)
 
-    def cmd_check(self, args_):
-        parser = argparse.ArgumentParser(description='check')
-        parser.add_argument('module', nargs='*', help='Modules')
-        parser.add_argument('-f', '--flavour', nargs='?', default = 'build')
-        args = parser.parse_args(args_)
-
-        graph = {}
-        path = []
-        self.read_dep_graph(args.module, args.flavour, graph)
-        unvisited = graph.keys()
-        temp = set()
-        while len(unvisited) != 0:
-            m = unvisited[0]
-            self.debug('checking circular dependency of', m)
-            last = self.check_circular_deps(m, args.flavour, self.flip_graph(graph), unvisited, temp, path)
-            if last is not None:
-                self.debug('found circular dependency below', m, ', last is', last)
-                print('found circular dependency in flavour', args.flavour, ':', ' -> '.join(path))
-                exit(1)
-
-        print('no circular dependency found for flavour', args.flavour, ' in modules:',
-                ' '.join(args.module))
-        exit(0)
-
-    def cmd_getval(self, args_):
-        parser = argparse.ArgumentParser(description='Get value from project config')
-        parser.add_argument('--project', default = self.top_name, help = 'Project name')
-        parser.add_argument('section', default = '', help = 'Config section')
-        parser.add_argument('key', default = '', help = 'Config key')
-        args = parser.parse_args(args_)
-        print(self.get_value(args.project, args.section, args.key))
-
     def run_from_cmd_module(self, name: str, args_) -> None:
         import importlib
         name = name.replace('-', '_')
@@ -995,7 +560,7 @@ class App(object):
         parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path')
         parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ], help='App Path Prefix')
         parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system')
-        parser.add_argument('cmd', default='', help='Command, one of: ' + self.commands())
+        parser.add_argument('cmd', default='', help='Command, run "{sys.argv[0]} commands" for a list of supported commands')
         parser.add_argument('arg', nargs='*', help='Command arguments')
         args = parser.parse_args(self.global_args)
 
@@ -1018,6 +583,7 @@ class App(object):
         try:
             return self.run_from_cmd_module(args.cmd, sys.argv[(len(self.global_args) + 1)::])
         except:
+            raise
             pass
         cmd_name = 'cmd_' + args.cmd.replace('-', '_')
         cmd = getattr(self, cmd_name)

src/python/jw/build/cmds/CmdBuild.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+import os, re, sys, subprocess, datetime
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdBuild(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('build', help='janware software project build tool')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
+        parser.add_argument('-n', '--dry-run', action='store_true',
+            default=False, help='Don\'t build anything, just print what would be done.')
+        parser.add_argument('-O', '--build-order', action='store_true',
+            default=False, help='Don\'t build anything, just print the build order.')
+        parser.add_argument('-I', '--ignore-deps', action='store_true',
+            default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
+        parser.add_argument('target', default='all', help='Build target')
+        parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
+
+    def _run(self, args: Namespace) -> None:
+
+        def read_deps(cur, prereq_type):
+            # dep cache doesn't make a difference at all
+            if prereq_type in self.app.dep_cache:
+                if cur in self.app.dep_cache[prereq_type]:
+                    return self.app.dep_cache[prereq_type][cur]
+            else:
+                self.app.dep_cache[prereq_type] = {}
+
+            r = self.app.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'],
+                prereq_type, scope = 2, add_self=False, names_only=True)
+            self.app.debug('prerequisites = ' + ' '.join(r))
+            if cur in r:
+                r.remove(cur)
+            self.app.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
+            self.app.dep_cache[prereq_type][cur] = r
+            return r
+
+        def read_deps_cached(cur, prereq_type):
+            return self.app.res_cache.run(read_deps, [ cur, prereq_type ])
+
+        def add_dep_tree(cur, prereq_types, tree, all_deps):
+            self.app.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
+            if cur in all_deps:
+                self.app.debug('already handled module ' + cur)
+                return 0
+            deps = set()
+            all_deps.add(cur)
+            for t in prereq_types:
+                self.app.debug("checking prereqisites of type " + t)
+                deps.update(read_deps_cached(cur, t))
+            for d in deps:
+                add_dep_tree(d, prereq_types, tree, all_deps)
+            tree[cur] = deps
+            return len(deps)
+
+        def calculate_order(order, modules, prereq_types):
+            all_deps = set()
+            dep_tree = {}
+            for m in modules:
+                self.app.debug("--- adding dependency tree of module " + m)
+                add_dep_tree(m, prereq_types, dep_tree, all_deps)
+            while len(all_deps):
+                # Find any leaf
+                for d in all_deps:
+                    if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself
+                        break # found
+                else: # no Leaf found
+                    print(all_deps)
+                    raise Exception("fatal: the dependencies between these modules are unresolvable")
+                order.append(d) # do it
+                # bookkeep it
+                all_deps.remove(d)
+                for k in dep_tree.keys():
+                    if d in dep_tree[k]:
+                        dep_tree[k].remove(d)
+            return 1
+
+        def run_make(module, target, cur_project, num_projects):
+            #make_cmd = "make " + target + " 2>&1"
+            make_cmd = [ "make", target ]
+            path = self.app.proj_dir(module)
+            delim_len = 120
+            delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
+            delim = delim + '-' * (delim_len - len(delim))
+
+            print(',' + delim + ' >')
+
+            patt = self.app.is_excluded_from_build(module)
+            if patt is not None:
+                    print('| Configured to skip build on platform >' + patt + '<')
+                    print('`' + delim + ' <')
+                    return
+
+            os.chdir(path)
+            p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
+            for line in iter(p.stdout.readline, b''):
+                line = line.decode(sys.stdout.encoding)
+                sys.stdout.write('| ' + line) # avoid extra newlines from print()
+                sys.stdout.flush()
+            p.wait()
+            print('`' + delim + ' <')
+            if p.returncode:
+                print(' '.join(make_cmd) + ' failed')
+                raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.app.projs_root)
+
+        def run_make_on_modules(modules, order, target):
+            cur_project = 0
+            num_projects = len(order)
+            if target in ["clean", "distclean"]:
+                for m in reversed(order):
+                    cur_project += 1
+                    run_make(m, target, cur_project, num_projects)
+                    if m in modules:
+                        modules.remove(m)
+                    if not len(modules):
+                        print("all modules cleaned")
+                        return
+            else:
+                for m in order:
+                    cur_project += 1
+                    run_make(m, target, cur_project, num_projects)
+
+        def run(args):
+
+            self.app.debug("----------------------------------------- running ", ' '.join(sys.argv))
+
+            modules = args.modules
+            exclude = args.exclude.split()
+            target = args.target
+
+            env_exclude = os.getenv('BUILD_EXCLUDE', '')
+            if len(env_exclude):
+                print("exluding modules from environment: " + env_exclude)
+                exclude += " " + env_exclude
+
+            # -- build
+            order = []
+
+            glob_prereq_types = [ "build" ]
+            if re.match("pkg-.*", target) is not None:
+                glob_prereq_types = [ "build", "run", "release", "devel" ]
+
+            if target != 'order' and not args.build_order:
+                print("using prerequisite types " + ' '.join(glob_prereq_types))
+                print("calculating order for modules ... ")
+
+            calculate_order(order, modules, glob_prereq_types)
+            if args.ignore_deps:
+                order = [m for m in order if m in args.modules]
+            order = [m for m in order if m not in exclude]
+            if target == 'order' or args.build_order:
+                print(' '.join(order))
+                exit(0)
+
+            cur_project = 0
+            print("Building target %s in %d projects:" % (target, len(order)))
+            for m in order:
+                cur_project += 1
+                print("  %3d %s" % (cur_project, m))
+
+            if args.dry_run:
+                exit(0)
+
+            run_make_on_modules(modules, order, target)
+
+            print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+        run(args)

src/python/jw/build/cmds/CmdCflags.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdCflags(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('cflags', help='cflags')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build',
+                scope = 2, add_self=True, names_only=True)
+        r = ''
+        for m in reversed(deps):
+            try:
+                pd = self.app.proj_dir(m)
+                if pd is None:
+                    continue
+                r = r + ' -I' + pd + '/include'
+            except Exception as e:
+                self.app.warn(f'No include path for module "{m}", ignoring: {e}')
+        print(r[1:])

src/python/jw/build/cmds/CmdCheck.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdCheck(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('check', help='Check for circular dependencies between given modules')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+        parser.add_argument('-f', '--flavour', nargs='?', default = 'build')
+
+    def _run(self, args: Namespace) -> None:
+        graph = {}
+        path = []
+        self.app.read_dep_graph(args.module, args.flavour, graph)
+        unvisited = list(graph.keys())
+        temp = set()
+        while len(unvisited) != 0:
+            m = unvisited[0]
+            self.app.debug('Checking circular dependency of', m)
+            last = self.app.check_circular_deps(m, args.flavour, self.app.flip_graph(graph), unvisited, temp, path)
+            if last is not None:
+                self.app.debug('Found circular dependency below', m, ', last is', last)
+                print('Found circular dependency in flavour', args.flavour, ':', ' -> '.join(path))
+                exit(1)
+        print('No circular dependency found for flavour', args.flavour, ' in modules:',
+                ' '.join(args.module))
+        exit(0)

src/python/jw/build/cmds/CmdCommands.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdCommands(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('commands', help='List available commands')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+
+    def _run(self, args: Namespace) -> None:
+        import sys, re, os, glob
+        this_dir = os.path.dirname(sys.modules[__name__].__file__)
+        ret = []
+        for file_name in glob.glob('Cmd*.py', root_dir=this_dir):
+            cc_name = re.sub(r'^Cmd|\.py', '', file_name)
+            name = re.sub(r'(?<!^)(?=[A-Z])', '-', cc_name).lower()
+            ret.append(name)
+        print(' '.join(ret))

src/python/jw/build/cmds/CmdExepath.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdExepath(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('exepath', help='exepath')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
+                scope = 2, add_self=True, names_only=True)
+        self.app.debug('deps = ', deps)
+        r = ''
+        for m in deps:
+            pd = self.app.proj_dir(m)
+            if pd is None:
+                continue
+            r = r + ':' + pd + '/bin'
+        print(r[1:])

src/python/jw/build/cmds/CmdGetval.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdGetval(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('getval', help='Get value from project config')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('--project', default = self.app.top_name, help = 'Project name')
+        parser.add_argument('section', default = '', help = 'Config section')
+        parser.add_argument('key', default = '', help = 'Config key')
+
+    def _run(self, args: Namespace) -> None:
+        print(self.app.get_value(args.project, args.section, args.key))

src/python/jw/build/cmds/CmdHtdocsDir.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdHtdocsDir(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('htdocs-dir', help='Print source directory containing document root of a given module')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        r = []
+        for m in args.module:
+            r.append(self.app.htdocs_dir(m))
+        print(' '.join(r))

src/python/jw/build/cmds/CmdLdflags.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdLdflags(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('ldflags', help='ldflags')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+        parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[])
+        parser.add_argument('-s', '--add-self', action='store_true',
+            default=False, help='Include libflags of specified modules, too, not only their dependencies')
+
+    def _run(self, args: Namespace) -> None:
+        print(self.app.get_ldflags(args.module, args.exclude, args.add_self))

src/python/jw/build/cmds/CmdLdlibpath.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdLdlibpath(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('ldlibpath', help='ldlibpath')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
+                scope = 2, add_self=True, names_only=True)
+        r = ''
+        for m in deps:
+            pd = self.app.proj_dir(m)
+            if pd is None:
+                continue
+            r = r + ':' + pd + '/lib'
+        print(r[1:])

src/python/jw/build/cmds/CmdLibname.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdLibname(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('libname', help='libname')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        print(self.app.get_libname(args.module))

src/python/jw/build/cmds/CmdModules.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdModules(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('modules', help='Query existing janware packages')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('-F', '--filter', nargs='?', default=None, help='Key-value pairs, seperated by commas, to be searched for in project.conf')
+
+    def _run(self, args: Namespace) -> None:
+        import pathlib
+        proj_root = self.app.projs_root
+        self.app.debug("proj_root = " + proj_root)
+        path = pathlib.Path(self.app.projs_root)
+        modules = [p.parents[1].name for p in path.glob('*/make/project.conf')]
+        self.app.debug("modules = ", modules)
+        out = []
+        filters = None if args.filter is None else [re.split("=", f) for f in re.split(",", args.filter)]
+        for m in modules:
+            if filters:
+                for f in filters:
+                    path = f[0].rsplit('.')
+                    if len(path) > 1:
+                        sec = path[0]
+                        key = path[1]
+                    else:
+                        sec = None
+                        key = path[0]
+                    val = self.app.get_value(m, sec, key)
+                    self.app.debug('Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val))
+                    if val and val == f[1]:
+                        out.append(m)
+                        break
+            else:
+                out.append(m)
+        print(' '.join(out))

src/python/jw/build/cmds/CmdOsCascade.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdOsCascade(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('os-cascade', help='Print project.conf\'s OS configuration precedence of machine this script runs on')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+
+
+    def _run(self, args: Namespace) -> None:
+        print(' '.join(self.app.os_cascade()))

src/python/jw/build/cmds/CmdPath.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPath(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('path', help='path')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'run',
+                scope = 2, add_self=True, names_only=True)
+        r = ''
+        for m in deps:
+            pd = self.app.proj_dir(m)
+            if pd is None:
+                continue
+            r = r + ':' + pd + '/bin'
+        print(r[1:])

src/python/jw/build/cmds/CmdPkgConflicts.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPkgConflicts(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('pkg-conflicts', help='Print packages conflicting with a given package')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.app.pkg_relations_add_arguments(parser)
+        super().add_arguments(parser)
+
+
+    def _run(self, args: Namespace) -> None:
+        return self.app.print_pkg_relations("conflicts", args)

src/python/jw/build/cmds/CmdPkgProvides.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPkgProvides(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('pkg-provides', help='Print packages and capabilities provided a given package')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.app.pkg_relations_add_arguments(parser)
+        super().add_arguments(parser)
+
+    def _run(self, args: Namespace) -> None:
+        return self.app.print_pkg_relations("provides", args)

src/python/jw/build/cmds/CmdPkgRequires.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPkgRequires(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('pkg-requires', help='Print packages required for a given package')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.app.pkg_relations_add_arguments(parser)
+        super().add_arguments(parser)
+
+    def _run(self, args: Namespace) -> None:
+        return self.app.print_pkg_relations("requires", args)

src/python/jw/build/cmds/CmdPrereq.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+# TODO: seems at least partly redundant to CmdPkgRequires / print_pkg_relations
+class CmdPrereq(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('prereq', help='path')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('flavour', help='Flavour')
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'],
+            args.flavour, scope = 2, add_self=False, names_only=True)
+        print(' '.join(deps))

src/python/jw/build/cmds/CmdProjDir.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdProjDir(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('proj-dir', help='Print directory of a given package')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        r = []
+        for m in args.module:
+            try:
+                pd = self.app.proj_dir(m)
+                if pd is None:
+                    continue
+                r.append(pd)
+            except Exception as e:
+                self.app.warn(f'No project directory for module "{m}: {e}')
+                continue
+        print(' '.join(r))

src/python/jw/build/cmds/CmdPythonpath.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPythonpath(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('pythonpath', help='Generate PYTHONPATH for given modules')
+
+    def add_arguments(self, p: ArgumentParser) -> None:
+        super().add_arguments(p)
+        p.add_argument('module', help='Modules', nargs='*')
+
+    def _run(self, args: Namespace) -> None:
+        import os
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ],
+                scope = 2, add_self=True, names_only=True)
+        r = ''
+        for m in deps:
+            pd = self.app.proj_dir(m)
+            if pd is None:
+                continue
+            for subdir in [ 'src/python', 'tools/python' ]:
+                cand = pd + "/" + subdir
+                if os.path.isdir(cand):
+                    r = r + ':' + cand
+        print(r[1:])

src/python/jw/build/cmds/CmdPythonpathOrig.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdPythonpathOrig(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('pythonpath_orig', help='pythonpath')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ],
+                scope = 2, add_self=True, names_only=True)
+        r = ''
+        for m in deps:
+            pd = self.app.proj_dir(m)
+            if pd is None:
+                continue
+            for subdir in [ 'src/python', 'tools/python' ]:
+                cand = pd + "/" + subdir
+                if isdir(cand):
+                    r = r + ':' + cand
+        print(r[1:])

src/python/jw/build/cmds/CmdRequiredOsPkg.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+# TODO: seems at least partly redundant to CmdPkgRequires / print_pkg_relations
+class CmdRequiredOsPkg(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('required-os-pkg', help='List distribution packages required for a package')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+        parser.add_argument('--flavours', help='Dependency flavours', default='build')
+        parser.add_argument('--skip-excluded', action='store_true', default=False,
+            help='Output empty prerequisite list if module is excluded')
+
+    def _run(self, args: Namespace) -> None:
+        modules = args.module
+        flavours = args.flavours.split()
+        if 'build' in flavours and not 'run' in flavours:
+            # TODO: This adds too much. Only the run dependencies of the build dependencies would be needed.
+            flavours.append('run')
+        self.app.debug("flavours = " + args.flavours)
+        deps = self.app.get_modules_from_project_txt(modules, ['pkg.requires.jw'], flavours,
+                scope = 2, add_self=True, names_only=True)
+        if args.skip_excluded:
+            for d in deps:
+                if self.app.is_excluded_from_build(d) is not None:
+                    deps.remove(d)
+        subsecs = self.app.os_cascade()
+        self.app.debug("subsecs = ", subsecs)
+        requires = []
+        for s in subsecs:
+            for f in flavours:
+                vals = self.app.collect_values(deps, 'pkg.requires.' + s, f)
+                if vals:
+                    requires = requires + vals
+        # TODO: add all not in build tree as -devel
+        r = ''
+        for m in requires:
+            r = r + ' ' + m
+        print(r[1:])

src/python/jw/build/cmds/CmdSummary.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdSummary(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('summary', help='Print summary description of given modules')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('module', nargs='*', help='Modules')
+
+    def _run(self, args: Namespace) -> None:
+        r = []
+        for m in args.module:
+            summary = self.app.get_value(m, "summary", None)
+            if summary is not None:
+                r.append(summary)
+        print(' '.join(r))

src/python/jw/build/cmds/CmdTest.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from argparse import Namespace, ArgumentParser
+
+from ..Cmd import Cmd
+
+class CmdTest(Cmd): # export
+
+    def __init__(self) -> None:
+        super().__init__('test', help='Test')
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('blah', default='', help='The blah argument')
+
+    def _run(self, args: Namespace) -> None:
+        print("blah = " + args.blah)