jw.build.cmds: Move build.cmds -> cmds.projects

Reorganize the Python module structure. Placing the command classes
under jw.cmds.projects instead of jw.build.cmds will allow to add a
nested command structure, with the current commands, being mostly
related to building software, found below a "projects" toplevel
command.

Other conceivable commands could be "package" for packaging, or
"distro" for commands wrapping the distribution's package manager.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-01-20 15:49:53 +01:00
commit 0b83c863a2
43 changed files with 49 additions and 13 deletions

View file

@ -22,7 +22,7 @@ Q ?= @
ifeq ($(origin PROJECTS_DIR_REMOTE_BASE),undefined) ifeq ($(origin PROJECTS_DIR_REMOTE_BASE),undefined)
ifneq ($(wildcard $(JWBDIR)),) ifneq ($(wildcard $(JWBDIR)),)
PROJECTS_DIR_REMOTE_BASE := $(shell /usr/bin/python3 $(JWBDIR)/scripts/jw-projects.py -p . get-auth-info --remote-base --only-values) PROJECTS_DIR_REMOTE_BASE := $(shell /usr/bin/python3 $(JWBDIR)/scripts/jw-projects.py -p . projects get-auth-info --remote-base --only-values)
endif endif
ifeq ($(PROJECTS_DIR_REMOTE_BASE),) ifeq ($(PROJECTS_DIR_REMOTE_BASE),)
PROJECTS_DIR_REMOTE_BASE := https://janware.com/code PROJECTS_DIR_REMOTE_BASE := https://janware.com/code
@ -44,7 +44,7 @@ endif
ifeq ($(JANWARE_USER),) ifeq ($(JANWARE_USER),)
ifneq ($(wildcard $(JWBDIR)),) ifneq ($(wildcard $(JWBDIR)),)
JANWARE_USER := $(shell /usr/bin/python3 $(JWBDIR)/scripts/jw-projects.py -p . get-auth-info --username --only-values) JANWARE_USER := $(shell /usr/bin/python3 $(JWBDIR)/scripts/jw-projects.py -p . projects get-auth-info --username --only-values)
endif endif
endif endif

View file

@ -116,7 +116,7 @@ ifeq ($(TIME),)
TIME += -p TIME += -p
endif endif
endif endif
PROJECTS_PY = $(TIME) python3 $(JWB_SCRIPT_DIR)/jw-projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS) PROJECTS_PY = $(TIME) python3 $(JWB_SCRIPT_DIR)/jw-projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS) projects
PROJECTS_PY_BUILD = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS) PROJECTS_PY_BUILD = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS)
PGIT_SH := /bin/bash $(JWB_SCRIPT_DIR)/pgit.sh PGIT_SH := /bin/bash $(JWB_SCRIPT_DIR)/pgit.sh

View file

@ -30,7 +30,7 @@ endif
include $(JWBDIR)/make/py-version.mk include $(JWBDIR)/make/py-version.mk
ifneq ($(TOPDIR),) ifneq ($(TOPDIR),)
proj_query_cmd = PYTHONPATH=$(JWBDIR)/src/python $(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_OPTS) proj_query_cmd = PYTHONPATH=$(JWBDIR)/src/python $(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_OPTS) projects
proj_query = $(shell $(proj_query_cmd) $(1)) proj_query = $(shell $(proj_query_cmd) $(1))
proj_dir = $(call proj_query,proj-dir $(1)) proj_dir = $(call proj_query,proj-dir $(1))
htdocs_dir = $(call proj_query,htdocs-dir $(1)) htdocs_dir = $(call proj_query,htdocs-dir $(1))

View file

@ -163,7 +163,7 @@ git-update-project-description:
ssh $(JANWARE_USER)@git.janware.com "/opt/jw-pkg/bin/git-srv-admin.sh $(OPT_JANWARE_PROJECT) update-descriptions $(PROJECT)" ssh $(JANWARE_USER)@git.janware.com "/opt/jw-pkg/bin/git-srv-admin.sh $(OPT_JANWARE_PROJECT) update-descriptions $(PROJECT)"
projects-%: projects-%:
$(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py build $* $(PROJECT) $(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py projects build $* $(PROJECT)
install-deps-devel: install-deps-devel:
sudo zypper in $(shell echo $(RPM_REQUIRES_DEVEL) | sed "s/ *= */-/g; s/ [^ ]\+-__NEXT_VERSION__//") sudo zypper in $(shell echo $(RPM_REQUIRES_DEVEL) | sed "s/ *= */-/g; s/ [^ ]\+-__NEXT_VERSION__//")

View file

@ -41,7 +41,7 @@ subpackage_description()
os_cascade() os_cascade()
{ {
# might want to run python3 path/to/jw-projects.py --os=suse-tumbleweed os-cascade # might want to run python3 path/to/jw-projects.py --os=suse-tumbleweed projects os-cascade
# or turn this into a python script and use jw-projects.py as a module. # or turn this into a python script and use jw-projects.py as a module.
if [ "$DISTRIBUTION" ]; then if [ "$DISTRIBUTION" ]; then

View file

@ -145,7 +145,7 @@ clone()
local git_srv_admin="$SSH $login@git.janware.com /opt/jw-pkg/bin/git-srv-admin.sh" local git_srv_admin="$SSH $login@git.janware.com /opt/jw-pkg/bin/git-srv-admin.sh"
if [ -z "$projects" ]; then if [ -z "$projects" ]; then
projects=`$jw_projects list-repos --from-user $fromuser $remote_base` projects=`$jw_projects projects list-repos --from-user $fromuser $remote_base`
[ "$?" != 0 ] && exit 1 [ "$?" != 0 ] && exit 1
fi fi

View file

@ -8,8 +8,6 @@ import os, sys, argparse, pwd, re
from .lib.App import App as Base from .lib.App import App as Base
from .lib.log import * from .lib.log import *
from .cmds.build import *
# meaning of pkg.requires.xxx variables # meaning of pkg.requires.xxx variables
# build: needs to be built and installed before this can be built # build: needs to be built and installed before this can be built
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this # devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
@ -52,7 +50,7 @@ class ResultCache(object):
class App(Base): class App(Base):
def __init__(self): def __init__(self):
super().__init__("jw-pkg swiss army knife", modules=["jw.pkg.cmds.build"]) super().__init__("jw-pkg swiss army knife", modules=["jw.pkg.cmds"])
self.global_args = [] self.global_args = []
self.opt_os = None self.opt_os = None

View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from argparse import ArgumentParser
from ..lib.Cmd import Cmd as Base
from ..lib.Types import LoadTypes
class Cmd(Base): # export
def __init__(self, name: str, help: str) -> None:
super().__init__(name, help)
def _add_subcommands(self) -> None:
self.add_subcommands(
LoadTypes(
[__name__.rsplit('.', 1)[0] + '.' + self.name],
type_name_filter=r'Cmd[^.]'
)
)
async def run(self, args):
return self._run(args)

View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
import sys, inspect, re
from argparse import ArgumentParser
from .Cmd import Cmd as CmdBase
class CmdProjects(CmdBase): # export
def __init__(self) -> None:
super().__init__('projects', help='Project metadata evaluation for building packages')
self._add_subcommands()
def add_arguments(self, p: ArgumentParser) -> None:
super().add_arguments(p)
def _run(self, args):
breakpoint()
raise Exception("Running with args", args)

View file

@ -0,0 +1,5 @@
TOPDIR = ../../../../..
PY_UPDATE_INIT_PY = false
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-mod.mk

View file

@ -0,0 +1 @@
from .CmdProjects import CmdProjects

View file

@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
import re
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class BaseCmdPkgRelations(Cmd):
def pkg_relations(self, rel_type, args):
version_pattern=re.compile("[0-9-.]*")
if args.subsections is None:
subsecs = self.app.os_cascade()
subsecs.append('jw')
else:
subsecs = args.subsections.split(',')
self.app.debug('flavour = ', args.flavour, ', subsecs = ', ' '.join(subsecs))
ignore = args.ignore.split(',')
self.app.debug("ignore = ", ignore)
r = []
flavours = args.flavour.split(',')
for flavour in flavours:
for s in subsecs:
section = 'pkg.' + rel_type + '.' + s
visited = set()
modules = args.module.copy()
while len(modules):
m = modules.pop(0)
if m in visited or m in ignore:
continue
visited.add(m)
value = self.app.get_value(m, section, flavour)
if not value:
continue
deps = value.split(',')
for spec in deps:
dep = re.split('([=><]+)', spec)
if args.no_version:
dep = dep[:1]
dep = list(map(str.strip, dep))
dep_name = re.sub('-dev$|-devel$|-run$', '', dep[0])
if dep_name in ignore or dep[0] in ignore:
continue
if args.no_subpackages:
dep[0] = dep_name
for i, item in enumerate(dep):
dep[i] = item.strip()
if s == 'jw':
if args.recursive and not dep_name in visited and not dep_name in modules:
modules.append(dep_name)
if len(dep) == 3:
if args.dont_expand_version_macros and dep_name in args.module:
version = dep[2]
else:
version = self.app.get_value(dep_name, 'version', '')
if dep[2] == 'VERSION':
if args.dont_strip_revision:
dep[2] = version
else:
dep[2] = version.split('-')[0]
elif dep[2] == 'VERSION-REVISION':
dep[2] = version
elif version_pattern.match(dep[2]):
# dep[2] = dep[2]
pass
else:
raise Exception("Unknown version specifier in " + spec)
cleaned_dep = ' '.join(dep)
if not cleaned_dep in r:
self.app.debug("appending", cleaned_dep)
r.append(cleaned_dep)
return args.delimiter.join(r)
def print_pkg_relations(self, rel_type, args_):
print(self.pkg_relations(rel_type, args_))
def __init__(self, relation: str, help: str) -> None:
super().__init__('pkg-' + relation, help=help)
self.relation = relation
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('-S', '--subsections', nargs='?', default=None, help='Subsections to consider, comma-separated')
parser.add_argument('-d', '--delimiter', nargs='?', default=', ', help='Output words delimiter')
parser.add_argument('flavour', help='Flavour')
parser.add_argument('module', nargs='*', help='Modules')
parser.add_argument('-p', '--no-subpackages', action='store_true',
default=False, help='Cut -run and -devel from package names')
parser.add_argument('--no-version', action='store_true',
default=False, help='Don\'t report version information')
parser.add_argument('--dont-strip-revision', action='store_true',
default=False, help='Always treat VERSION macro as VERSION-REVISION')
parser.add_argument('--recursive', action='store_true',
default=False, help='Find dependencies recursively')
parser.add_argument('--dont-expand-version-macros', action='store_true',
default=False, help='Don\'t expand VERSION and REVISION macros')
parser.add_argument('--ignore', nargs='?', default='', help='Packages that '
'should be ignored together with their dependencies')
def _run(self, args: Namespace) -> None:
return self.print_pkg_relations(self.relation, args)

View file

@ -0,0 +1,174 @@
# -*- coding: utf-8 -*-
import os, re, sys, subprocess, datetime, time
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdBuild(Cmd): # export
def __init__(self) -> None:
super().__init__('build', help='janware software project build tool')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
parser.add_argument('-n', '--dry-run', action='store_true',
default=False, help='Don\'t build anything, just print what would be done.')
parser.add_argument('-O', '--build-order', action='store_true',
default=False, help='Don\'t build anything, just print the build order.')
parser.add_argument('-I', '--ignore-deps', action='store_true',
default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
parser.add_argument('target', default='all', help='Build target')
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
def _run(self, args: Namespace) -> None:
def read_deps(cur, prereq_type):
# dep cache doesn't make a difference at all
if prereq_type in self.app.dep_cache:
if cur in self.app.dep_cache[prereq_type]:
return self.app.dep_cache[prereq_type][cur]
else:
self.app.dep_cache[prereq_type] = {}
r = self.app.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'],
prereq_type, scope = 2, add_self=False, names_only=True)
self.app.debug('prerequisites = ' + ' '.join(r))
if cur in r:
r.remove(cur)
self.app.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
self.app.dep_cache[prereq_type][cur] = r
return r
def read_deps_cached(cur, prereq_type):
return self.app.res_cache.run(read_deps, [ cur, prereq_type ])
def add_dep_tree(cur, prereq_types, tree, all_deps):
self.app.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
if cur in all_deps:
self.app.debug('already handled module ' + cur)
return 0
deps = set()
all_deps.add(cur)
for t in prereq_types:
self.app.debug("checking prereqisites of type " + t)
deps.update(read_deps_cached(cur, t))
for d in deps:
add_dep_tree(d, prereq_types, tree, all_deps)
tree[cur] = deps
return len(deps)
def calculate_order(order, modules, prereq_types):
all_deps = set()
dep_tree = {}
for m in modules:
self.app.debug("--- adding dependency tree of module " + m)
add_dep_tree(m, prereq_types, dep_tree, all_deps)
while len(all_deps):
# Find any leaf
for d in all_deps:
if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself
break # found
else: # no Leaf found
print(all_deps)
raise Exception("fatal: the dependencies between these modules are unresolvable")
order.append(d) # do it
# bookkeep it
all_deps.remove(d)
for k in dep_tree.keys():
if d in dep_tree[k]:
dep_tree[k].remove(d)
return 1
def run_make(module, target, cur_project, num_projects):
#make_cmd = "make " + target + " 2>&1"
make_cmd = [ "make", target ]
path = self.app.proj_dir(module)
delim_len = 120
delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
delim = delim + '-' * (delim_len - len(delim))
print(',' + delim + ' >')
patt = self.app.is_excluded_from_build(module)
if patt is not None:
print('| Configured to skip build on platform >' + patt + '<')
print('`' + delim + ' <')
return
os.chdir(path)
p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
for line in iter(p.stdout.readline, b''):
line = line.decode(sys.stdout.encoding)
sys.stdout.write('| ' + line) # avoid extra newlines from print()
sys.stdout.flush()
p.wait()
print('`' + delim + ' <')
if p.returncode:
print(' '.join(make_cmd) + ' failed')
raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.app.projs_root)
def run_make_on_modules(modules, order, target):
cur_project = 0
num_projects = len(order)
if target in ["clean", "distclean"]:
for m in reversed(order):
cur_project += 1
run_make(m, target, cur_project, num_projects)
if m in modules:
modules.remove(m)
if not len(modules):
print("all modules cleaned")
return
else:
for m in order:
cur_project += 1
run_make(m, target, cur_project, num_projects)
def run(args):
self.app.debug("----------------------------------------- running ", ' '.join(sys.argv))
modules = args.modules
exclude = args.exclude.split()
target = args.target
env_exclude = os.getenv('BUILD_EXCLUDE', '')
if len(env_exclude):
print("exluding modules from environment: " + env_exclude)
exclude += " " + env_exclude
# -- build
order = []
glob_prereq_types = [ "build" ]
if re.match("pkg-.*", target) is not None:
glob_prereq_types = [ "build", "run", "release", "devel" ]
if target != 'order' and not args.build_order:
print("using prerequisite types " + ' '.join(glob_prereq_types))
print("calculating order for modules ... ")
calculate_order(order, modules, glob_prereq_types)
if args.ignore_deps:
order = [m for m in order if m in args.modules]
order = [m for m in order if m not in exclude]
if target == 'order' or args.build_order:
print(' '.join(order))
exit(0)
cur_project = 0
print("Building target %s in %d projects:" % (target, len(order)))
for m in order:
cur_project += 1
print(" %3d %s" % (cur_project, m))
if args.dry_run:
exit(0)
run_make_on_modules(modules, order, target)
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
run(args)

View file

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdCflags(Cmd): # export
def __init__(self) -> None:
super().__init__('cflags', help='cflags')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('module', nargs='*', help='Modules')
def _run(self, args: Namespace) -> None:
deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build',
scope = 2, add_self=True, names_only=True)
r = ''
for m in reversed(deps):
try:
pd = self.app.proj_dir(m)
if pd is None:
continue
r = r + ' -I' + pd + '/include'
except Exception as e:
self.app.warn(f'No include path for module "{m}", ignoring: {e}')
print(r[1:])

View file

@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdCheck(Cmd): # export
def __init__(self) -> None:
super().__init__('check', help='Check for circular dependencies between given modules')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('module', nargs='*', help='Modules')
parser.add_argument('-f', '--flavour', nargs='?', default = 'build')
def _run(self, args: Namespace) -> None:
graph = {}
path = []
self.app.read_dep_graph(args.module, args.flavour, graph)
unvisited = list(graph.keys())
temp = set()
while len(unvisited) != 0:
m = unvisited[0]
self.app.debug('Checking circular dependency of', m)
last = self.app.check_circular_deps(m, args.flavour, self.app.flip_graph(graph), unvisited, temp, path)
if last is not None:
self.app.debug('Found circular dependency below', m, ', last is', last)
print('Found circular dependency in flavour', args.flavour, ':', ' -> '.join(path))
exit(1)
print('No circular dependency found for flavour', args.flavour, ' in modules:',
' '.join(args.module))
exit(0)

View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdCommands(Cmd): # export
def __init__(self) -> None:
super().__init__('commands', help='List available commands')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
def _run(self, args: Namespace) -> None:
import sys, re, os, glob
this_dir = os.path.dirname(sys.modules[__name__].__file__)
ret = []
for file_name in glob.glob('Cmd*.py', root_dir=this_dir):
cc_name = re.sub(r'^Cmd|\.py', '', file_name)
name = re.sub(r'(?<!^)(?=[A-Z])', '-', cc_name).lower()
ret.append(name)
print(' '.join(ret))

View file

@ -0,0 +1,57 @@
# -*- coding: utf-8 -*-
import textwrap
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdCreatePkgConfig(Cmd): # export
def __init__(self) -> None:
super().__init__('create-pkg-config', help='Generate a pkg-config file for a module')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('-F', '--project-descr-file', default=None)
parser.add_argument('-d', '--description', default=None)
parser.add_argument('-n', '--name', default=None)
parser.add_argument('-s', '--summary', default=None)
parser.add_argument('-p', '--prefix', default=None)
parser.add_argument('-v', '--version', default=None)
parser.add_argument('-c', '--cflags', default=None)
parser.add_argument('-l', '--libflags', default=None)
parser.add_argument('-r', '--requires_run', default=None)
parser.add_argument('-R', '--requires_build', default=None)
parser.add_argument('-V', '--variables', nargs='*')
def _run(self, args: Namespace) -> None:
project_conf_var_keys = ['description', 'summary', 'requires_run', 'requires_build']
merged: dict[str, str] = {}
for key in project_conf_var_keys:
val = getattr(args, key)
if val is not None and args.project_descr_file:
val = self.app.get_value(args.name, key, None)
merged[key] = val
contents = textwrap.dedent(f"""\
prefix={args.prefix}
exec_prefix={{prefix}}
includedir={{prefix}}/include
libdir={{exec_prefix}}/lib
Name: {args.name}
Description: {merged['summary']}
Version: {args.version}
""")
if args.cflags is not None:
contents += f"Cflags: {args.cflags}\n"
if args.libflags is not None:
contents += f"Libs: {args.libflags}\n"
if merged['requires_run'] is not None:
contents += f"Requires: {cleanup_requires(merged['requires_run'])}"
if merged['requires_build'] is not None:
contents += f"Requires.private: {cleanup_requires(merged['requires_build'])}"
# not sure what to do with requires_devel
print(contents)

View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
from argparse import Namespace, ArgumentParser
from ..Cmd import Cmd
class CmdExepath(Cmd): # export
def __init__(self) -> None:
super().__init__('exepath', help='exepath')
def add_arguments(self, parser: ArgumentParser) -> None:
super().add_arguments(parser)
parser.add_argument('module', nargs='*', help='Modules')
def _run(self, args: Namespace) -> None:
deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
scope = 2, add_self=True, names_only=True)
self.app.debug('deps = ', deps)
r = ''
for m in deps:
pd = self.app.proj_dir(m)
if pd is None:
continue
r = r + ':' + pd + '/bin'
print(r[1:])

Some files were not shown because too many files have changed in this diff Show more