Jan Lindemann
41ca202142
CmdBuild supports the targets all, clean and pkg-*. It uses builtin dependency flavours matching those targets to resolve build order. To make it more flexible in general, and allow it to support more targets, e.g. "check" and "test", this commit adds a --dep-flavours option. Signed-off-by: Jan Lindemann <jan@janware.com>
295 lines
9.9 KiB
Python
295 lines
9.9 KiB
Python
from __future__ import annotations
|
|
|
|
import datetime
|
|
import os
|
|
import re
|
|
|
|
from functools import lru_cache
|
|
from typing import TYPE_CHECKING
|
|
|
|
from ...App import Scope
|
|
from ...lib.log import DEBUG, ERR, NOTICE, log
|
|
from ...lib.util import get_profile_env, pretty_cmd
|
|
from .Cmd import Cmd, Parent
|
|
|
|
if TYPE_CHECKING:
|
|
from argparse import ArgumentParser, Namespace
|
|
from typing import Iterable, TypeAlias
|
|
DepNode: TypeAlias = dict[str, set[str]]
|
|
|
|
class CmdBuild(Cmd): # export
|
|
|
|
def __init__(self, parent: Parent) -> None:
|
|
super().__init__(parent, 'build', help = 'janware software project build tool')
|
|
|
|
def add_arguments(self, parser: ArgumentParser) -> None:
|
|
super().add_arguments(parser)
|
|
parser.add_argument(
|
|
'--exclude',
|
|
default = '',
|
|
help = 'Space seperated ist of modules to be excluded from build',
|
|
)
|
|
parser.add_argument(
|
|
'-n',
|
|
'--dry-run',
|
|
action = 'store_true',
|
|
default = False,
|
|
help = "Don't build anything, just print what would be done.",
|
|
)
|
|
parser.add_argument(
|
|
'-O',
|
|
'--build-order',
|
|
action = 'store_true',
|
|
default = False,
|
|
help = "Don't build anything, just print the build order.",
|
|
)
|
|
parser.add_argument(
|
|
'-I',
|
|
'--ignore-deps',
|
|
action = 'store_true',
|
|
default = False,
|
|
help = (
|
|
"Don't build dependencies, i.e. build only modules specified "
|
|
'on the command line'
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
'--dep-flavours',
|
|
default = 'auto',
|
|
help = (
|
|
'Dependency flavours to take into consideration for build, '
|
|
'comma or space separated'
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
'--env-reinit',
|
|
action = 'store_true',
|
|
default = False,
|
|
help = (
|
|
'Source /etc/profile before each build step. Discard environment '
|
|
'unless --env-keep is specified'
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
'--env-keep',
|
|
default = 'none',
|
|
help = (
|
|
'Comma seperated list of environment variables to keep, '
|
|
'"all" or "none", only meaningful if --env-reinit is specified'
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
'target',
|
|
default = 'all',
|
|
help = 'Build target',
|
|
)
|
|
parser.add_argument(
|
|
'modules',
|
|
nargs = '+',
|
|
default = '',
|
|
help = 'Modules to be built',
|
|
)
|
|
|
|
async def _run(self, args: Namespace) -> None:
|
|
|
|
@lru_cache(maxsize = None)
|
|
def read_deps(cur: str, dep_flavour: str) -> list[str]:
|
|
# dep cache doesn't make a difference at all
|
|
if dep_flavour in dep_cache:
|
|
if cur in dep_cache[dep_flavour]:
|
|
return dep_cache[dep_flavour][cur]
|
|
else:
|
|
dep_cache[dep_flavour] = {}
|
|
|
|
ret = self.app.get_project_refs(
|
|
[cur],
|
|
['pkg.requires.jw'],
|
|
dep_flavour,
|
|
scope = Scope.Subtree,
|
|
add_self = False,
|
|
names_only = True,
|
|
)
|
|
log(DEBUG, f'Prerequisites: {" ".join(ret)}')
|
|
if cur in ret:
|
|
ret.remove(cur)
|
|
|
|
log(
|
|
DEBUG,
|
|
(f'Inserting {dep_flavour}, prerequisites of {cur}: {" ".join(ret)}'),
|
|
)
|
|
|
|
dep_cache[dep_flavour][cur] = ret
|
|
return ret
|
|
|
|
def add_dep_tree(
|
|
cur: str,
|
|
dep_flavours: Iterable[str],
|
|
tree: DepNode,
|
|
all_deps: set[str],
|
|
) -> int:
|
|
log(DEBUG, f'Adding deps "{" ".join(dep_flavours)}" of module {cur}')
|
|
if cur in all_deps:
|
|
log(DEBUG, f'Already handled module "{cur}"')
|
|
return 0
|
|
deps: set[str] = set()
|
|
all_deps.add(cur)
|
|
for t in dep_flavours:
|
|
log(DEBUG, f'Checking deps of type "{t}"')
|
|
deps.update(read_deps(cur, t))
|
|
for d in deps:
|
|
add_dep_tree(d, dep_flavours, tree, all_deps)
|
|
tree[cur] = deps
|
|
return len(deps)
|
|
|
|
def calculate_order(
|
|
order: list[str],
|
|
modules: set[str],
|
|
dep_flavours: Iterable[str],
|
|
) -> int:
|
|
all_deps: set[str] = set()
|
|
dep_tree: DepNode = {}
|
|
for m in modules:
|
|
log(DEBUG, f'--- Adding dependency tree of module "{m}"')
|
|
add_dep_tree(m, dep_flavours, dep_tree, all_deps)
|
|
while len(all_deps):
|
|
# Find any leaf
|
|
for d in all_deps:
|
|
# Dependency d doesn't have dependencies itself
|
|
if not len(dep_tree[d]):
|
|
break # found
|
|
else: # no Leaf found
|
|
raise Exception(
|
|
'Fatal: Dependencies between these modules are unresolvable: '
|
|
', '.join(all_deps)
|
|
)
|
|
order.append(d) # do it
|
|
# bookkeep it
|
|
all_deps.remove(d)
|
|
for k in dep_tree.keys():
|
|
if d in dep_tree[k]:
|
|
dep_tree[k].remove(d)
|
|
return 1
|
|
|
|
async def run_make(
|
|
module: str, target: str, cur_project: int, num_projects: int
|
|
) -> None:
|
|
|
|
patt = self.app.is_excluded_from_build(module)
|
|
if patt is not None:
|
|
title = f'---- {module}'
|
|
log(NOTICE, f',{title} >')
|
|
log(NOTICE, f'| Configured to skip build on platform >{patt}<')
|
|
log(NOTICE, f'`{title} <')
|
|
return
|
|
|
|
make_cmd = ['make', target]
|
|
wd = self.app.find_dir(module, pretty = False)
|
|
title = '---- [%d/%d]: Running "%s" in %s -' % (
|
|
cur_project,
|
|
num_projects,
|
|
' '.join(make_cmd),
|
|
wd,
|
|
)
|
|
|
|
mod_env = None
|
|
if args.env_reinit:
|
|
keep: bool | list[str] = False
|
|
if args.env_keep is not None:
|
|
match args.env_keep:
|
|
case 'all':
|
|
keep = True
|
|
case 'none':
|
|
keep = False
|
|
case _:
|
|
keep = args.env_keep.split(',')
|
|
mod_env = await get_profile_env(keep = keep)
|
|
|
|
try:
|
|
await self.app.exec_context.run(
|
|
make_cmd,
|
|
wd = wd,
|
|
throw = True,
|
|
verbose = True,
|
|
mod_env = mod_env,
|
|
title = title,
|
|
)
|
|
except Exception as e:
|
|
log(
|
|
ERR,
|
|
f'Failed to make target "{target}" in module "{module}" '
|
|
f'below base {self.app.projs_root}: {str(e)}'
|
|
)
|
|
raise
|
|
|
|
async def run_make_on_modules(
|
|
modules: set[str], order: list[str], target: str
|
|
) -> None:
|
|
cur_project = 0
|
|
num_projects = len(order)
|
|
if target not in ['clean', 'distclean']:
|
|
for m in order:
|
|
cur_project += 1
|
|
await run_make(m, target, cur_project, num_projects)
|
|
return
|
|
for m in reversed(order):
|
|
cur_project += 1
|
|
await run_make(m, target, cur_project, num_projects)
|
|
if m in modules:
|
|
modules.remove(m)
|
|
if not len(modules):
|
|
log(NOTICE, 'All modules cleaned')
|
|
|
|
async def run(args: Namespace) -> None:
|
|
|
|
log(DEBUG, f'-------------------------------------- Running {pretty_cmd()}')
|
|
|
|
modules = set(args.modules)
|
|
exclude = set(args.exclude.split())
|
|
target = args.target
|
|
|
|
env_exclude = os.getenv('BUILD_EXCLUDE', '')
|
|
if env_exclude is not None:
|
|
log(NOTICE, f'Exluding modules from environment: {env_exclude}')
|
|
exclude |= set(env_exclude.split())
|
|
|
|
# -- build
|
|
order: list[str] = []
|
|
|
|
if args.dep_flavours != 'auto':
|
|
dep_flavours = re.split(r'[\s,]', args.dep_flavours)
|
|
else:
|
|
dep_flavours = ['build']
|
|
if re.match('pkg-.*', target) is not None:
|
|
dep_flavours.extend(['run', 'release', 'devel'])
|
|
|
|
if target != 'order' and not args.build_order:
|
|
log(NOTICE, 'Using prerequisite flavours ' + ' '.join(dep_flavours))
|
|
log(NOTICE, 'Calculating order for modules ... ')
|
|
|
|
calculate_order(order, modules, dep_flavours)
|
|
if args.ignore_deps:
|
|
order = [m for m in order if m in args.modules]
|
|
order = [m for m in order if m not in exclude]
|
|
if target == 'order' or args.build_order:
|
|
print(' '.join(order))
|
|
exit(0)
|
|
|
|
cur_project = 0
|
|
log(NOTICE, f'Building target {target} in {len(order)} projects:')
|
|
for m in order:
|
|
cur_project += 1
|
|
log(NOTICE, ' %3d %s' % (cur_project, m))
|
|
|
|
if args.dry_run:
|
|
exit(0)
|
|
|
|
await run_make_on_modules(modules, order, target)
|
|
|
|
log(
|
|
NOTICE,
|
|
'Build done at %s' %
|
|
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
|
|
)
|
|
|
|
dep_cache: dict[str, dict[str, list[str]]] = {}
|
|
await run(args)
|