mirror of
ssh://git.janware.com/srv/git/janware/proj/jw-pkg
synced 2026-01-15 12:03:31 +01:00
projects.py: Dissolve class Build into class Projects
This commit is the next code cleanup step. Needs more testing and more fixing. Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
parent
7d12ad7f8c
commit
a629faab93
1 changed files with 203 additions and 207 deletions
|
|
@ -1,4 +1,8 @@
|
||||||
#!/usr/bin/python3 -u
|
#!/usr/bin/python3 -u
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# This source code file is a merge of various build tools and a horrible mess.
|
||||||
|
#
|
||||||
|
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
import os
|
import os
|
||||||
|
|
@ -58,211 +62,6 @@ class ResultCache(object):
|
||||||
#d = d[k]
|
#d = d[k]
|
||||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||||
|
|
||||||
# ----------------------------------------------------------------- class Build
|
|
||||||
|
|
||||||
class Build(object):
|
|
||||||
|
|
||||||
def __init__(self, projects):
|
|
||||||
self.projects = projects
|
|
||||||
pass
|
|
||||||
|
|
||||||
def find_proj_path(self, name):
|
|
||||||
name = name.replace("dspider-", "")
|
|
||||||
search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
|
|
||||||
for sub in search_path:
|
|
||||||
path = self.projects.projs_root + "/" + sub + "/" + name
|
|
||||||
if os.path.exists(path):
|
|
||||||
return os.path.abspath(path)
|
|
||||||
raise Exception("module " + name + " not found below " + self.projects.projs_root)
|
|
||||||
|
|
||||||
def find_proj_path_cached(self, name):
|
|
||||||
return self.projects.res_cache.run(self.find_proj_path, [ name ])
|
|
||||||
|
|
||||||
def read_deps(self, cur, prereq_type):
|
|
||||||
# dep cache doesn't make a difference at all
|
|
||||||
if prereq_type in self.projects.dep_cache:
|
|
||||||
if cur in self.projects.dep_cache[prereq_type]:
|
|
||||||
return self.projects.dep_cache[prereq_type][cur]
|
|
||||||
else:
|
|
||||||
self.projects.dep_cache[prereq_type] = {}
|
|
||||||
|
|
||||||
if True:
|
|
||||||
r = self.projects.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'],
|
|
||||||
prereq_type, scope = 2, add_self=False, names_only=True)
|
|
||||||
self.projects.debug('prerequisites = ' + ' '.join(r))
|
|
||||||
else: # legacy from build.py
|
|
||||||
projects_py = sys.executable + " " + self.projects.my_dir + "/projects.py --prefix " + self.projects.projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
|
|
||||||
cmd = projects_py + " prereq " + prereq_type + " " + cur
|
|
||||||
self.projects.debug('running', cmd)
|
|
||||||
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
|
|
||||||
p.wait()
|
|
||||||
if p.returncode:
|
|
||||||
raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd)
|
|
||||||
r = set()
|
|
||||||
pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
|
|
||||||
for line in iter(p.stdout.readline, b''):
|
|
||||||
line = line.decode(sys.stdout.encoding)
|
|
||||||
self.projects.debug(cmd + ' returned: ', line)
|
|
||||||
if not pattern.match(line):
|
|
||||||
continue
|
|
||||||
for d in line.split():
|
|
||||||
r.add(d)
|
|
||||||
if cur in r:
|
|
||||||
r.remove(cur)
|
|
||||||
self.projects.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
|
|
||||||
self.projects.dep_cache[prereq_type][cur] = r
|
|
||||||
return r
|
|
||||||
|
|
||||||
def read_deps_cached(self, cur, prereq_type):
|
|
||||||
return self.projects.res_cache.run(self.read_deps, [ cur, prereq_type ])
|
|
||||||
|
|
||||||
def add_dep_tree(self, cur, prereq_types, tree, all_deps):
|
|
||||||
self.projects.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
|
|
||||||
if cur in all_deps:
|
|
||||||
self.projects.debug('already handled module ' + cur)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
deps = set()
|
|
||||||
all_deps.add(cur)
|
|
||||||
for t in prereq_types:
|
|
||||||
self.projects.debug("checking prereqisites of type " + t)
|
|
||||||
deps.update(self.read_deps_cached(cur, t))
|
|
||||||
for d in deps:
|
|
||||||
self.add_dep_tree(d, prereq_types, tree, all_deps)
|
|
||||||
tree[cur] = deps
|
|
||||||
return len(deps)
|
|
||||||
|
|
||||||
def calculate_order(self, order, modules, prereq_types):
|
|
||||||
all_deps = set()
|
|
||||||
dep_tree = {}
|
|
||||||
for m in modules:
|
|
||||||
self.projects.debug("--- adding dependency tree of module " + m)
|
|
||||||
self.add_dep_tree(m, prereq_types, dep_tree, all_deps)
|
|
||||||
while len(all_deps):
|
|
||||||
for d in all_deps:
|
|
||||||
if not len(dep_tree[d]):
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
print(all_deps)
|
|
||||||
raise Exception("fatal: the dependencies between these modules are unresolvable")
|
|
||||||
order.append(d)
|
|
||||||
all_deps.remove(d)
|
|
||||||
for k in dep_tree.keys():
|
|
||||||
if d in dep_tree[k]:
|
|
||||||
dep_tree[k].remove(d)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
def run_make(self, module, target, cur_project, num_projects):
|
|
||||||
#make_cmd = "make " + target + " 2>&1"
|
|
||||||
make_cmd = [ "make", target ]
|
|
||||||
path = self.find_proj_path_cached(module)
|
|
||||||
delim_len=120
|
|
||||||
delim='---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
|
|
||||||
delim = delim + '-' * (delim_len - len(delim))
|
|
||||||
|
|
||||||
print(',' + delim + ' >')
|
|
||||||
|
|
||||||
patt = self.projects.is_excluded_from_build(module)
|
|
||||||
if patt is not None:
|
|
||||||
print('| Configured to skip build on platform >' + patt + '<')
|
|
||||||
print('`' + delim + ' <')
|
|
||||||
return
|
|
||||||
|
|
||||||
os.chdir(path)
|
|
||||||
p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
|
|
||||||
for line in iter(p.stdout.readline, b''):
|
|
||||||
line = line.decode(sys.stdout.encoding)
|
|
||||||
sys.stdout.write('| ' + line) # avoid extra newlines from print()
|
|
||||||
sys.stdout.flush()
|
|
||||||
p.wait()
|
|
||||||
print('`' + delim + ' <')
|
|
||||||
if p.returncode:
|
|
||||||
print(make_cmd + ' failed')
|
|
||||||
raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projects.projs_root)
|
|
||||||
|
|
||||||
def run_make_on_modules(self, modules, order, target):
|
|
||||||
cur_project = 0
|
|
||||||
num_projects = len(order)
|
|
||||||
if target in ["clean", "distclean"]:
|
|
||||||
for m in reversed(order):
|
|
||||||
cur_project += 1
|
|
||||||
self.run_make(m, target, cur_project, num_projects)
|
|
||||||
if m in modules:
|
|
||||||
modules.remove(m)
|
|
||||||
if not len(modules):
|
|
||||||
print("all modules cleaned")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
for m in order:
|
|
||||||
cur_project += 1
|
|
||||||
self.run_make(m, target, cur_project, num_projects)
|
|
||||||
|
|
||||||
def run(self, args_):
|
|
||||||
visited = {}
|
|
||||||
glob_order = []
|
|
||||||
self.projects.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
|
|
||||||
|
|
||||||
# -- parse command line
|
|
||||||
parser = argparse.ArgumentParser(description='janware software project build tool')
|
|
||||||
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
|
|
||||||
parser.add_argument('--debug', '-d', action='store_true',
|
|
||||||
default=False, help='Output debug information to stderr')
|
|
||||||
parser.add_argument('--dry-run', '-n', action='store_true',
|
|
||||||
default=False, help='Don\'t build anything, just print what would be done.')
|
|
||||||
parser.add_argument('--build-order', '-O', action='store_true',
|
|
||||||
default=False, help='Don\'t build anything, just print the build order.')
|
|
||||||
parser.add_argument('--ignore-deps', '-I', action='store_true',
|
|
||||||
default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
|
|
||||||
parser.add_argument('target', default='all', help='Build target')
|
|
||||||
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
|
|
||||||
|
|
||||||
args=parser.parse_args(args_)
|
|
||||||
if args.debug:
|
|
||||||
self.projects.opt_debug = True
|
|
||||||
|
|
||||||
self.projects.debug("----------------------------------------- running ", ' '.join(args_))
|
|
||||||
|
|
||||||
modules=args.modules
|
|
||||||
exclude=args.exclude.split()
|
|
||||||
target=args.target
|
|
||||||
|
|
||||||
env_exclude=os.getenv('BUILD_EXCLUDE', '')
|
|
||||||
if len(env_exclude):
|
|
||||||
print("exluding modules from environment: " + env_exclude)
|
|
||||||
exclude += " " + env_exclude
|
|
||||||
|
|
||||||
# -- build
|
|
||||||
order = []
|
|
||||||
|
|
||||||
glob_prereq_types = [ "build" ]
|
|
||||||
if re.match("pkg-.*", target) is not None:
|
|
||||||
glob_prereq_types = [ "build", "run", "release", "devel" ]
|
|
||||||
|
|
||||||
if target != 'order' and not args.build_order:
|
|
||||||
print("using prerequisite types " + ' '.join(glob_prereq_types))
|
|
||||||
print("calculating order for modules ... ")
|
|
||||||
|
|
||||||
self.calculate_order(order, modules, glob_prereq_types)
|
|
||||||
if args.ignore_deps:
|
|
||||||
order = [m for m in order if m in args.modules]
|
|
||||||
order = [m for m in order if m not in exclude]
|
|
||||||
if target == 'order' or args.build_order:
|
|
||||||
print(' '.join(order))
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
cur_project = 0
|
|
||||||
print("Building target %s in %d projects:" % (target, len(order)))
|
|
||||||
for m in order:
|
|
||||||
cur_project += 1
|
|
||||||
print(" %3d %s" % (cur_project, m))
|
|
||||||
|
|
||||||
if args.dry_run:
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
self.run_make_on_modules(modules, order, target)
|
|
||||||
|
|
||||||
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
|
||||||
|
|
||||||
# ----------------------------------------------------------------- class Projects
|
# ----------------------------------------------------------------- class Projects
|
||||||
|
|
||||||
class Projects(object):
|
class Projects(object):
|
||||||
|
|
@ -651,8 +450,205 @@ class Projects(object):
|
||||||
print(self.commands())
|
print(self.commands())
|
||||||
|
|
||||||
def cmd_build(self, args_):
|
def cmd_build(self, args_):
|
||||||
build = Build(self)
|
|
||||||
build.run(args_)
|
def find_proj_path(name):
|
||||||
|
name = name.replace("dspider-", "")
|
||||||
|
search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
|
||||||
|
for sub in search_path:
|
||||||
|
path = self.projs_root + "/" + sub + "/" + name
|
||||||
|
if os.path.exists(path):
|
||||||
|
return os.path.abspath(path)
|
||||||
|
raise Exception("module " + name + " not found below " + self.projs_root)
|
||||||
|
|
||||||
|
def find_proj_path_cached(name):
|
||||||
|
return self.res_cache.run(find_proj_path, [ name ])
|
||||||
|
|
||||||
|
def read_deps(cur, prereq_type):
|
||||||
|
# dep cache doesn't make a difference at all
|
||||||
|
if prereq_type in self.dep_cache:
|
||||||
|
if cur in self.dep_cache[prereq_type]:
|
||||||
|
return self.dep_cache[prereq_type][cur]
|
||||||
|
else:
|
||||||
|
self.dep_cache[prereq_type] = {}
|
||||||
|
|
||||||
|
if True:
|
||||||
|
r = self.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'],
|
||||||
|
prereq_type, scope = 2, add_self=False, names_only=True)
|
||||||
|
self.debug('prerequisites = ' + ' '.join(r))
|
||||||
|
else: # legacy from build.py
|
||||||
|
projects_py = sys.executable + " " + self.my_dir + "/projects.py --prefix " + self.projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
|
||||||
|
cmd = projects_py + " prereq " + prereq_type + " " + cur
|
||||||
|
self.debug('running', cmd)
|
||||||
|
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
|
||||||
|
p.wait()
|
||||||
|
if p.returncode:
|
||||||
|
raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd)
|
||||||
|
r = set()
|
||||||
|
pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
|
||||||
|
for line in iter(p.stdout.readline, b''):
|
||||||
|
line = line.decode(sys.stdout.encoding)
|
||||||
|
self.debug(cmd + ' returned: ', line)
|
||||||
|
if not pattern.match(line):
|
||||||
|
continue
|
||||||
|
for d in line.split():
|
||||||
|
r.add(d)
|
||||||
|
if cur in r:
|
||||||
|
r.remove(cur)
|
||||||
|
self.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
|
||||||
|
self.dep_cache[prereq_type][cur] = r
|
||||||
|
return r
|
||||||
|
|
||||||
|
def read_deps_cached(cur, prereq_type):
|
||||||
|
return self.res_cache.run(read_deps, [ cur, prereq_type ])
|
||||||
|
|
||||||
|
def add_dep_tree(cur, prereq_types, tree, all_deps):
|
||||||
|
self.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
|
||||||
|
if cur in all_deps:
|
||||||
|
self.debug('already handled module ' + cur)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
deps = set()
|
||||||
|
all_deps.add(cur)
|
||||||
|
for t in prereq_types:
|
||||||
|
self.debug("checking prereqisites of type " + t)
|
||||||
|
deps.update(read_deps_cached(cur, t))
|
||||||
|
for d in deps:
|
||||||
|
add_dep_tree(d, prereq_types, tree, all_deps)
|
||||||
|
tree[cur] = deps
|
||||||
|
return len(deps)
|
||||||
|
|
||||||
|
def calculate_order(order, modules, prereq_types):
|
||||||
|
all_deps = set()
|
||||||
|
dep_tree = {}
|
||||||
|
for m in modules:
|
||||||
|
self.debug("--- adding dependency tree of module " + m)
|
||||||
|
add_dep_tree(m, prereq_types, dep_tree, all_deps)
|
||||||
|
while len(all_deps):
|
||||||
|
for d in all_deps:
|
||||||
|
if not len(dep_tree[d]):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print(all_deps)
|
||||||
|
raise Exception("fatal: the dependencies between these modules are unresolvable")
|
||||||
|
order.append(d)
|
||||||
|
all_deps.remove(d)
|
||||||
|
for k in dep_tree.keys():
|
||||||
|
if d in dep_tree[k]:
|
||||||
|
dep_tree[k].remove(d)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def run_make(module, target, cur_project, num_projects):
|
||||||
|
#make_cmd = "make " + target + " 2>&1"
|
||||||
|
make_cmd = [ "make", target ]
|
||||||
|
path = find_proj_path_cached(module)
|
||||||
|
delim_len=120
|
||||||
|
delim='---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
|
||||||
|
delim = delim + '-' * (delim_len - len(delim))
|
||||||
|
|
||||||
|
print(',' + delim + ' >')
|
||||||
|
|
||||||
|
patt = self.is_excluded_from_build(module)
|
||||||
|
if patt is not None:
|
||||||
|
print('| Configured to skip build on platform >' + patt + '<')
|
||||||
|
print('`' + delim + ' <')
|
||||||
|
return
|
||||||
|
|
||||||
|
os.chdir(path)
|
||||||
|
p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
|
||||||
|
for line in iter(p.stdout.readline, b''):
|
||||||
|
line = line.decode(sys.stdout.encoding)
|
||||||
|
sys.stdout.write('| ' + line) # avoid extra newlines from print()
|
||||||
|
sys.stdout.flush()
|
||||||
|
p.wait()
|
||||||
|
print('`' + delim + ' <')
|
||||||
|
if p.returncode:
|
||||||
|
print(make_cmd + ' failed')
|
||||||
|
raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projs_root)
|
||||||
|
|
||||||
|
def run_make_on_modules(modules, order, target):
|
||||||
|
cur_project = 0
|
||||||
|
num_projects = len(order)
|
||||||
|
if target in ["clean", "distclean"]:
|
||||||
|
for m in reversed(order):
|
||||||
|
cur_project += 1
|
||||||
|
run_make(m, target, cur_project, num_projects)
|
||||||
|
if m in modules:
|
||||||
|
modules.remove(m)
|
||||||
|
if not len(modules):
|
||||||
|
print("all modules cleaned")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
for m in order:
|
||||||
|
cur_project += 1
|
||||||
|
run_make(m, target, cur_project, num_projects)
|
||||||
|
|
||||||
|
def run(args_):
|
||||||
|
visited = {}
|
||||||
|
glob_order = []
|
||||||
|
self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
|
||||||
|
|
||||||
|
# -- parse command line
|
||||||
|
parser = argparse.ArgumentParser(description='janware software project build tool')
|
||||||
|
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
|
||||||
|
parser.add_argument('--debug', '-d', action='store_true',
|
||||||
|
default=False, help='Output debug information to stderr')
|
||||||
|
parser.add_argument('--dry-run', '-n', action='store_true',
|
||||||
|
default=False, help='Don\'t build anything, just print what would be done.')
|
||||||
|
parser.add_argument('--build-order', '-O', action='store_true',
|
||||||
|
default=False, help='Don\'t build anything, just print the build order.')
|
||||||
|
parser.add_argument('--ignore-deps', '-I', action='store_true',
|
||||||
|
default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
|
||||||
|
parser.add_argument('target', default='all', help='Build target')
|
||||||
|
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
|
||||||
|
|
||||||
|
args=parser.parse_args(args_)
|
||||||
|
if args.debug:
|
||||||
|
self.opt_debug = True
|
||||||
|
|
||||||
|
self.debug("----------------------------------------- running ", ' '.join(args_))
|
||||||
|
|
||||||
|
modules=args.modules
|
||||||
|
exclude=args.exclude.split()
|
||||||
|
target=args.target
|
||||||
|
|
||||||
|
env_exclude=os.getenv('BUILD_EXCLUDE', '')
|
||||||
|
if len(env_exclude):
|
||||||
|
print("exluding modules from environment: " + env_exclude)
|
||||||
|
exclude += " " + env_exclude
|
||||||
|
|
||||||
|
# -- build
|
||||||
|
order = []
|
||||||
|
|
||||||
|
glob_prereq_types = [ "build" ]
|
||||||
|
if re.match("pkg-.*", target) is not None:
|
||||||
|
glob_prereq_types = [ "build", "run", "release", "devel" ]
|
||||||
|
|
||||||
|
if target != 'order' and not args.build_order:
|
||||||
|
print("using prerequisite types " + ' '.join(glob_prereq_types))
|
||||||
|
print("calculating order for modules ... ")
|
||||||
|
|
||||||
|
calculate_order(order, modules, glob_prereq_types)
|
||||||
|
if args.ignore_deps:
|
||||||
|
order = [m for m in order if m in args.modules]
|
||||||
|
order = [m for m in order if m not in exclude]
|
||||||
|
if target == 'order' or args.build_order:
|
||||||
|
print(' '.join(order))
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
cur_project = 0
|
||||||
|
print("Building target %s in %d projects:" % (target, len(order)))
|
||||||
|
for m in order:
|
||||||
|
cur_project += 1
|
||||||
|
print(" %3d %s" % (cur_project, m))
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
run_make_on_modules(modules, order, target)
|
||||||
|
|
||||||
|
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||||
|
|
||||||
|
run(args_)
|
||||||
|
|
||||||
def cmd_test(self, args_):
|
def cmd_test(self, args_):
|
||||||
parser = argparse.ArgumentParser(description='Test')
|
parser = argparse.ArgumentParser(description='Test')
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue