diff --git a/scripts/projects.py b/scripts/projects.py index 1add916e..6c549d25 100644 --- a/scripts/projects.py +++ b/scripts/projects.py @@ -4,6 +4,7 @@ from __future__ import print_function import os import sys import argparse +import pwd from sets import Set from os.path import isfile from os.path import isdir @@ -51,6 +52,186 @@ class ResultCache(object): #d = d[k] raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) +class Build(object): + + def find_proj_path(self, name): + name=name.replace("dspider-", "") + search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] + for sub in search_path: + path=projs_root + "/" + sub + "/" + name + if os.path.exists(path): + return os.path.abspath(path) + raise Exception("module " + name + " not found below " + projs_root) + + def find_proj_path_cached(self, name): + return res_cache.run(self.find_proj_path, [ name ]) + + def read_deps(self, cur, prereq_type): + # dep cache doesn't make a difference at all + projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") + if prereq_type in dep_cache: + if cur in dep_cache[prereq_type]: + return dep_cache[prereq_type][cur] + else: + dep_cache[prereq_type] = {} + cmd = projects_py + " prereq " + prereq_type + " " + cur + debug('running', cmd) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + p.wait() + if p.returncode: + raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) + r = Set() + pattern = re.compile(r'.*') # might be useful at a later point, currently pointless + for line in iter(p.stdout.readline,''): + debug(cmd + ' returned: ', line) + if not pattern.match(line): + continue + for d in line.split(): + r.add(d) + if cur in r: + r.remove(cur) + debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) + dep_cache[prereq_type][cur] = r + return r + + def read_deps_cached(self, cur, prereq_type): + return res_cache.run(self.read_deps, [ cur, prereq_type ]) + + def add_dep_tree(self, cur, prereq_types, tree, all_deps): + debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) + if cur in all_deps: + debug('already handled module ' + cur) + return 0 + + deps = Set() + all_deps.add(cur) + for t in prereq_types: + debug("checking prereqisites of type " + t) + deps.update(self.read_deps_cached(cur, t)) + for d in deps: + self.add_dep_tree(d, prereq_types, tree, all_deps) + tree[cur] = deps + return len(deps) + + def calculate_order(self, order, modules, prereq_types, all_deps): + dep_tree = {} + for m in modules: + debug("--- adding dependency tree of module " + m) + self.add_dep_tree(m, prereq_types, dep_tree, all_deps) + while len(all_deps): + for d in all_deps: + if not len(dep_tree[d]): + break + else: + print(all_deps) + raise Exception("fatal: the dependencies between these modules are unresolvable") + order.append(d) + all_deps.remove(d) + for k in dep_tree.keys(): + if d in dep_tree[k]: + dep_tree[k].remove(d) + return 1 + + def run_make(self, module, target, cur_project, num_projects): + #make_cmd = "make " + target + " 2>&1" + make_cmd = [ "make", target ] + path = self.find_proj_path_cached(module) + delim_len=120 + delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path) + delim = delim + '-' * (delim_len - len(delim)) + + print(',' + delim + ' >') + os.chdir(path) + p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) + for line in iter(p.stdout.readline, ''): + sys.stdout.write('| ' + line) # avoid extra newlines from print() + sys.stdout.flush() + p.wait() + print('`' + delim + ' <') + if p.returncode: + print(make_cmd + ' failed') + raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root) + + def run_make_on_modules(self, modules, order, target): + cur_project = 0 + num_projects = len(order) + if target in ["clean", "distclean"]: + for m in reversed(order): + cur_project += 1 + self.run_make(m, target, cur_project, num_projects) + if m in modules: + modules.remove(m) + if not len(modules): + print("all modules cleaned") + return + else: + for m in order: + cur_project += 1 + self.run_make(m, target, cur_project, num_projects) + + def run(self, args_): + all_deps = Set() + visited = {} + glob_order = [] + projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" + + # -- parse command line + parser = argparse.ArgumentParser(description='janware software project build tool') + parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory') + parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + parser.add_argument('--debug', '-d', action='store_true', + default=False, help='Output debug information to stderr') + parser.add_argument('--dry-run', '-n', action='store_true', + default=False, help='Don\'t build anything, just print what would be done.') + parser.add_argument('--ignore-deps', '-I', action='store_true', + default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') + parser.add_argument('target', default='all', help='Build target') + parser.add_argument('modules', nargs='+', default='', help='Modules to be built') + + args=parser.parse_args(args_) + + debug("----------------------------------------- running ", ' '.join(args_)) + + modules=args.modules + exclude=args.exclude.split() + target=args.target + + env_exclude=os.getenv('BUILD_EXCLUDE', '') + if len(env_exclude): + print("exluding modules from environment: " + env_exclude) + exclude += " " + env_exclude + + # -- build + if target != 'order': + print("calculating order for modules ... ") + order = [] + + glob_prereq_types = [ "build" ] + if re.match("pkg-.*", target) is not None: + glob_prereq_types = [ "build", "run", "release", "devel" ] + print("using prerequisite types " + ' '.join(glob_prereq_types)) + + self.calculate_order(order, modules, glob_prereq_types, all_deps) + if args.ignore_deps: + order = [m for m in order if m in args.modules] + order = [m for m in order if m not in exclude] + if target == 'order': + print(' '.join(order)) + exit(0) + + cur_project = 0 + print("Building target %s in %d projects:" % (target, len(order))) + for m in order: + cur_project += 1 + print(" %3d %s" % (cur_project, m)) + + if args.dry_run: + exit(0) + + self.run_make_on_modules(modules, order, target) + + print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) @@ -374,6 +555,10 @@ def commands(): def cmd_commands(args_): print(commands()) +def cmd_build(args_): + build = Build() + build.run(args_) + def cmd_test(args_): parser = argparse.ArgumentParser(description='Test') parser.add_argument('blah', default='', help='The blah argument')