From a629faab93ed88fa70152ed5c102d18c7ec1180e Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Wed, 12 Jun 2019 11:26:15 +0000 Subject: [PATCH] projects.py: Dissolve class Build into class Projects This commit is the next code cleanup step. Needs more testing and more fixing. Signed-off-by: Jan Lindemann --- scripts/projects.py | 410 ++++++++++++++++++++++---------------------- 1 file changed, 203 insertions(+), 207 deletions(-) diff --git a/scripts/projects.py b/scripts/projects.py index e4678875..9caafa12 100644 --- a/scripts/projects.py +++ b/scripts/projects.py @@ -1,4 +1,8 @@ #!/usr/bin/python3 -u +# -*- coding: utf-8 -*- +# +# This source code file is a merge of various build tools and a horrible mess. +# from __future__ import print_function import os @@ -58,211 +62,6 @@ class ResultCache(object): #d = d[k] raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) -# ----------------------------------------------------------------- class Build - -class Build(object): - - def __init__(self, projects): - self.projects = projects - pass - - def find_proj_path(self, name): - name = name.replace("dspider-", "") - search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] - for sub in search_path: - path = self.projects.projs_root + "/" + sub + "/" + name - if os.path.exists(path): - return os.path.abspath(path) - raise Exception("module " + name + " not found below " + self.projects.projs_root) - - def find_proj_path_cached(self, name): - return self.projects.res_cache.run(self.find_proj_path, [ name ]) - - def read_deps(self, cur, prereq_type): - # dep cache doesn't make a difference at all - if prereq_type in self.projects.dep_cache: - if cur in self.projects.dep_cache[prereq_type]: - return self.projects.dep_cache[prereq_type][cur] - else: - self.projects.dep_cache[prereq_type] = {} - - if True: - r = self.projects.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'], - prereq_type, scope = 2, add_self=False, names_only=True) - self.projects.debug('prerequisites = ' + ' '.join(r)) - else: # legacy from build.py - projects_py = sys.executable + " " + self.projects.my_dir + "/projects.py --prefix " + self.projects.projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") - cmd = projects_py + " prereq " + prereq_type + " " + cur - self.projects.debug('running', cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - p.wait() - if p.returncode: - raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) - r = set() - pattern = re.compile(r'.*') # might be useful at a later point, currently pointless - for line in iter(p.stdout.readline, b''): - line = line.decode(sys.stdout.encoding) - self.projects.debug(cmd + ' returned: ', line) - if not pattern.match(line): - continue - for d in line.split(): - r.add(d) - if cur in r: - r.remove(cur) - self.projects.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) - self.projects.dep_cache[prereq_type][cur] = r - return r - - def read_deps_cached(self, cur, prereq_type): - return self.projects.res_cache.run(self.read_deps, [ cur, prereq_type ]) - - def add_dep_tree(self, cur, prereq_types, tree, all_deps): - self.projects.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) - if cur in all_deps: - self.projects.debug('already handled module ' + cur) - return 0 - - deps = set() - all_deps.add(cur) - for t in prereq_types: - self.projects.debug("checking prereqisites of type " + t) - deps.update(self.read_deps_cached(cur, t)) - for d in deps: - self.add_dep_tree(d, prereq_types, tree, all_deps) - tree[cur] = deps - return len(deps) - - def calculate_order(self, order, modules, prereq_types): - all_deps = set() - dep_tree = {} - for m in modules: - self.projects.debug("--- adding dependency tree of module " + m) - self.add_dep_tree(m, prereq_types, dep_tree, all_deps) - while len(all_deps): - for d in all_deps: - if not len(dep_tree[d]): - break - else: - print(all_deps) - raise Exception("fatal: the dependencies between these modules are unresolvable") - order.append(d) - all_deps.remove(d) - for k in dep_tree.keys(): - if d in dep_tree[k]: - dep_tree[k].remove(d) - return 1 - - def run_make(self, module, target, cur_project, num_projects): - #make_cmd = "make " + target + " 2>&1" - make_cmd = [ "make", target ] - path = self.find_proj_path_cached(module) - delim_len=120 - delim='---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path) - delim = delim + '-' * (delim_len - len(delim)) - - print(',' + delim + ' >') - - patt = self.projects.is_excluded_from_build(module) - if patt is not None: - print('| Configured to skip build on platform >' + patt + '<') - print('`' + delim + ' <') - return - - os.chdir(path) - p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) - for line in iter(p.stdout.readline, b''): - line = line.decode(sys.stdout.encoding) - sys.stdout.write('| ' + line) # avoid extra newlines from print() - sys.stdout.flush() - p.wait() - print('`' + delim + ' <') - if p.returncode: - print(make_cmd + ' failed') - raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projects.projs_root) - - def run_make_on_modules(self, modules, order, target): - cur_project = 0 - num_projects = len(order) - if target in ["clean", "distclean"]: - for m in reversed(order): - cur_project += 1 - self.run_make(m, target, cur_project, num_projects) - if m in modules: - modules.remove(m) - if not len(modules): - print("all modules cleaned") - return - else: - for m in order: - cur_project += 1 - self.run_make(m, target, cur_project, num_projects) - - def run(self, args_): - visited = {} - glob_order = [] - self.projects.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" - - # -- parse command line - parser = argparse.ArgumentParser(description='janware software project build tool') - parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') - parser.add_argument('--debug', '-d', action='store_true', - default=False, help='Output debug information to stderr') - parser.add_argument('--dry-run', '-n', action='store_true', - default=False, help='Don\'t build anything, just print what would be done.') - parser.add_argument('--build-order', '-O', action='store_true', - default=False, help='Don\'t build anything, just print the build order.') - parser.add_argument('--ignore-deps', '-I', action='store_true', - default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') - parser.add_argument('target', default='all', help='Build target') - parser.add_argument('modules', nargs='+', default='', help='Modules to be built') - - args=parser.parse_args(args_) - if args.debug: - self.projects.opt_debug = True - - self.projects.debug("----------------------------------------- running ", ' '.join(args_)) - - modules=args.modules - exclude=args.exclude.split() - target=args.target - - env_exclude=os.getenv('BUILD_EXCLUDE', '') - if len(env_exclude): - print("exluding modules from environment: " + env_exclude) - exclude += " " + env_exclude - - # -- build - order = [] - - glob_prereq_types = [ "build" ] - if re.match("pkg-.*", target) is not None: - glob_prereq_types = [ "build", "run", "release", "devel" ] - - if target != 'order' and not args.build_order: - print("using prerequisite types " + ' '.join(glob_prereq_types)) - print("calculating order for modules ... ") - - self.calculate_order(order, modules, glob_prereq_types) - if args.ignore_deps: - order = [m for m in order if m in args.modules] - order = [m for m in order if m not in exclude] - if target == 'order' or args.build_order: - print(' '.join(order)) - exit(0) - - cur_project = 0 - print("Building target %s in %d projects:" % (target, len(order))) - for m in order: - cur_project += 1 - print(" %3d %s" % (cur_project, m)) - - if args.dry_run: - exit(0) - - self.run_make_on_modules(modules, order, target) - - print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - # ----------------------------------------------------------------- class Projects class Projects(object): @@ -651,8 +450,205 @@ class Projects(object): print(self.commands()) def cmd_build(self, args_): - build = Build(self) - build.run(args_) + + def find_proj_path(name): + name = name.replace("dspider-", "") + search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] + for sub in search_path: + path = self.projs_root + "/" + sub + "/" + name + if os.path.exists(path): + return os.path.abspath(path) + raise Exception("module " + name + " not found below " + self.projs_root) + + def find_proj_path_cached(name): + return self.res_cache.run(find_proj_path, [ name ]) + + def read_deps(cur, prereq_type): + # dep cache doesn't make a difference at all + if prereq_type in self.dep_cache: + if cur in self.dep_cache[prereq_type]: + return self.dep_cache[prereq_type][cur] + else: + self.dep_cache[prereq_type] = {} + + if True: + r = self.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'], + prereq_type, scope = 2, add_self=False, names_only=True) + self.debug('prerequisites = ' + ' '.join(r)) + else: # legacy from build.py + projects_py = sys.executable + " " + self.my_dir + "/projects.py --prefix " + self.projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") + cmd = projects_py + " prereq " + prereq_type + " " + cur + self.debug('running', cmd) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + p.wait() + if p.returncode: + raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) + r = set() + pattern = re.compile(r'.*') # might be useful at a later point, currently pointless + for line in iter(p.stdout.readline, b''): + line = line.decode(sys.stdout.encoding) + self.debug(cmd + ' returned: ', line) + if not pattern.match(line): + continue + for d in line.split(): + r.add(d) + if cur in r: + r.remove(cur) + self.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) + self.dep_cache[prereq_type][cur] = r + return r + + def read_deps_cached(cur, prereq_type): + return self.res_cache.run(read_deps, [ cur, prereq_type ]) + + def add_dep_tree(cur, prereq_types, tree, all_deps): + self.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) + if cur in all_deps: + self.debug('already handled module ' + cur) + return 0 + + deps = set() + all_deps.add(cur) + for t in prereq_types: + self.debug("checking prereqisites of type " + t) + deps.update(read_deps_cached(cur, t)) + for d in deps: + add_dep_tree(d, prereq_types, tree, all_deps) + tree[cur] = deps + return len(deps) + + def calculate_order(order, modules, prereq_types): + all_deps = set() + dep_tree = {} + for m in modules: + self.debug("--- adding dependency tree of module " + m) + add_dep_tree(m, prereq_types, dep_tree, all_deps) + while len(all_deps): + for d in all_deps: + if not len(dep_tree[d]): + break + else: + print(all_deps) + raise Exception("fatal: the dependencies between these modules are unresolvable") + order.append(d) + all_deps.remove(d) + for k in dep_tree.keys(): + if d in dep_tree[k]: + dep_tree[k].remove(d) + return 1 + + def run_make(module, target, cur_project, num_projects): + #make_cmd = "make " + target + " 2>&1" + make_cmd = [ "make", target ] + path = find_proj_path_cached(module) + delim_len=120 + delim='---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path) + delim = delim + '-' * (delim_len - len(delim)) + + print(',' + delim + ' >') + + patt = self.is_excluded_from_build(module) + if patt is not None: + print('| Configured to skip build on platform >' + patt + '<') + print('`' + delim + ' <') + return + + os.chdir(path) + p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) + for line in iter(p.stdout.readline, b''): + line = line.decode(sys.stdout.encoding) + sys.stdout.write('| ' + line) # avoid extra newlines from print() + sys.stdout.flush() + p.wait() + print('`' + delim + ' <') + if p.returncode: + print(make_cmd + ' failed') + raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projs_root) + + def run_make_on_modules(modules, order, target): + cur_project = 0 + num_projects = len(order) + if target in ["clean", "distclean"]: + for m in reversed(order): + cur_project += 1 + run_make(m, target, cur_project, num_projects) + if m in modules: + modules.remove(m) + if not len(modules): + print("all modules cleaned") + return + else: + for m in order: + cur_project += 1 + run_make(m, target, cur_project, num_projects) + + def run(args_): + visited = {} + glob_order = [] + self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" + + # -- parse command line + parser = argparse.ArgumentParser(description='janware software project build tool') + parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + parser.add_argument('--debug', '-d', action='store_true', + default=False, help='Output debug information to stderr') + parser.add_argument('--dry-run', '-n', action='store_true', + default=False, help='Don\'t build anything, just print what would be done.') + parser.add_argument('--build-order', '-O', action='store_true', + default=False, help='Don\'t build anything, just print the build order.') + parser.add_argument('--ignore-deps', '-I', action='store_true', + default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') + parser.add_argument('target', default='all', help='Build target') + parser.add_argument('modules', nargs='+', default='', help='Modules to be built') + + args=parser.parse_args(args_) + if args.debug: + self.opt_debug = True + + self.debug("----------------------------------------- running ", ' '.join(args_)) + + modules=args.modules + exclude=args.exclude.split() + target=args.target + + env_exclude=os.getenv('BUILD_EXCLUDE', '') + if len(env_exclude): + print("exluding modules from environment: " + env_exclude) + exclude += " " + env_exclude + + # -- build + order = [] + + glob_prereq_types = [ "build" ] + if re.match("pkg-.*", target) is not None: + glob_prereq_types = [ "build", "run", "release", "devel" ] + + if target != 'order' and not args.build_order: + print("using prerequisite types " + ' '.join(glob_prereq_types)) + print("calculating order for modules ... ") + + calculate_order(order, modules, glob_prereq_types) + if args.ignore_deps: + order = [m for m in order if m in args.modules] + order = [m for m in order if m not in exclude] + if target == 'order' or args.build_order: + print(' '.join(order)) + exit(0) + + cur_project = 0 + print("Building target %s in %d projects:" % (target, len(order))) + for m in order: + cur_project += 1 + print(" %3d %s" % (cur_project, m)) + + if args.dry_run: + exit(0) + + run_make_on_modules(modules, order, target) + + print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + run(args_) def cmd_test(self, args_): parser = argparse.ArgumentParser(description='Test')