From 4fc4fc3c622e96d4d4e8f605270c589691e1f3a0 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Wed, 7 Mar 2018 10:32:37 +0000 Subject: [PATCH] build.py: Encapsulate build functionality in class Build Signed-off-by: Jan Lindemann --- scripts/build.py | 338 ++++++++++++++++++++++++----------------------- 1 file changed, 172 insertions(+), 166 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index 8cdb35ea..6b2ce4d9 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -51,183 +51,189 @@ def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) -def build_find_proj_path(name): - name=name.replace("dspider-", "") - search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] - for sub in search_path: - path=projs_root + "/" + sub + "/" + name - if os.path.exists(path): - return os.path.abspath(path) - raise Exception("module " + name + " not found below " + projs_root) +class Build(object): -def build_find_proj_path_cached(name): - return res_cache.run(build_find_proj_path, [ name ]) + def find_proj_path(self, name): + name=name.replace("dspider-", "") + search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] + for sub in search_path: + path=projs_root + "/" + sub + "/" + name + if os.path.exists(path): + return os.path.abspath(path) + raise Exception("module " + name + " not found below " + projs_root) -def build_read_deps(cur, prereq_type): - # dep cache doesn't make a difference at all - projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") - if prereq_type in dep_cache: - if cur in dep_cache[prereq_type]: - return dep_cache[prereq_type][cur] - else: - dep_cache[prereq_type] = {} - cmd = projects_py + " prereq " + prereq_type + " " + cur - debug('running', cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - p.wait() - if p.returncode: - raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) - r = Set() - pattern = re.compile(r'.*') # might be useful at a later point, currently pointless - for line in iter(p.stdout.readline,''): - debug(cmd + ' returned: ', line) - if not pattern.match(line): - continue - for d in line.split(): - r.add(d) - if cur in r: - r.remove(cur) - debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) - dep_cache[prereq_type][cur] = r - return r + def find_proj_path_cached(self, name): + return res_cache.run(self.find_proj_path, [ name ]) -def build_read_deps_cached(cur, prereq_type): - return res_cache.run(build_read_deps, [ cur, prereq_type ]) - -def build_add_dep_tree(cur, prereq_types, tree, all_deps): - debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) - if cur in all_deps: - debug('already handled module ' + cur) - return 0 - - deps = Set() - all_deps.add(cur) - for t in prereq_types: - debug("checking prereqisites of type " + t) - deps.update(build_read_deps_cached(cur, t)) - for d in deps: - build_add_dep_tree(d, prereq_types, tree, all_deps) - tree[cur] = deps - return len(deps) - -def build_calculate_order(order, modules, prereq_types, all_deps): - dep_tree = {} - for m in modules: - debug("--- adding dependency tree of module " + m) - build_add_dep_tree(m, prereq_types, dep_tree, all_deps) - while len(all_deps): - for d in all_deps: - if not len(dep_tree[d]): - break + def read_deps(self, cur, prereq_type): + # dep cache doesn't make a difference at all + projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") + if prereq_type in dep_cache: + if cur in dep_cache[prereq_type]: + return dep_cache[prereq_type][cur] else: - print(all_deps) - raise Exception("fatal: the dependencies between these modules are unresolvable") - order.append(d) - all_deps.remove(d) - for k in dep_tree.keys(): - if d in dep_tree[k]: - dep_tree[k].remove(d) - return 1 + dep_cache[prereq_type] = {} + cmd = projects_py + " prereq " + prereq_type + " " + cur + debug('running', cmd) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + p.wait() + if p.returncode: + raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) + r = Set() + pattern = re.compile(r'.*') # might be useful at a later point, currently pointless + for line in iter(p.stdout.readline,''): + debug(cmd + ' returned: ', line) + if not pattern.match(line): + continue + for d in line.split(): + r.add(d) + if cur in r: + r.remove(cur) + debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) + dep_cache[prereq_type][cur] = r + return r -def build_run_make(module, target, cur_project, num_projects): - #make_cmd = "make " + target + " 2>&1" - make_cmd = [ "make", target ] - path = build_find_proj_path_cached(module) - delim_len=120 - delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path) - delim = delim + '-' * (delim_len - len(delim)) + def read_deps_cached(self, cur, prereq_type): + return res_cache.run(self.read_deps, [ cur, prereq_type ]) - print(',' + delim + ' >') - os.chdir(path) - p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) - for line in iter(p.stdout.readline, ''): - sys.stdout.write('| ' + line) # avoid extra newlines from print() - sys.stdout.flush() - p.wait() - print('`' + delim + ' <') - if p.returncode: - print(make_cmd + ' failed') - raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root) + def add_dep_tree(self, cur, prereq_types, tree, all_deps): + debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) + if cur in all_deps: + debug('already handled module ' + cur) + return 0 -def build_run_make_on_modules(modules, order, target): - cur_project = 0 - num_projects = len(order) - if target in ["clean", "distclean"]: - for m in reversed(order): - cur_project += 1 - build_run_make(m, target, cur_project, num_projects) - if m in modules: - modules.remove(m) - if not len(modules): - print("all modules cleaned") - return - else: + deps = Set() + all_deps.add(cur) + for t in prereq_types: + debug("checking prereqisites of type " + t) + deps.update(self.read_deps_cached(cur, t)) + for d in deps: + self.add_dep_tree(d, prereq_types, tree, all_deps) + tree[cur] = deps + return len(deps) + + def calculate_order(self, order, modules, prereq_types, all_deps): + dep_tree = {} + for m in modules: + debug("--- adding dependency tree of module " + m) + self.add_dep_tree(m, prereq_types, dep_tree, all_deps) + while len(all_deps): + for d in all_deps: + if not len(dep_tree[d]): + break + else: + print(all_deps) + raise Exception("fatal: the dependencies between these modules are unresolvable") + order.append(d) + all_deps.remove(d) + for k in dep_tree.keys(): + if d in dep_tree[k]: + dep_tree[k].remove(d) + return 1 + + def run_make(self, module, target, cur_project, num_projects): + #make_cmd = "make " + target + " 2>&1" + make_cmd = [ "make", target ] + path = self.find_proj_path_cached(module) + delim_len=120 + delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path) + delim = delim + '-' * (delim_len - len(delim)) + + print(',' + delim + ' >') + os.chdir(path) + p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) + for line in iter(p.stdout.readline, ''): + sys.stdout.write('| ' + line) # avoid extra newlines from print() + sys.stdout.flush() + p.wait() + print('`' + delim + ' <') + if p.returncode: + print(make_cmd + ' failed') + raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root) + + def run_make_on_modules(self, modules, order, target): + cur_project = 0 + num_projects = len(order) + if target in ["clean", "distclean"]: + for m in reversed(order): + cur_project += 1 + self.run_make(m, target, cur_project, num_projects) + if m in modules: + modules.remove(m) + if not len(modules): + print("all modules cleaned") + return + else: + for m in order: + cur_project += 1 + self.run_make(m, target, cur_project, num_projects) + + def run(self, args_): + all_deps = Set() + visited = {} + glob_order = [] + projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" + + # -- parse command line + parser = argparse.ArgumentParser(description='janware software project build tool') + parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory') + parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + parser.add_argument('--debug', '-d', action='store_true', + default=False, help='Output debug information to stderr') + parser.add_argument('--dry-run', '-n', action='store_true', + default=False, help='Don\'t build anything, just print what would be done.') + parser.add_argument('--ignore-deps', '-I', action='store_true', + default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') + parser.add_argument('target', default='all', help='Build target') + parser.add_argument('modules', nargs='+', default='', help='Modules to be built') + + args=parser.parse_args(args_) + + debug("----------------------------------------- running ", ' '.join(args_)) + + modules=args.modules + exclude=args.exclude.split() + target=args.target + + env_exclude=os.getenv('BUILD_EXCLUDE', '') + if len(env_exclude): + print("exluding modules from environment: " + env_exclude) + exclude += " " + env_exclude + + # -- build + if target != 'order': + print("calculating order for modules ... ") + order = [] + + glob_prereq_types = [ "build" ] + if re.match("pkg-.*", target) is not None: + glob_prereq_types = [ "build", "run", "release", "devel" ] + print("using prerequisite types " + ' '.join(glob_prereq_types)) + + self.calculate_order(order, modules, glob_prereq_types, all_deps) + if args.ignore_deps: + order = [m for m in order if m in args.modules] + order = [m for m in order if m not in exclude] + if target == 'order': + print(' '.join(order)) + exit(0) + + cur_project = 0 + print("Building target %s in %d projects:" % (target, len(order))) for m in order: cur_project += 1 - build_run_make(m, target, cur_project, num_projects) + print(" %3d %s" % (cur_project, m)) + + if args.dry_run: + exit(0) + + self.run_make_on_modules(modules, order, target) + + print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) def cmd_build(args_): - all_deps = Set() - visited = {} - glob_order = [] - projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" - - # -- parse command line - parser = argparse.ArgumentParser(description='janware software project build tool') - parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory') - parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') - parser.add_argument('--debug', '-d', action='store_true', - default=False, help='Output debug information to stderr') - parser.add_argument('--dry-run', '-n', action='store_true', - default=False, help='Don\'t build anything, just print what would be done.') - parser.add_argument('--ignore-deps', '-I', action='store_true', - default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') - parser.add_argument('target', default='all', help='Build target') - parser.add_argument('modules', nargs='+', default='', help='Modules to be built') - - args=parser.parse_args(args_) - - debug("----------------------------------------- running ", ' '.join(args_)) - - modules=args.modules - exclude=args.exclude.split() - target=args.target - - env_exclude=os.getenv('BUILD_EXCLUDE', '') - if len(env_exclude): - print("exluding modules from environment: " + env_exclude) - exclude += " " + env_exclude - - # -- build - if target != 'order': - print("calculating order for modules ... ") - order = [] - - glob_prereq_types = [ "build" ] - if re.match("pkg-.*", target) is not None: - glob_prereq_types = [ "build", "run", "release", "devel" ] - print("using prerequisite types " + ' '.join(glob_prereq_types)) - - build_calculate_order(order, modules, glob_prereq_types, all_deps) - if args.ignore_deps: - order = [m for m in order if m in args.modules] - order = [m for m in order if m not in exclude] - if target == 'order': - print(' '.join(order)) - exit(0) - - cur_project = 0 - print("Building target %s in %d projects:" % (target, len(order))) - for m in order: - cur_project += 1 - print(" %3d %s" % (cur_project, m)) - - if args.dry_run: - exit(0) - - build_run_make_on_modules(modules, order, target) - - print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + build = Build() + build.run(args_) # -------------------------------------------------------------------- here we go