#!/usr/bin/python -u # -*- coding: iso-8859-15 -*- from __future__ import print_function import os import sys import dircache import linecache import fileinput import glob import subprocess from sets import Set import pwd import argparse import time import datetime import re def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) def find_proj_path(name): name=name.replace("dspider-", "") for sub in search_path: path=proj_base + "/" + sub + "/" + name if os.path.exists(path): return os.path.abspath(path) raise Exception("module " + name + " not found below " + proj_base) def read_deps(cur, prereq_type): # dep cache doesn't make a difference at all if prereq_type in dep_cache: if cur in dep_cache[prereq_type]: return dep_cache[prereq_type][cur] else: dep_cache[prereq_type] = {} cmd = projects_py + " prereq " + prereq_type + " " + cur p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p.wait() if p.returncode: raise Exception("failed to get " + prereq_type + " prerequisites from " + path) r = Set() pattern = re.compile(r'.*') # might be useful at a later point, currently pointless for line in iter(p.stdout.readline,''): debug(cmd + ' returned: ', line) if not pattern.match(line): continue for d in line.split(): r.add(d) if cur in r: r.remove(cur) debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) dep_cache[prereq_type][cur] = r return r def add_tree(cur, prereq_types): debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) if cur in all_deps: debug('already handled module ' + cur) return 0 deps = Set() all_deps.add(cur) for t in prereq_types: debug("checking prereqisites of type " + t) deps.update(read_deps(cur, t)) for d in deps: add_tree(d, prereq_types) dep_tree[cur] = deps return len(deps) def calculate_order(order, modules, prereq_types): for m in modules: debug("--- adding dependency tree of module " + m) add_tree(m, prereq_types) while len(all_deps): for d in all_deps: if not len(dep_tree[d]): break else: print(all_deps) raise Exception("fatal: the dependencies between these modules are unresolvable") order.append(d) all_deps.remove(d) for k in dep_tree.keys(): if d in dep_tree[k]: dep_tree[k].remove(d) return 1 def run_make(module, target): global cur_project cur_project=cur_project+1 make_cmd = "make " + target + " 2>&1" path = find_proj_path(module) delim_len=120 delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path) delim = delim + '-' * (delim_len - len(delim)) print(',' + delim + ' >') os.chdir(path) p = subprocess.Popen(make_cmd, shell=True, stdout=subprocess.PIPE) for line in iter(p.stdout.readline, ''): sys.stdout.write('| ' + line) # avoid extra newlines from print() sys.stdout.flush() p.wait() print('`' + delim + ' <') if p.returncode: print(make_cmd + ' failed') raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base) def build(modules, order, target): if target in ["clean", "distclean"]: for m in reversed(order): run_make(m, target) if m in modules: modules.remove(m) if not len(modules): print("all modules cleaned") return else: for m in order: run_make(m, target) # ------------ here we go all_deps = Set() visited = {} dep_tree = {} glob_order = [] proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] dep_cache = {} my_dir=os.path.dirname(os.path.realpath(__file__)) # -- parse command line parser = argparse.ArgumentParser(description='janware software project build tool') parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory') parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') parser.add_argument('--debug', '-d', action='store_true', default=False, help='Output debug information to stderr') parser.add_argument('--dry-run', '-n', action='store_true', default=False, help='Don\'t build anything, just print what would be done.') parser.add_argument('--ignore-deps', '-I', action='store_true', default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') parser.add_argument('target', default='all', help='Build target') parser.add_argument('modules', nargs='+', default='', help='Modules to be built') args=parser.parse_args() debug("----------------------------------------- running ", ' '.join(sys.argv)) modules=args.modules exclude=args.exclude.split() proj_base=args.base target=args.target projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base env_exclude=os.getenv('BUILD_EXCLUDE', '') if len(env_exclude): print("exluding modules from environment: " + env_exclude) exclude += " " + env_exclude # -- build if target != 'order': print("calculating order for modules ... ") order = [] glob_prereq_types = [ "build" ] if re.match("pkg-.*", target) is not None: glob_prereq_types = [ "build", "run", "release", "devel" ] print("using prerequisite types " + ' '.join(glob_prereq_types)) calculate_order(order, modules, glob_prereq_types) if args.ignore_deps: order = [m for m in order if m in args.modules] order = [m for m in order if m not in exclude] if target == 'order': print(' '.join(order)) exit(0) cur_project=0 print("Building target %s in %d projects:" % (target, len(order))) for m in order: cur_project=cur_project+1 print(" %3d %s" % (cur_project, m)) cur_project=0 if args.dry_run: exit(0) build(modules, order, target) print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))