#!/usr/bin/python2 -u # -*- coding: iso-8859-15 -*- from __future__ import print_function import os import sys import dircache import linecache import fileinput import glob import subprocess from sets import Set import pwd import argparse import time import datetime import re from os.path import expanduser def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) def build_cache_func(func, args): global build_cache d = build_cache depth = 0 keys = [ func.__name__ ] + args l = len(keys) for k in keys: if k is None: k = 'None' depth += 1 #debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) if k in d: if l == depth: return d[k] d = d[k] continue if l == depth: r = func(*args) d[k] = r return r d[k] = {} d = d[k] raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) def build_find_proj_path(name): name=name.replace("dspider-", "") search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] for sub in search_path: path=projs_root + "/" + sub + "/" + name if os.path.exists(path): return os.path.abspath(path) raise Exception("module " + name + " not found below " + projs_root) def build_find_proj_path_cached(name): return build_cache_func(build_find_proj_path, [ name ]) def build_read_deps(cur, prereq_type): # dep cache doesn't make a difference at all if prereq_type in dep_cache: if cur in dep_cache[prereq_type]: return dep_cache[prereq_type][cur] else: dep_cache[prereq_type] = {} cmd = projects_py + " prereq " + prereq_type + " " + cur debug('running', cmd) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p.wait() if p.returncode: raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd) r = Set() pattern = re.compile(r'.*') # might be useful at a later point, currently pointless for line in iter(p.stdout.readline,''): debug(cmd + ' returned: ', line) if not pattern.match(line): continue for d in line.split(): r.add(d) if cur in r: r.remove(cur) debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) dep_cache[prereq_type][cur] = r return r def build_read_deps_cached(cur, prereq_type): return build_cache_func(build_read_deps, [ cur, prereq_type ]) def build_add_dep_tree(cur, prereq_types, tree, all_deps): debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) if cur in all_deps: debug('already handled module ' + cur) return 0 deps = Set() all_deps.add(cur) for t in prereq_types: debug("checking prereqisites of type " + t) deps.update(build_read_deps_cached(cur, t)) for d in deps: build_add_dep_tree(d, prereq_types, tree, all_deps) tree[cur] = deps return len(deps) def build_calculate_order(order, modules, prereq_types, all_deps): dep_tree = {} for m in modules: debug("--- adding dependency tree of module " + m) build_add_dep_tree(m, prereq_types, dep_tree, all_deps) while len(all_deps): for d in all_deps: if not len(dep_tree[d]): break else: print(all_deps) raise Exception("fatal: the dependencies between these modules are unresolvable") order.append(d) all_deps.remove(d) for k in dep_tree.keys(): if d in dep_tree[k]: dep_tree[k].remove(d) return 1 def build_run_make(module, target, cur_project, num_projects): #make_cmd = "make " + target + " 2>&1" make_cmd = [ "make", target ] path = build_find_proj_path_cached(module) delim_len=120 delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path) delim = delim + '-' * (delim_len - len(delim)) print(',' + delim + ' >') os.chdir(path) p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) for line in iter(p.stdout.readline, ''): sys.stdout.write('| ' + line) # avoid extra newlines from print() sys.stdout.flush() p.wait() print('`' + delim + ' <') if p.returncode: print(make_cmd + ' failed') raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root) def build_run_make_on_modules(modules, order, target): cur_project = 0 num_projects = len(order) if target in ["clean", "distclean"]: for m in reversed(order): cur_project += 1 build_run_make(m, target, cur_project, num_projects) if m in modules: modules.remove(m) if not len(modules): print("all modules cleaned") return else: for m in order: cur_project += 1 build_run_make(m, target, cur_project, num_projects) def cmd_build(args_): all_deps = Set() visited = {} glob_order = [] projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" # -- parse command line parser = argparse.ArgumentParser(description='janware software project build tool') parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory') parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') parser.add_argument('--debug', '-d', action='store_true', default=False, help='Output debug information to stderr') parser.add_argument('--dry-run', '-n', action='store_true', default=False, help='Don\'t build anything, just print what would be done.') parser.add_argument('--ignore-deps', '-I', action='store_true', default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') parser.add_argument('target', default='all', help='Build target') parser.add_argument('modules', nargs='+', default='', help='Modules to be built') args=parser.parse_args(args_) debug("----------------------------------------- running ", ' '.join(args_)) modules=args.modules exclude=args.exclude.split() target=args.target env_exclude=os.getenv('BUILD_EXCLUDE', '') if len(env_exclude): print("exluding modules from environment: " + env_exclude) exclude += " " + env_exclude # -- build if target != 'order': print("calculating order for modules ... ") order = [] glob_prereq_types = [ "build" ] if re.match("pkg-.*", target) is not None: glob_prereq_types = [ "build", "run", "release", "devel" ] print("using prerequisite types " + ' '.join(glob_prereq_types)) build_calculate_order(order, modules, glob_prereq_types, all_deps) if args.ignore_deps: order = [m for m in order if m in args.modules] order = [m for m in order if m not in exclude] if target == 'order': print(' '.join(order)) exit(0) cur_project = 0 print("Building target %s in %d projects:" % (target, len(order))) for m in order: cur_project += 1 print(" %3d %s" % (cur_project, m)) if args.dry_run: exit(0) build_run_make_on_modules(modules, order, target) print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # -------------------------------------------------------------------- here we go global_args = [] #cache = ResultCache() skip = 0 for a in sys.argv[1::]: global_args.append(a) if a in [ '--prefix', '-p', '--topdir', '-t' ]: skip = 1 continue if skip > 0: skip = skip -1 continue if a[0] != '-': break topdir = None top_name = None parser = argparse.ArgumentParser(description='Project metadata evaluation') parser.add_argument('cmd', default='', help='Command') parser.add_argument('--debug', '-d', action='store_true', default=False, help='Output debug information to stderr') parser.add_argument('--topdir', '-t', nargs='?', help='Project Path') parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") + '/local/src/jw.dev/proj', help='Projects Path Prefix') parser.add_argument('arg', nargs='*', help='Command arguments') args=parser.parse_args(global_args) debug("----------------------------------------- running ", ' '.join(sys.argv)) projs_root = args.prefix if args.topdir: topdir = args.topdir top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name']) if not top_name: top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir))) # ---- global variables build_cache = {} dep_cache = {} my_dir=os.path.dirname(os.path.realpath(__file__)) projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_')) cmd(sys.argv[(len(global_args) + 1)::])