diff --git a/make/projects-dir.mk b/make/projects-dir.mk index 71ef52b2..d61560d1 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -96,7 +96,7 @@ endif # ------------ external programs II PGIT = CLONE_PROJECTS="$(PROJECTS)" /bin/bash $(MOD_SCRIPT_DIR)/pgit.sh -BUILD_PY = python2 $(MOD_SCRIPT_DIR)/build.py -b $(shell pwd) $(BUILD_PY_EXTRA_ARGS) +BUILD_PY = python2 $(MOD_SCRIPT_DIR)/build.py --prefix $(shell pwd) build $(BUILD_PY_EXTRA_ARGS) PROJECTS_PY = python2 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_ARGS) PURGE_SH = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/purge-stale-projects.sh /opt/jw-build/bin/purge-stale-projects.sh) purge-not-found) RELEASE_SH = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/build-release.sh /opt/jw-build/bin/build-release.sh) release-not-found) diff --git a/scripts/build.py b/scripts/build.py index c3ad525b..c502be36 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -15,14 +15,15 @@ import argparse import time import datetime import re +from os.path import expanduser def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) -def cache_func(func, args): - global cache - d = cache +def build_cache_func(func, args): + global build_cache + d = build_cache depth = 0 keys = [ func.__name__ ] + args l = len(keys) @@ -44,18 +45,19 @@ def cache_func(func, args): d = d[k] raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) -def find_proj_path(name): +def build_find_proj_path(name): name=name.replace("dspider-", "") + search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] for sub in search_path: - path=proj_base + "/" + sub + "/" + name + path=projs_root + "/" + sub + "/" + name if os.path.exists(path): return os.path.abspath(path) - raise Exception("module " + name + " not found below " + proj_base) + raise Exception("module " + name + " not found below " + projs_root) -def find_proj_path_cached(name): - return cache_func(find_proj_path, [ name ]) +def build_find_proj_path_cached(name): + return build_cache_func(build_find_proj_path, [ name ]) -def read_deps(cur, prereq_type): +def build_read_deps(cur, prereq_type): # dep cache doesn't make a difference at all if prereq_type in dep_cache: if cur in dep_cache[prereq_type]: @@ -82,10 +84,10 @@ def read_deps(cur, prereq_type): dep_cache[prereq_type][cur] = r return r -def read_deps_cached(cur, prereq_type): - return cache_func(read_deps, [ cur, prereq_type ]) +def build_read_deps_cached(cur, prereq_type): + return build_cache_func(build_read_deps, [ cur, prereq_type ]) -def add_tree(cur, prereq_types): +def build_add_tree(cur, prereq_types, all_deps): debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) if cur in all_deps: debug('already handled module ' + cur) @@ -95,16 +97,16 @@ def add_tree(cur, prereq_types): all_deps.add(cur) for t in prereq_types: debug("checking prereqisites of type " + t) - deps.update(read_deps_cached(cur, t)) + deps.update(build_read_deps_cached(cur, t)) for d in deps: - add_tree(d, prereq_types) + build_add_tree(d, prereq_types, all_deps) dep_tree[cur] = deps return len(deps) -def calculate_order(order, modules, prereq_types): +def build_calculate_order(order, modules, prereq_types, all_deps): for m in modules: debug("--- adding dependency tree of module " + m) - add_tree(m, prereq_types) + build_add_tree(m, prereq_types, all_deps) while len(all_deps): for d in all_deps: if not len(dep_tree[d]): @@ -119,14 +121,12 @@ def calculate_order(order, modules, prereq_types): dep_tree[k].remove(d) return 1 -def run_make(module, target): - global cur_project - cur_project=cur_project+1 +def build_run_make(module, target, cur_project, num_projects): #make_cmd = "make " + target + " 2>&1" make_cmd = [ "make", target ] - path = find_proj_path_cached(module) + path = build_find_proj_path_cached(module) delim_len=120 - delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path) + delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path) delim = delim + '-' * (delim_len - len(delim)) print(',' + delim + ' >') @@ -139,12 +139,15 @@ def run_make(module, target): print('`' + delim + ' <') if p.returncode: print(make_cmd + ' failed') - raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base) + raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root) -def build(modules, order, target): +def build_run_make_on_modules(modules, order, target): + cur_project = 0 + num_projects = len(order) if target in ["clean", "distclean"]: for m in reversed(order): - run_make(m, target) + cur_project += 1 + build_run_make(m, target, cur_project, num_projects) if m in modules: modules.remove(m) if not len(modules): @@ -152,78 +155,118 @@ def build(modules, order, target): return else: for m in order: - run_make(m, target) + cur_project += 1 + build_run_make(m, target, cur_project, num_projects) -# ------------ here we go +def cmd_build(args_): + all_deps = Set() + visited = {} + glob_order = [] + projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" -all_deps = Set() -cache = {} -visited = {} -dep_tree = {} -glob_order = [] -proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" -search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] -dep_cache = {} -my_dir=os.path.dirname(os.path.realpath(__file__)) + # -- parse command line + parser = argparse.ArgumentParser(description='janware software project build tool') + parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory') + parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + parser.add_argument('--debug', '-d', action='store_true', + default=False, help='Output debug information to stderr') + parser.add_argument('--dry-run', '-n', action='store_true', + default=False, help='Don\'t build anything, just print what would be done.') + parser.add_argument('--ignore-deps', '-I', action='store_true', + default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') + parser.add_argument('target', default='all', help='Build target') + parser.add_argument('modules', nargs='+', default='', help='Modules to be built') -# -- parse command line -parser = argparse.ArgumentParser(description='janware software project build tool') -parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory') -parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + args=parser.parse_args(args_) + + debug("----------------------------------------- running ", ' '.join(args_)) + + modules=args.modules + exclude=args.exclude.split() + target=args.target + + env_exclude=os.getenv('BUILD_EXCLUDE', '') + if len(env_exclude): + print("exluding modules from environment: " + env_exclude) + exclude += " " + env_exclude + + # -- build + if target != 'order': + print("calculating order for modules ... ") + order = [] + + glob_prereq_types = [ "build" ] + if re.match("pkg-.*", target) is not None: + glob_prereq_types = [ "build", "run", "release", "devel" ] + print("using prerequisite types " + ' '.join(glob_prereq_types)) + + build_calculate_order(order, modules, glob_prereq_types, all_deps) + if args.ignore_deps: + order = [m for m in order if m in args.modules] + order = [m for m in order if m not in exclude] + if target == 'order': + print(' '.join(order)) + exit(0) + + cur_project = 0 + print("Building target %s in %d projects:" % (target, len(order))) + for m in order: + cur_project += 1 + print(" %3d %s" % (cur_project, m)) + + if args.dry_run: + exit(0) + + build_run_make_on_modules(modules, order, target) + + print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + +# -------------------------------------------------------------------- here we go + +global_args = [] +#cache = ResultCache() + +skip = 0 +for a in sys.argv[1::]: + global_args.append(a) + if a in [ '--prefix', '-p', '--topdir', '-t' ]: + skip = 1 + continue + if skip > 0: + skip = skip -1 + continue + if a[0] != '-': + break + +topdir = None +top_name = None + +parser = argparse.ArgumentParser(description='Project metadata evaluation') +parser.add_argument('cmd', default='', help='Command') parser.add_argument('--debug', '-d', action='store_true', default=False, help='Output debug information to stderr') -parser.add_argument('--dry-run', '-n', action='store_true', - default=False, help='Don\'t build anything, just print what would be done.') -parser.add_argument('--ignore-deps', '-I', action='store_true', - default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') -parser.add_argument('target', default='all', help='Build target') -parser.add_argument('modules', nargs='+', default='', help='Modules to be built') - -args=parser.parse_args() +parser.add_argument('--topdir', '-t', nargs='?', help='Project Path') +parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") + + '/local/src/jw.dev/proj', help='Projects Path Prefix') +parser.add_argument('arg', nargs='*', help='Command arguments') +args=parser.parse_args(global_args) debug("----------------------------------------- running ", ' '.join(sys.argv)) -modules=args.modules -exclude=args.exclude.split() -proj_base=args.base -target=args.target +projs_root = args.prefix -projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") +if args.topdir: + topdir = args.topdir + top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name']) + if not top_name: + top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir))) -env_exclude=os.getenv('BUILD_EXCLUDE', '') -if len(env_exclude): - print("exluding modules from environment: " + env_exclude) - exclude += " " + env_exclude - -# -- build -if target != 'order': - print("calculating order for modules ... ") -order = [] - -glob_prereq_types = [ "build" ] -if re.match("pkg-.*", target) is not None: - glob_prereq_types = [ "build", "run", "release", "devel" ] - print("using prerequisite types " + ' '.join(glob_prereq_types)) - -calculate_order(order, modules, glob_prereq_types) -if args.ignore_deps: - order = [m for m in order if m in args.modules] -order = [m for m in order if m not in exclude] -if target == 'order': - print(' '.join(order)) - exit(0) - -cur_project=0 -print("Building target %s in %d projects:" % (target, len(order))) -for m in order: - cur_project=cur_project+1 - print(" %3d %s" % (cur_project, m)) -cur_project=0 - -if args.dry_run: - exit(0) - -build(modules, order, target) - -print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) +# ---- global variables +build_cache = {} +dep_cache = {} +my_dir=os.path.dirname(os.path.realpath(__file__)) +projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") +dep_tree = {} +cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_')) +cmd(sys.argv[(len(global_args) + 1)::])