build.py: Preparare build.py for merge into projects.py

Signed-off-by: Jan Lindemann <jan@janware.com>

make/projects-dir.mk

@@ -96,7 +96,7 @@ endif
 # ------------ external programs II
 
 PGIT                  = CLONE_PROJECTS="$(PROJECTS)" /bin/bash $(MOD_SCRIPT_DIR)/pgit.sh
-BUILD_PY              = python2 $(MOD_SCRIPT_DIR)/build.py -b $(shell pwd) $(BUILD_PY_EXTRA_ARGS)
+BUILD_PY              = python2 $(MOD_SCRIPT_DIR)/build.py    --prefix $(shell pwd) build $(BUILD_PY_EXTRA_ARGS)
 PROJECTS_PY           = python2 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_ARGS)
 PURGE_SH              = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/purge-stale-projects.sh /opt/jw-build/bin/purge-stale-projects.sh) purge-not-found)
 RELEASE_SH            = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/build-release.sh /opt/jw-build/bin/build-release.sh) release-not-found)

scripts/build.py

@@ -15,14 +15,15 @@ import argparse
 import time
 import datetime
 import re
+from os.path import expanduser
 
 def debug(*objs):
     if args.debug:
         print("DEBUG: ", *objs, file=sys.stderr)
 
-def cache_func(func, args):
+def build_cache_func(func, args):
-    global cache
+    global build_cache
-    d = cache
+    d = build_cache
     depth = 0
     keys = [ func.__name__ ] + args
     l = len(keys)
@@ -44,18 +45,19 @@ def cache_func(func, args):
         d = d[k]
     raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
 
-def find_proj_path(name):
+def build_find_proj_path(name):
     name=name.replace("dspider-", "")
+    search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
     for sub in search_path:
-        path=proj_base + "/" + sub + "/" + name
+        path=projs_root + "/" + sub + "/" + name
         if os.path.exists(path):
             return os.path.abspath(path)
-    raise Exception("module " + name + " not found below " + proj_base)
+    raise Exception("module " + name + " not found below " + projs_root)
 
-def find_proj_path_cached(name):
+def build_find_proj_path_cached(name):
-    return cache_func(find_proj_path, [ name ])
+    return build_cache_func(build_find_proj_path, [ name ])
 
-def read_deps(cur, prereq_type):
+def build_read_deps(cur, prereq_type):
     # dep cache doesn't make a difference at all
     if prereq_type in dep_cache:
         if cur in dep_cache[prereq_type]:
@@ -82,10 +84,10 @@ def read_deps(cur, prereq_type):
     dep_cache[prereq_type][cur] = r
     return r
 
-def read_deps_cached(cur, prereq_type):
+def build_read_deps_cached(cur, prereq_type):
-    return cache_func(read_deps, [ cur, prereq_type ])
+    return build_cache_func(build_read_deps, [ cur, prereq_type ])
 
-def add_tree(cur, prereq_types):
+def build_add_tree(cur, prereq_types, all_deps):
     debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
     if cur in all_deps:
         debug('already handled module ' + cur)
@@ -95,16 +97,16 @@ def add_tree(cur, prereq_types):
     all_deps.add(cur)
     for t in prereq_types:
         debug("checking prereqisites of type " + t)
-        deps.update(read_deps_cached(cur, t))
+        deps.update(build_read_deps_cached(cur, t))
     for d in deps:
-        add_tree(d, prereq_types)
+        build_add_tree(d, prereq_types, all_deps)
     dep_tree[cur] = deps
     return len(deps)
 
-def calculate_order(order, modules, prereq_types):
+def build_calculate_order(order, modules, prereq_types, all_deps):
     for m in modules:
         debug("--- adding dependency tree of module " + m)
-        add_tree(m, prereq_types)
+        build_add_tree(m, prereq_types, all_deps)
     while len(all_deps):
         for d in all_deps:
             if not len(dep_tree[d]):
@@ -119,14 +121,12 @@ def calculate_order(order, modules, prereq_types):
                 dep_tree[k].remove(d)
     return 1
 
-def run_make(module, target):
+def build_run_make(module, target, cur_project, num_projects):
-    global cur_project
-    cur_project=cur_project+1
     #make_cmd = "make " + target + " 2>&1"
     make_cmd = [ "make", target ]
-    path = find_proj_path_cached(module)
+    path = build_find_proj_path_cached(module)
     delim_len=120
-    delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
+    delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
     delim = delim + '-' * (delim_len - len(delim))
 
     print(',' + delim + ' >')
@@ -139,12 +139,15 @@ def run_make(module, target):
     print('`' + delim + ' <')
     if p.returncode:
         print(make_cmd + ' failed')
-        raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base)
+        raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root)
 
-def build(modules, order, target):
+def build_run_make_on_modules(modules, order, target):
+    cur_project = 0
+    num_projects = len(order)
     if target in ["clean", "distclean"]:
         for m in reversed(order):
-            run_make(m, target)
+            cur_project += 1
+            build_run_make(m, target, cur_project, num_projects)
             if m in modules:
                 modules.remove(m)
             if not len(modules):
@@ -152,78 +155,118 @@ def build(modules, order, target):
                 return
     else:
         for m in order:
-            run_make(m, target)
+            cur_project += 1
+            build_run_make(m, target, cur_project, num_projects)
 
-# ------------ here we go
+def cmd_build(args_):
+    all_deps = Set()
+    visited = {}
+    glob_order = []
+    projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
 
-all_deps = Set()
+    # -- parse command line
-cache = {}
+    parser = argparse.ArgumentParser(description='janware software project build tool')
-visited = {}
+    parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory')
-dep_tree = {}
+    parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
-glob_order = []
+    parser.add_argument('--debug', '-d', action='store_true',
-proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
+        default=False, help='Output debug information to stderr')
-search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
+    parser.add_argument('--dry-run', '-n', action='store_true',
-dep_cache = {}
+        default=False, help='Don\'t build anything, just print what would be done.')
-my_dir=os.path.dirname(os.path.realpath(__file__))
+    parser.add_argument('--ignore-deps', '-I', action='store_true',
+        default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
+    parser.add_argument('target', default='all', help='Build target')
+    parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
 
-# -- parse command line
+    args=parser.parse_args(args_)
-parser = argparse.ArgumentParser(description='janware software project build tool')
+
-parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory')
+    debug("----------------------------------------- running ", ' '.join(args_))
-parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
+
+    modules=args.modules
+    exclude=args.exclude.split()
+    target=args.target
+
+    env_exclude=os.getenv('BUILD_EXCLUDE', '')
+    if len(env_exclude):
+        print("exluding modules from environment: " + env_exclude)
+        exclude += " " + env_exclude
+
+    # -- build
+    if target != 'order':
+        print("calculating order for modules ... ")
+    order = []
+
+    glob_prereq_types = [ "build" ]
+    if re.match("pkg-.*", target) is not None:
+        glob_prereq_types = [ "build", "run", "release", "devel" ]
+        print("using prerequisite types " + ' '.join(glob_prereq_types))
+
+    build_calculate_order(order, modules, glob_prereq_types, all_deps)
+    if args.ignore_deps:
+        order = [m for m in order if m in args.modules]
+    order = [m for m in order if m not in exclude]
+    if target == 'order':
+        print(' '.join(order))
+        exit(0)
+
+    cur_project = 0
+    print("Building target %s in %d projects:" % (target, len(order)))
+    for m in order:
+        cur_project += 1
+        print("  %3d %s" % (cur_project, m))
+
+    if args.dry_run:
+        exit(0)
+
+    build_run_make_on_modules(modules, order, target)
+
+    print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+# -------------------------------------------------------------------- here we go
+
+global_args = []
+#cache = ResultCache()
+
+skip = 0
+for a in sys.argv[1::]:
+    global_args.append(a)
+    if a in [ '--prefix', '-p', '--topdir', '-t' ]:
+        skip = 1
+        continue
+    if skip > 0:
+        skip = skip -1
+        continue
+    if a[0] != '-':
+        break
+
+topdir = None
+top_name = None
+
+parser = argparse.ArgumentParser(description='Project metadata evaluation')
+parser.add_argument('cmd', default='', help='Command')
 parser.add_argument('--debug', '-d', action='store_true',
     default=False, help='Output debug information to stderr')
-parser.add_argument('--dry-run', '-n', action='store_true',
+parser.add_argument('--topdir', '-t', nargs='?', help='Project Path')
-    default=False, help='Don\'t build anything, just print what would be done.')
+parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") +
-parser.add_argument('--ignore-deps', '-I', action='store_true',
+    '/local/src/jw.dev/proj', help='Projects Path Prefix')
-    default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
+parser.add_argument('arg', nargs='*', help='Command arguments')
-parser.add_argument('target', default='all', help='Build target')
+args=parser.parse_args(global_args)
-parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
-
-args=parser.parse_args()
 
 debug("----------------------------------------- running ", ' '.join(sys.argv))
 
-modules=args.modules
+projs_root = args.prefix
-exclude=args.exclude.split()
-proj_base=args.base
-target=args.target
 
-projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
+if args.topdir:
+    topdir = args.topdir
+    top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
+    if not top_name:
+        top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))
 
-env_exclude=os.getenv('BUILD_EXCLUDE', '')
+# ---- global variables
-if len(env_exclude):
+build_cache = {}
-    print("exluding modules from environment: " + env_exclude)
+dep_cache = {}
-    exclude += " " + env_exclude
+my_dir=os.path.dirname(os.path.realpath(__file__))
-
+projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
-# -- build
+dep_tree = {}
-if target != 'order':
-    print("calculating order for modules ... ")
-order = []
-
-glob_prereq_types = [ "build" ]
-if re.match("pkg-.*", target) is not None:
-    glob_prereq_types = [ "build", "run", "release", "devel" ]
-    print("using prerequisite types " + ' '.join(glob_prereq_types))
-
-calculate_order(order, modules, glob_prereq_types)
-if args.ignore_deps:
-    order = [m for m in order if m in args.modules]
-order = [m for m in order if m not in exclude]
-if target == 'order':
-    print(' '.join(order))
-    exit(0)
-
-cur_project=0
-print("Building target %s in %d projects:" % (target, len(order)))
-for m in order:
-    cur_project=cur_project+1
-    print("  %3d %s" % (cur_project, m))
-cur_project=0
-
-if args.dry_run:
-    exit(0)
-
-build(modules, order, target)
-
-print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
+cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_'))
+cmd(sys.argv[(len(global_args) + 1)::])