projects.py: Merge build.py

Signed-off-by: Jan Lindemann <jan@janware.com>

scripts/projects.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 import os
 import sys
 import argparse
+import pwd
 from sets import Set
 from os.path import isfile
 from os.path import isdir
@@ -51,6 +52,186 @@ class ResultCache(object):
             #d = d[k]
         raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
 
+class Build(object):
+
+    def find_proj_path(self, name):
+        name=name.replace("dspider-", "")
+        search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
+        for sub in search_path:
+            path=projs_root + "/" + sub + "/" + name
+            if os.path.exists(path):
+                return os.path.abspath(path)
+        raise Exception("module " + name + " not found below " + projs_root)
+
+    def find_proj_path_cached(self, name):
+        return res_cache.run(self.find_proj_path, [ name ])
+
+    def read_deps(self, cur, prereq_type):
+        # dep cache doesn't make a difference at all
+        projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
+        if prereq_type in dep_cache:
+            if cur in dep_cache[prereq_type]:
+                return dep_cache[prereq_type][cur]
+        else:
+            dep_cache[prereq_type] = {}
+        cmd = projects_py + " prereq " + prereq_type + " " + cur
+        debug('running', cmd)
+        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+        p.wait()
+        if p.returncode:
+            raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd)
+        r = Set()
+        pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
+        for line in iter(p.stdout.readline,''):
+            debug(cmd + ' returned: ', line)
+            if not pattern.match(line):
+                continue
+            for d in line.split():
+                r.add(d)
+        if cur in r:
+            r.remove(cur)
+        debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
+        dep_cache[prereq_type][cur] = r
+        return r
+
+    def read_deps_cached(self, cur, prereq_type):
+        return res_cache.run(self.read_deps, [ cur, prereq_type ])
+
+    def add_dep_tree(self, cur, prereq_types, tree, all_deps):
+        debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
+        if cur in all_deps:
+            debug('already handled module ' + cur)
+            return 0
+
+        deps = Set()
+        all_deps.add(cur)
+        for t in prereq_types:
+            debug("checking prereqisites of type " + t)
+            deps.update(self.read_deps_cached(cur, t))
+        for d in deps:
+            self.add_dep_tree(d, prereq_types, tree, all_deps)
+        tree[cur] = deps
+        return len(deps)
+
+    def calculate_order(self, order, modules, prereq_types, all_deps):
+        dep_tree = {}
+        for m in modules:
+            debug("--- adding dependency tree of module " + m)
+            self.add_dep_tree(m, prereq_types, dep_tree, all_deps)
+        while len(all_deps):
+            for d in all_deps:
+                if not len(dep_tree[d]):
+                    break
+            else:
+                print(all_deps)
+                raise Exception("fatal: the dependencies between these modules are unresolvable")
+            order.append(d)
+            all_deps.remove(d)
+            for k in dep_tree.keys():
+                if d in dep_tree[k]:
+                    dep_tree[k].remove(d)
+        return 1
+
+    def run_make(self, module, target, cur_project, num_projects):
+        #make_cmd = "make " + target + " 2>&1"
+        make_cmd = [ "make", target ]
+        path = self.find_proj_path_cached(module)
+        delim_len=120
+        delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
+        delim = delim + '-' * (delim_len - len(delim))
+
+        print(',' + delim + ' >')
+        os.chdir(path)
+        p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
+        for line in iter(p.stdout.readline, ''):
+            sys.stdout.write('| ' + line) # avoid extra newlines from print()
+            sys.stdout.flush()
+        p.wait()
+        print('`' + delim + ' <')
+        if p.returncode:
+            print(make_cmd + ' failed')
+            raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root)
+
+    def run_make_on_modules(self, modules, order, target):
+        cur_project = 0
+        num_projects = len(order)
+        if target in ["clean", "distclean"]:
+            for m in reversed(order):
+                cur_project += 1
+                self.run_make(m, target, cur_project, num_projects)
+                if m in modules:
+                    modules.remove(m)
+                if not len(modules):
+                    print("all modules cleaned")
+                    return
+        else:
+            for m in order:
+                cur_project += 1
+                self.run_make(m, target, cur_project, num_projects)
+
+    def run(self, args_):
+        all_deps = Set()
+        visited = {}
+        glob_order = []
+        projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
+
+        # -- parse command line
+        parser = argparse.ArgumentParser(description='janware software project build tool')
+        parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory')
+        parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
+        parser.add_argument('--debug', '-d', action='store_true',
+            default=False, help='Output debug information to stderr')
+        parser.add_argument('--dry-run', '-n', action='store_true',
+            default=False, help='Don\'t build anything, just print what would be done.')
+        parser.add_argument('--ignore-deps', '-I', action='store_true',
+            default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
+        parser.add_argument('target', default='all', help='Build target')
+        parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
+
+        args=parser.parse_args(args_)
+
+        debug("----------------------------------------- running ", ' '.join(args_))
+
+        modules=args.modules
+        exclude=args.exclude.split()
+        target=args.target
+
+        env_exclude=os.getenv('BUILD_EXCLUDE', '')
+        if len(env_exclude):
+            print("exluding modules from environment: " + env_exclude)
+            exclude += " " + env_exclude
+
+        # -- build
+        if target != 'order':
+            print("calculating order for modules ... ")
+        order = []
+
+        glob_prereq_types = [ "build" ]
+        if re.match("pkg-.*", target) is not None:
+            glob_prereq_types = [ "build", "run", "release", "devel" ]
+            print("using prerequisite types " + ' '.join(glob_prereq_types))
+
+        self.calculate_order(order, modules, glob_prereq_types, all_deps)
+        if args.ignore_deps:
+            order = [m for m in order if m in args.modules]
+        order = [m for m in order if m not in exclude]
+        if target == 'order':
+            print(' '.join(order))
+            exit(0)
+
+        cur_project = 0
+        print("Building target %s in %d projects:" % (target, len(order)))
+        for m in order:
+            cur_project += 1
+            print("  %3d %s" % (cur_project, m))
+
+        if args.dry_run:
+            exit(0)
+
+        self.run_make_on_modules(modules, order, target)
+
+        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
 def debug(*objs):
     if args.debug:
         print("DEBUG: ", *objs, file=sys.stderr)
@@ -374,6 +555,10 @@ def commands():
 def cmd_commands(args_):
     print(commands())
 
+def cmd_build(args_):
+    build = Build()
+    build.run(args_)
+
 def cmd_test(args_):
     parser = argparse.ArgumentParser(description='Test')
     parser.add_argument('blah', default='', help='The blah argument')