jw-pkg/scripts/build.py

#!/usr/bin/python2 -u
# -*- coding: iso-8859-15 -*-

from __future__ import print_function
import os
import sys
import dircache
import linecache
import fileinput
import glob
import subprocess
from sets import Set
import pwd
import argparse
import time
import datetime
import re
from os.path import expanduser

class ResultCache(object):

    def __init__(self):
        self.__cache = {}

    def run(self, func, args):
        d = self.__cache
        depth = 0
        keys = [ func.__name__ ] + args
        l = len(keys)
        for k in keys:
            if k is None:
                k = 'None'
            else:
                k = str(k)
            depth += 1
            #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
            if k in d:
                if l == depth:
                    return d[k]
                d = d[k]
                continue
            if l == depth:
                r = func(*args)
                d[k] = r
                return r
            d = d[k] = {}
            #d = d[k]
        raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

class Build(object):

    def find_proj_path(self, name):
        name=name.replace("dspider-", "")
        search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
        for sub in search_path:
            path=projs_root + "/" + sub + "/" + name
            if os.path.exists(path):
                return os.path.abspath(path)
        raise Exception("module " + name + " not found below " + projs_root)

    def find_proj_path_cached(self, name):
        return res_cache.run(self.find_proj_path, [ name ])

    def read_deps(self, cur, prereq_type):
        # dep cache doesn't make a difference at all
        projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
        if prereq_type in dep_cache:
            if cur in dep_cache[prereq_type]:
                return dep_cache[prereq_type][cur]
        else:
            dep_cache[prereq_type] = {}
        cmd = projects_py + " prereq " + prereq_type + " " + cur
        debug('running', cmd)
        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
        p.wait()
        if p.returncode:
            raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd)
        r = Set()
        pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
        for line in iter(p.stdout.readline,''):
            debug(cmd + ' returned: ', line)
            if not pattern.match(line):
                continue
            for d in line.split():
                r.add(d)
        if cur in r:
            r.remove(cur)
        debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
        dep_cache[prereq_type][cur] = r
        return r

    def read_deps_cached(self, cur, prereq_type):
        return res_cache.run(self.read_deps, [ cur, prereq_type ])

    def add_dep_tree(self, cur, prereq_types, tree, all_deps):
        debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
        if cur in all_deps:
            debug('already handled module ' + cur)
            return 0

        deps = Set()
        all_deps.add(cur)
        for t in prereq_types:
            debug("checking prereqisites of type " + t)
            deps.update(self.read_deps_cached(cur, t))
        for d in deps:
            self.add_dep_tree(d, prereq_types, tree, all_deps)
        tree[cur] = deps
        return len(deps)

    def calculate_order(self, order, modules, prereq_types, all_deps):
        dep_tree = {}
        for m in modules:
            debug("--- adding dependency tree of module " + m)
            self.add_dep_tree(m, prereq_types, dep_tree, all_deps)
        while len(all_deps):
            for d in all_deps:
                if not len(dep_tree[d]):
                    break
            else:
                print(all_deps)
                raise Exception("fatal: the dependencies between these modules are unresolvable")
            order.append(d)
            all_deps.remove(d)
            for k in dep_tree.keys():
                if d in dep_tree[k]:
                    dep_tree[k].remove(d)
        return 1

    def run_make(self, module, target, cur_project, num_projects):
        #make_cmd = "make " + target + " 2>&1"
        make_cmd = [ "make", target ]
        path = self.find_proj_path_cached(module)
        delim_len=120
        delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
        delim = delim + '-' * (delim_len - len(delim))

        print(',' + delim + ' >')
        os.chdir(path)
        p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
        for line in iter(p.stdout.readline, ''):
            sys.stdout.write('| ' + line) # avoid extra newlines from print()
            sys.stdout.flush()
        p.wait()
        print('`' + delim + ' <')
        if p.returncode:
            print(make_cmd + ' failed')
            raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root)

    def run_make_on_modules(self, modules, order, target):
        cur_project = 0
        num_projects = len(order)
        if target in ["clean", "distclean"]:
            for m in reversed(order):
                cur_project += 1
                self.run_make(m, target, cur_project, num_projects)
                if m in modules:
                    modules.remove(m)
                if not len(modules):
                    print("all modules cleaned")
                    return
        else:
            for m in order:
                cur_project += 1
                self.run_make(m, target, cur_project, num_projects)

    def run(self, args_):
        all_deps = Set()
        visited = {}
        glob_order = []
        projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"

        # -- parse command line
        parser = argparse.ArgumentParser(description='janware software project build tool')
        parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory')
        parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
        parser.add_argument('--debug', '-d', action='store_true',
            default=False, help='Output debug information to stderr')
        parser.add_argument('--dry-run', '-n', action='store_true',
            default=False, help='Don\'t build anything, just print what would be done.')
        parser.add_argument('--ignore-deps', '-I', action='store_true',
            default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
        parser.add_argument('target', default='all', help='Build target')
        parser.add_argument('modules', nargs='+', default='', help='Modules to be built')

        args=parser.parse_args(args_)

        debug("----------------------------------------- running ", ' '.join(args_))

        modules=args.modules
        exclude=args.exclude.split()
        target=args.target

        env_exclude=os.getenv('BUILD_EXCLUDE', '')
        if len(env_exclude):
            print("exluding modules from environment: " + env_exclude)
            exclude += " " + env_exclude

        # -- build
        if target != 'order':
            print("calculating order for modules ... ")
        order = []

        glob_prereq_types = [ "build" ]
        if re.match("pkg-.*", target) is not None:
            glob_prereq_types = [ "build", "run", "release", "devel" ]
            print("using prerequisite types " + ' '.join(glob_prereq_types))

        self.calculate_order(order, modules, glob_prereq_types, all_deps)
        if args.ignore_deps:
            order = [m for m in order if m in args.modules]
        order = [m for m in order if m not in exclude]
        if target == 'order':
            print(' '.join(order))
            exit(0)

        cur_project = 0
        print("Building target %s in %d projects:" % (target, len(order)))
        for m in order:
            cur_project += 1
            print("  %3d %s" % (cur_project, m))

        if args.dry_run:
            exit(0)

        self.run_make_on_modules(modules, order, target)

        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

def cmd_build(args_):
    build = Build()
    build.run(args_)

# -------------------------------------------------------------------- here we go

global_args = []
#res_cache = ResultCache()

skip = 0
for a in sys.argv[1::]:
    global_args.append(a)
    if a in [ '--prefix', '-p', '--topdir', '-t' ]:
        skip = 1
        continue
    if skip > 0:
        skip = skip -1
        continue
    if a[0] != '-':
        break

topdir = None
top_name = None

parser = argparse.ArgumentParser(description='Project metadata evaluation')
parser.add_argument('cmd', default='', help='Command')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('--topdir', '-t', nargs='?', help='Project Path')
parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") +
    '/local/src/jw.dev/proj', help='Projects Path Prefix')
parser.add_argument('arg', nargs='*', help='Command arguments')
args=parser.parse_args(global_args)

debug("----------------------------------------- running ", ' '.join(sys.argv))

projs_root = args.prefix
res_cache = ResultCache()
dep_cache = {}
my_dir=os.path.dirname(os.path.realpath(__file__))

if args.topdir:
    topdir = args.topdir
    top_name = res_cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
    if not top_name:
        top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))

cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_'))
cmd(sys.argv[(len(global_args) + 1)::])