jw-pkg/scripts/build.py

#!/usr/bin/python2 -u
# -*- coding: iso-8859-15 -*-

from __future__ import print_function
import os
import sys
import dircache
import linecache
import fileinput
import glob
import subprocess
from sets import Set
import pwd
import argparse
import time
import datetime
import re

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

def cache_func(func, args):
    global cache
    d = cache
    depth = 0
    keys = [ func.__name__ ] + args
    l = len(keys)
    for k in keys:
        if k is None:
            k = 'None'
        depth += 1
        #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
        if k in d:
            if l == depth:
                return d[k]
            d = d[k]
            continue
        if l == depth:
            r = func(*args)
            d[k] = r
            return r
        d[k] = {}
        d = d[k]
    raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)

def find_proj_path(name):
    name=name.replace("dspider-", "")
    for sub in search_path:
        path=proj_base + "/" + sub + "/" + name
        if os.path.exists(path):
            return os.path.abspath(path)
    raise Exception("module " + name + " not found below " + proj_base)

def find_proj_path_cached(name):
    return cache_func(find_proj_path, [ name ])

def read_deps(cur, prereq_type):
    # dep cache doesn't make a difference at all
    if prereq_type in dep_cache:
        if cur in dep_cache[prereq_type]:
            return dep_cache[prereq_type][cur]
    else:
        dep_cache[prereq_type] = {}
    cmd = projects_py + " prereq " + prereq_type + " " + cur
    debug('running', cmd)
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    p.wait()
    if p.returncode:
        raise Exception("failed to get " + prereq_type + " prerequisites from " + path)
    r = Set()
    pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
    for line in iter(p.stdout.readline,''):
        debug(cmd + ' returned: ', line)
        if not pattern.match(line):
            continue
        for d in line.split():
            r.add(d)
    if cur in r:
        r.remove(cur)
    debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
    dep_cache[prereq_type][cur] = r
    return r

def read_deps_cached(cur, prereq_type):
    return cache_func(read_deps, [ cur, prereq_type ])

def add_tree(cur, prereq_types):
    debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
    if cur in all_deps:
        debug('already handled module ' + cur)
        return 0

    deps = Set()
    all_deps.add(cur)
    for t in prereq_types:
        debug("checking prereqisites of type " + t)
        deps.update(read_deps_cached(cur, t))
    for d in deps:
        add_tree(d, prereq_types)
    dep_tree[cur] = deps
    return len(deps)

def calculate_order(order, modules, prereq_types):
    for m in modules:
        debug("--- adding dependency tree of module " + m)
        add_tree(m, prereq_types)
    while len(all_deps):
        for d in all_deps:
            if not len(dep_tree[d]):
                break
        else:
            print(all_deps)
            raise Exception("fatal: the dependencies between these modules are unresolvable")
        order.append(d)
        all_deps.remove(d)
        for k in dep_tree.keys():
            if d in dep_tree[k]:
                dep_tree[k].remove(d)
    return 1

def run_make(module, target):
    global cur_project
    cur_project=cur_project+1
    #make_cmd = "make " + target + " 2>&1"
    make_cmd = [ "make", target ]
    path = find_proj_path_cached(module)
    delim_len=120
    delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
    delim = delim + '-' * (delim_len - len(delim))

    print(',' + delim + ' >')
    os.chdir(path)
    p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
    for line in iter(p.stdout.readline, ''):
        sys.stdout.write('| ' + line) # avoid extra newlines from print()
        sys.stdout.flush()
    p.wait()
    print('`' + delim + ' <')
    if p.returncode:
        print(make_cmd + ' failed')
        raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base)

def build(modules, order, target):
    if target in ["clean", "distclean"]:
        for m in reversed(order):
            run_make(m, target)
            if m in modules:
                modules.remove(m)
            if not len(modules):
                print("all modules cleaned")
                return
    else:
        for m in order:
            run_make(m, target)

# ------------ here we go

all_deps = Set()
cache = {}
visited = {}
dep_tree = {}
glob_order = []
proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
dep_cache = {}
my_dir=os.path.dirname(os.path.realpath(__file__))

# -- parse command line
parser = argparse.ArgumentParser(description='janware software project build tool')
parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory')
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('--dry-run', '-n', action='store_true',
    default=False, help='Don\'t build anything, just print what would be done.')
parser.add_argument('--ignore-deps', '-I', action='store_true',
    default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
parser.add_argument('target', default='all', help='Build target')
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')

args=parser.parse_args()

debug("----------------------------------------- running ", ' '.join(sys.argv))

modules=args.modules
exclude=args.exclude.split()
proj_base=args.base
target=args.target

projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")

env_exclude=os.getenv('BUILD_EXCLUDE', '')
if len(env_exclude):
    print("exluding modules from environment: " + env_exclude)
    exclude += " " + env_exclude

# -- build
if target != 'order':
    print("calculating order for modules ... ")
order = []

glob_prereq_types = [ "build" ]
if re.match("pkg-.*", target) is not None:
    glob_prereq_types = [ "build", "run", "release", "devel" ]
    print("using prerequisite types " + ' '.join(glob_prereq_types))

calculate_order(order, modules, glob_prereq_types)
if args.ignore_deps:
    order = [m for m in order if m in args.modules]
order = [m for m in order if m not in exclude]
if target == 'order':
    print(' '.join(order))
    exit(0)

cur_project=0
print("Building target %s in %d projects:" % (target, len(order)))
for m in order:
    cur_project=cur_project+1
    print("  %3d %s" % (cur_project, m))
cur_project=0

if args.dry_run:
    exit(0)

build(modules, order, target)

print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))