jw-pkg/scripts/build.py

#!/usr/bin/python -u
# -*- coding: iso-8859-15 -*-

from __future__ import print_function
import os
import sys
import dircache
import linecache
import fileinput
import glob
import subprocess
from sets import Set
import pwd
import argparse
import time
import datetime
import re

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

def find_proj_path(name):
    name=name.replace("dspider-", "")
    for sub in search_path:
        path=proj_base + "/" + sub + "/" + name
        if os.path.exists(path):
            return os.path.abspath(path)
    raise Exception("module " + name + " not found below " + proj_base)

def read_deps(cur, prereq_type):
    # dep cache doesn't make a difference at all
    if prereq_type in dep_cache:
        if cur in dep_cache[prereq_type]:
            return dep_cache[prereq_type][cur]
    else:
        dep_cache[prereq_type] = {}
    cmd = projects_py + " prereq " + prereq_type + " " + cur
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    p.wait()
    if p.returncode:
        raise Exception("failed to get " + prereq_type + " prerequisites from " + path)
    r = Set()
    pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
    for line in iter(p.stdout.readline,''):
        debug(cmd + ' returned: ', line)
        if not pattern.match(line):
            continue
        for d in line.split():
            r.add(d)
    if cur in r:
        r.remove(cur)
    debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
    dep_cache[prereq_type][cur] = r
    return r

def add_tree(cur, prereq_types):
    debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
    if cur in all_deps:
        debug('already handled module ' + cur)
        return 0

    deps = Set()
    all_deps.add(cur)
    for t in prereq_types:
        debug("checking prereqisites of type " + t)
        deps.update(read_deps(cur, t))
    for d in deps:
        add_tree(d, prereq_types)
    dep_tree[cur] = deps
    return len(deps)

def calculate_order(order, modules, prereq_types):
    for m in modules:
        debug("--- adding dependency tree of module " + m)
        add_tree(m, prereq_types)
    while len(all_deps):
        for d in all_deps:
            if not len(dep_tree[d]):
                break
        else:
            print(all_deps)
            raise Exception("fatal: the dependencies between these modules are unresolvable")
        order.append(d)
        all_deps.remove(d)
        for k in dep_tree.keys():
            if d in dep_tree[k]:
                dep_tree[k].remove(d)
    return 1

def run_make(module, target):
    global cur_project
    cur_project=cur_project+1
    make_cmd = "make " + target + " 2>&1"
    path = find_proj_path(module)
    delim_len=120
    delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
    delim = delim + '-' * (delim_len - len(delim))

    print(',' + delim + ' >')
    os.chdir(path)
    p = subprocess.Popen(make_cmd, shell=True, stdout=subprocess.PIPE)
    for line in iter(p.stdout.readline, ''):
        sys.stdout.write('| ' + line) # avoid extra newlines from print()
        sys.stdout.flush()
    p.wait()
    print('`' + delim + ' <')
    if p.returncode:
        print(make_cmd + ' failed')
        raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base)

def build(modules, order, target):
    if target in ["clean", "distclean"]:
        for m in reversed(order):
            run_make(m, target)
            if m in modules:
                modules.remove(m)
            if not len(modules):
                print("all modules cleaned")
                return
    else:
        for m in order:
            run_make(m, target)

# ------------ here we go

all_deps = Set()
visited = {}
dep_tree = {}
glob_order = []
proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
dep_cache = {}
my_dir=os.path.dirname(os.path.realpath(__file__))

# -- parse command line
parser = argparse.ArgumentParser(description='janware software project build tool')
parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory')
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('--dry-run', '-n', action='store_true',
    default=False, help='Don\'t build anything, just print what would be done.')
parser.add_argument('--ignore-deps', '-I', action='store_true',
    default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
parser.add_argument('target', default='all', help='Build target')
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')

args=parser.parse_args()

debug("----------------------------------------- running ", ' '.join(sys.argv))

modules=args.modules
exclude=args.exclude.split()
proj_base=args.base
target=args.target
projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base

env_exclude=os.getenv('BUILD_EXCLUDE', '')
if len(env_exclude):
    print("exluding modules from environment: " + env_exclude)
    exclude += " " + env_exclude

# -- build
if target != 'order':
    print("calculating order for modules ... ")
order = []

glob_prereq_types = [ "build" ]
if re.match("pkg-.*", target) is not None:
    glob_prereq_types = [ "build", "run", "release", "devel" ]
    print("using prerequisite types " + ' '.join(glob_prereq_types))

calculate_order(order, modules, glob_prereq_types)
if args.ignore_deps:
    order = [m for m in order if m in args.modules]
order = [m for m in order if m not in exclude]
if target == 'order':
    print(' '.join(order))
    exit(0)

cur_project=0
print("Building target %s in %d projects:" % (target, len(order)))
for m in order:
    cur_project=cur_project+1
    print("  %3d %s" % (cur_project, m))
cur_project=0

if args.dry_run:
    exit(0)

build(modules, order, target)

print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))