jw-pkg/scripts/build.py

#!/usr/bin/python -u
# -*- coding: iso-8859-15 -*-

from __future__ import print_function
import os
import sys
import dircache
import linecache
import fileinput
import glob
import subprocess
from sets import Set
import pwd
import argparse
import time
import datetime
import re

all_deps = Set()
dep_tree = {}
glob_order = []
proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
glob_prereq_type="BUILD"
dep_cache = {}

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

def find_proj_path(name):
    name=name.replace("dspider-", "")
    for sub in search_path:
        path=proj_base + "/" + sub + "/" + name
        if os.path.exists(path):
            return path
    raise Exception("module " + name + " not found below " + proj_base)

def read_deps(cur, prereq_type):
    # dep cache doesn't make a difference at all
    if prereq_type in dep_cache:
        if cur in dep_cache[prereq_type]:
            return dep_cache[cur][prereq_type]
    else:
        dep_cache[prereq_type] = {}
    path = find_proj_path(cur)
    os.chdir(path)
    # ignoring prereq_type, as it has never been anything but BUILD, now
    # only looking for PREREQ without anything
    # p = subprocess.Popen("LD_LIBRARY_PATH= make echo-prereq | sed '/PREREQ " + " *=/ !d; s/.*=//'", shell=True, stdout=subprocess.PIPE)
    p = subprocess.Popen("LD_LIBRARY_PATH= make echo-prereq", shell=True, stdout=subprocess.PIPE)
    p.wait()
    if p.returncode:
        raise Exception("failed to get " + prereq_type + " prerequisites from " + path)
    r = Set()
    pattern = re.compile(r'^ *PREREQ.*=')
    for line in iter(p.stdout.readline,''):
        if not pattern.match(line):
            continue
        for d in re.sub(pattern, '', line) .split():
            r.add(d)
#    for d in p.stdout.read().split():
#        r.add(d)
    if cur in r:
        r.remove(cur)
    debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r))
    dep_cache[prereq_type][cur] = r
    return r

def add_tree(cur, prereq_type):
    if cur in all_deps:
        return 0
    all_deps.add(cur)
    deps = read_deps(cur, prereq_type)
    for d in deps:
        add_tree(d, prereq_type)
    dep_tree[cur] = deps
    return len(deps)

def calculate_order(order, modules, prereq_type):
    for m in modules:
        add_tree(m, prereq_type)
    while len(all_deps):
        for d in all_deps:
            if not len(dep_tree[d]):
                break
        else:
            print(all_deps)
            raise Exception("fatal: the dependencies between these modules are unresolvable")
        order.append(d)
        all_deps.remove(d)
        for k in dep_tree.keys():
            if d in dep_tree[k]:
                dep_tree[k].remove(d)
    return 1

def run_make(module, target):
    global cur_project
    cur_project=cur_project+1
    make_cmd = "make " + target + " 2>&1"
    path = find_proj_path(module)
    print(',---------- %d/%d: running %s in %s -------------------------- >' % (cur_project, len(order), make_cmd, path))
    os.chdir(path)
    p = subprocess.Popen(make_cmd, shell=True, stdout=subprocess.PIPE)
    for line in iter(p.stdout.readline, ''):
        sys.stdout.write('| ' + line) # avoid extra newlines from print()
        sys.stdout.flush()
    p.wait()
    print('`---------- %d/%d: running %s in %s -------------------------- <' % (cur_project, len(order), make_cmd, path))
    if p.returncode:
        print(make_cmd + ' failed')
        raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base)

def build(modules, order, target):
    if target in ["clean", "distclean"]:
        for m in reversed(order):
            run_make(m, target)
            if m in modules:
                modules.remove(m)
            if not len(modules):
                print("all modules cleaned")
                return
    else:
        for m in order:
            run_make(m, target)

# -- parse command line
parser = argparse.ArgumentParser(description='jannet software project build tool')
parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory')
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('target', default='all', help='Build target')
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')

args=parser.parse_args()

debug("----------------------------------------- running ", ' '.join(sys.argv))

modules=args.modules
exclude=args.exclude.split()
proj_base=args.base
target=args.target

env_exclude=os.getenv('BUILD_EXCLUDE', '')
if len(env_exclude):
    print("exluding modules from environment: " + env_exclude)
    exclude += " " + env_exclude

# -- build
if target != 'order':
    print("calculating order for modules: " + ' '.join(modules))
order = []
calculate_order(order, modules, glob_prereq_type)
order = [m for m in order if m not in exclude]
if target == 'order':
    print(' '.join(order))
    exit(0)

cur_project=0
print("Building target %s in %d projects:" % (target, len(order)))
for m in order:
    cur_project=cur_project+1
    print("  %3d %s" % (cur_project, m))
cur_project=0

build(modules, order, target)

print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))