jw-pkg/scripts/projects.py

#!/usr/bin/python -u

from __future__ import print_function
import sys
import argparse
from sets import Set
from os.path import isdir
from os.path import expanduser
from os.path import basename
from os.path import realpath
import re

# meaning of pkg.required variables
# build: needs to be built and installed before this can be built
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
# run:   needs to be installed before this-run can be installed, i.e. before this and other packages can run with this

def re_section(name):
    return re.compile('[' + name + ']'
                 '.*?'
                 '(?=[)',
                 re.DOTALL)
# --------------------------------------------------------------------- helpers

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

def proj_dir(name):
    return projs_root + '/' + name

# TODO: add support for customizing this in project.conf
def htdocs_dir(name):
    pd = proj_dir(name)
    for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name
            ]:
        if isdir(r):
            return r
    return None

def strip_module_from_spec(mod):
    return re.sub(r'-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())

def get_section(path, section):
    r = ''
    file = open(path)
    pat = '[' + section + ']'
    in_section = False
    for line in file:
        if (line.rstrip() == pat):
            in_section = True
            continue
        if in_section:
            if len(line) and line[0] == '[':
                break
            r = r + line
    file.close()
    return r.rstrip()
 
def read_value(path, section, key):
    debug("opening ", path)
    try:
        file = open(path)
    except:
        return None
    r = []
    if not len(section):
        for line in file:
            r = re.findall('^ *' + key + ' *= *(.*)', line)
            if (len(r) > 0):
                break
    else:
        in_section = False
        pat = '[' + section + ']'
        for line in file:
            if (line.rstrip() == pat):
                in_section = True
                continue
            if in_section:
                if len(line) and line[0] == '[':
                    break
                if key is None:
                    r.append(line)
                else:
                    r = re.findall('^ *' + key + ' *= *(.*)', line)
                    if (len(r) > 0):
                        break
    file.close()

    if len(r):
        return r[0]
    return None

def get_value(name, section, key):
    #print("top_name = ", top_name)
    if top_name and name == top_name:
        proj_root = topdir
    else:
        proj_root = projs_root + '/' + name

    if section == 'version':
        file = open(proj_root + '/VERSION', 'r')
        r=file.read().replace('\n', '').replace('-dev', '')
        file.close()
        return r

    path = proj_root + '/make/project.conf'
    #print('path = ', path, 'top_name = ', top_name, 'name = ', name)
    return read_value(path, section, key)

# scope 0: no children
# scope 1: children
# scope 2: recursive

def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope,
        names_only):
    name = strip_module_from_spec(spec)
    if names_only:
        spec = name
    if spec in buf:
        return
    if spec in visited:
        if add_self:
            buf.append(spec)
        return
    visited.add(spec)
    deps = get_value(name, section, key)
    debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited)
    if deps and scope > 0:
        if scope == 1:
            subscope = 0
        else:
            subscope = 2
        deps = deps.split(',')
        for dep in deps:
            add_modules_from_project_txt(buf, visited, dep,
                    section, key, add_self=True, scope=subscope,
                    names_only=names_only)
    if add_self:
        buf.append(spec)

def get_modules_from_project_txt(names, section, key, add_self, scope,
        names_only = True):
    #r = Set()
    r = []
    visited = Set()
    for name in names:
        add_modules_from_project_txt(r, visited, name, section, key, add_self, scope,
                names_only)
    return r

def get_libname(names):
    vals = get_modules_from_project_txt(names, 'build', 'libname',
            scope = 1, add_self=False, names_only=True)
    if not vals:
        return ' '.join(names)
    if 'none' in vals:
        vals.remove('none')
    return ' '.join(reversed(vals))

# -L needs to contain more paths than libs linked with -l would require
def get_ldpathflags(names):
    deps = get_modules_from_project_txt(names, 'pkg.required', 'build',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        libname = get_libname([m])
        if len(libname):
        	r = r + ' -L' + proj_dir(m) + '/lib'
    print(r[1:])

def get_ldflags(names):
    #print(names)
    deps = get_modules_from_project_txt(names, 'pkg.required', 'build',
            scope = 1, add_self=False, names_only=True)
    #print(deps)
    r = ''
    for m in reversed(deps):
        libname = get_libname([m])
        if len(libname):
            #r = r + ' -L' + proj_dir(m) + '/lib -l' + libname
            r = r + ' -l' + libname
    if len(r):
	ldpathflags = get_ldpathflags(names)
	if ldpathflags:
		r = ldpathflags + ' ' + r
        return r[1::]
    return ''

def commands():
    f = open(sys.argv[0])
    cmds = []
    for line in f:
        debug("checking line ", line)
        rr = re.findall('^def  *cmd_([a-z0-9_]+).*', line)
        if len(rr):
            cmds.append(rr[0].replace('_', '-'))
    f.close()
    return ' '.join(cmds)

# --------------------------------------------------------------------- commands

def cmd_commands(args_):
    print(commands())

def cmd_test(args_):
    parser = argparse.ArgumentParser(description='Test')
    parser.add_argument('blah', default='', help='The blah argument')
    args=parser.parse_args(args_)
    print("blah = " + args.blah)

def cmd_ldlibpath(args_):
    parser = argparse.ArgumentParser(description='ldlibpath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.required', 'run',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/lib'
    print(r[1:])

def cmd_exepath(args_):
    parser = argparse.ArgumentParser(description='exepath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.required', 'run',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/bin'
    print(r[1:])

def cmd_libname(args_):
    parser = argparse.ArgumentParser(description='libname')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    print(get_libname(args.module))

def cmd_ldflags(args_):
    parser = argparse.ArgumentParser(description='ldlibpath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    print(get_ldflags(args.module))

def cmd_cflags(args_):
    parser = argparse.ArgumentParser(description='ldlibpath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.required', 'build',
            scope = 1, add_self=True, names_only=True)
    r = ''
    for m in reversed(deps):
        r = r + ' -I' + proj_dir(m) + '/include'
    print(r[1:])

def cmd_path(args_):
    parser = argparse.ArgumentParser(description='path')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.required', 'run',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/bin'
    print(r[1:])

def cmd_prereq(args_):
    parser = argparse.ArgumentParser(description='path')
    parser.add_argument('flavour', help='Flavour')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.required',
        args.flavour, scope = 2, add_self=False, names_only=True)
    print(' '.join(deps))

def cmd_pkg_requires(args_):
    parser = argparse.ArgumentParser(description='pkg-requires')
    parser.add_argument('flavour', help='Flavour')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        value = get_value(m, 'pkg.required', args.flavour)
        if not value:
            continue
        deps = value.split(',')
        for spec in deps:
            dep = re.split('([=><]+)', spec)
            for i, item in enumerate(dep):
                dep[i] = item.strip()
            if len(dep) == 3:
                dep_project = re.sub(r'-devel$|-run$', '', dep[0])
                version = get_value(dep_project, 'version', '')
                if dep[2] == 'VERSION':
                    dep[2] = version.split('-')[0]
                elif dep[2] == 'VERSION-REVISION':
                    dep[2] = version
                else:
                    raise Exception("Unknown version specifier in " + spec)
            r.append(' '.join(dep))
    print(', '.join(r))

def cmd_proj_dir(args_):
    parser = argparse.ArgumentParser(description='proj-dir')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        r.append(proj_dir(m))
    print(' '.join(r))

def cmd_htdocs_dir(args_):
    parser = argparse.ArgumentParser(description='htdocs-dir')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        r.append(htdocs_dir(m))
    print(' '.join(r))

def cmd_summary(args_):
    parser = argparse.ArgumentParser(description='summary')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        summary = get_value(m, "summary", None)
        if summary is not None:
            r.append(summary)
    print(' '.join(r))

def contains(small, big):
    for i in xrange(len(big)-len(small)+1):
        for j in xrange(len(small)):
            if big[i+j] != small[j]:
                break
        else:
            return i, i+len(small)
    return False

def read_dep_graph(modules, section, graph):
    for m in modules:
        if m in graph:
            continue
        deps = get_modules_from_project_txt([ m ], 'pkg.required', section,
                scope = 1, add_self=False, names_only=True)
        if not deps is None:
            graph[m] = deps
            for d in deps:
                read_dep_graph([ d ], section, graph)

def flip_graph(graph):
    r = {}
    for m, deps in graph.iteritems():
        for d in deps:
            if not d in r:
                r[d] = Set()
            r[d].add(m)
    return r

def check_circular_deps(module, section, graph, unvisited, temp, path):
    if module in temp:
        debug('found circular dependency at module', module)
        return module
    if not module in unvisited:
        return None
    temp.add(module)
    if module in graph:
        for m in graph[module]:
            last = check_circular_deps(m, section, graph, unvisited, temp, path)
            if last is not None:
                path.insert(0, m)
                return last
    unvisited.remove(module)
    temp.remove(module)

def cmd_check(args_):
    parser = argparse.ArgumentParser(description='check')
    parser.add_argument('module', nargs='*', help='Modules')
    parser.add_argument('--flavour', '-f', nargs='?', default = 'build')
    args=parser.parse_args(args_)

    graph = {}
    path = []
    read_dep_graph(args.module, args.flavour, graph)
    unvisited = graph.keys()
    temp = Set()
    while len(unvisited) is not 0:
        m = unvisited[0]
        debug('checking circular dependency of', m)
        last = check_circular_deps(m, args.flavour, flip_graph(graph), unvisited, temp, path)
        if last is not None:
            debug('found circular dependency below', m, ', last is', last)
            print('found circular dependency in flavour', args.flavour, ':', ' -> '.join(path))
            exit(1)

    print('no circular dependency found for flavour', args.flavour, ' in modules:',
            ' '.join(args.module))
    exit(0)

# -------------------------------------------------------------------- here we go

global_args = []
skip = 0
for a in sys.argv[1::]:
    global_args.append(a)
    if a in [ '--prefix', '-p', '--topdir', '-t' ]:
        skip = 1
        continue
    if skip > 0:
        skip = skip -1
        continue
    if a[0] != '-':
        break

topdir = None
top_name = None

parser = argparse.ArgumentParser(description='Project metadata evaluation')
parser.add_argument('cmd', default='', help='Command')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('--topdir', '-t', nargs='?', help='Project Path')
parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") +
    '/local/src/cvs.stable/proj', help='Projects Path Prefix')
parser.add_argument('arg', nargs='*', help='Command arguments')
args=parser.parse_args(global_args)

debug("----------------------------------------- running ", ' '.join(sys.argv))

projs_root = args.prefix
if args.topdir:
    topdir = args.topdir
    top_name = read_value(topdir + '/make/project.conf', 'build', 'name')
    if not top_name:
        top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))

cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_'))
cmd(sys.argv[(len(global_args) + 1)::])