jw-pkg/scripts/projects.py

#!/usr/bin/python -u

from __future__ import print_function
import sys
import argparse
from sets import Set
from os.path import isfile
from os.path import isdir
from os.path import expanduser
from os.path import basename
from os.path import realpath
import subprocess
import re
import platform

# meaning of pkg.requires.xxx variables
# build: needs to be built and installed before this can be built
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
# run:   needs to be installed before this-run can be installed, i.e. before this and other packages can run with this

# --------------------------------------------------------------------- helpers

def debug(*objs):
    if args.debug:
        print("DEBUG: ", *objs, file=sys.stderr)

def err(*objs):
        print("ERR: ", *objs, file=sys.stderr)

def cache_func(func, args):
    d = cache
    depth = 0
    keys = [ func.__name__ ] + args
    l = len(keys)
    for k in keys:
        if k is None:
            k = 'None'
        depth += 1
        #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
        if k in d:
            if l == depth:
                return d[k]
            d = d[k]
            continue
        if l == depth:
            r = func(*args)
            d[k] = r
            return r
        d[k] = {}
        d = d[k]
    raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)

def proj_dir(name):
    if name == top_name:
        return topdir
    return projs_root + '/' + name

def re_section(name):
    return re.compile('[' + name + ']'
                 '.*?'
                 '(?=[)',
                 re.DOTALL)

def remove_duplicates(seq):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def get_os(args = ""):
    for d in [ projs_root + '/jw-build/scripts', '/opt/jw-build/bin' ]:
        script = d + '/get-os.sh'
        if isfile(script):
            cmd = '/bin/bash ' + script
            if args:
                cmd = cmd + ' ' + args
            p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
            (out, rr) = p.communicate()
            if rr:
                err("failed to run ", cmd)
                continue
            out = re.sub('\n', '', out)
            return out
    return "linux"

def get_os_cached(args = ""):
    if 'os' in cache:
        return cache['os']
    return get_os(args)

# TODO: add support for customizing this in project.conf
def htdocs_dir(name):
    pd = proj_dir(name)
    for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name
            ]:
        if isdir(r):
            return r
    return None

def os_cascade():
    os = get_os_cached()
    name = re.sub('-.*', '', os)
    # e.g. os, linux, suse, suse-tumbleweed
    return [ 'os', platform.system().lower(), name, os ]

def strip_module_from_spec(mod):
    return re.sub(r'-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())

def get_section(path, section):
    r = ''
    file = open(path)
    pat = '[' + section + ']'
    in_section = False
    for line in file:
        if (line.rstrip() == pat):
            in_section = True
            continue
        if in_section:
            if len(line) and line[0] == '[':
                break
            r = r + line
    file.close()
    return r.rstrip()

def get_section_cached(path, section):
    return cache_func(get_section, [ path, section ])

def read_value(path, section, key):
    debug("opening ", path)
    try:
        file = open(path)
    except:
        debug(path, "not found")
        # TODO: handle this special case cleaner somewhere up the stack
        if section == 'build' and key == 'libname':
            return 'none'
        return None
    r = []
    if not len(section):
        for line in file:
            r = re.findall('^ *' + key + ' *= *(.*)', line)
            if (len(r) > 0):
                break
    else:
        in_section = False
        pat = '[' + section + ']'
        for line in file:
            if (line.rstrip() == pat):
                in_section = True
                continue
            if in_section:
                if len(line) and line[0] == '[':
                    break
                if key is None:
                    r.append(line)
                else:
                    r = re.findall('^ *' + key + ' *= *(.*)', line)
                    if (len(r) > 0):
                        break
    file.close()

    if len(r):
        return r[0]
    return None

def read_value_cached(path, section, key):
    return cache_func(read_value, [path, section, key])

def get_value(name, section, key):
    debug("getting value [%s].%s for project %s (%s)" %(section, key, name, top_name))
    if top_name and name == top_name:
        proj_root = topdir
    else:
        proj_root = projs_root + '/' + name

    debug("proj_root = " + proj_root)
    if section == 'version':
        file = open(proj_root + '/VERSION', 'r')
        r=file.read().replace('\n', '').replace('-dev', '')
        file.close()
        return r

    path = proj_root + '/make/project.conf'
    #print('path = ', path, 'top_name = ', top_name, 'name = ', name)
    return read_value_cached(path, section, key)

def collect_values(names, section, key):
    r = ""
    for n in names:
        val = get_value(n, section, key)
        if val:
            r = r + " " + val
    return remove_duplicates([x.strip() for x in r.split(",")])

# scope 0: no children
# scope 1: children
# scope 2: recursive

def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope,
        names_only):
    name = strip_module_from_spec(spec)
    if names_only:
        spec = name
    if spec in buf:
        return
    if spec in visited:
        if add_self:
            buf.append(spec)
        return
    visited.add(spec)
    deps = get_value(name, section, key)
    debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited)
    if deps and scope > 0:
        if scope == 1:
            subscope = 0
        else:
            subscope = 2
        deps = deps.split(',')
        for dep in deps:
            add_modules_from_project_txt(buf, visited, dep,
                    section, key, add_self=True, scope=subscope,
                    names_only=names_only)
    if add_self:
        buf.append(spec)

def get_modules_from_project_txt(names, section, keys, add_self, scope,
        names_only = True):
    if isinstance(keys, basestring):
        keys = [ keys ]
    #r = Set()
    r = []
    for key in keys:
        visited = Set()
        for name in names:
            rr = []
            add_modules_from_project_txt(rr, visited, name, section, key, add_self, scope,
                    names_only)
            # TODO: this looks like a performance hogger
            for m in rr:
                if not m in r:
                   r.append(m)
    return r

def pkg_relations(rel_type, args_):
    parser = argparse.ArgumentParser(description='pkg-' + rel_type)
    # TODO: implement Vendor evaluation

    parser.add_argument('--subsections', '-S', nargs='?', default=None, help='Subsections to consider, comma-separated')
    parser.add_argument('--delimiter', '-d', nargs='?', default=', ', help='Output words delimiter')
    parser.add_argument('flavour', help='Flavour')
    parser.add_argument('module', nargs='*', help='Modules')
    parser.add_argument('--no-subpackages', '-p', action='store_true',
        default=False, help='Cut -run and -devel from package names')
    parser.add_argument('--no-version', action='store_true',
        default=False, help='Don\'t report version information')
    parser.add_argument('--dont-strip-revision', action='store_true',
        default=False, help='Always treat VERSION macro as VERSION-REVISION')
    parser.add_argument('--dont-expand-version-macros', action='store_true',
        default=False, help='Don\'t expand VERSION and REVISION macros')
    args=parser.parse_args(args_)
    version_pattern=re.compile("[0-9-.]*")
    if args.subsections is None:
        subsecs = os_cascade()
        subsecs.append('jw')
    else:
        subsecs = args.subsections.split(',')
    debug('flavour = ', args.flavour, ', subsecs = ', ' '.join(subsecs))
    r = []
    for s in subsecs:
        for m in args.module:
            value = get_value(m, 'pkg.' + rel_type + '.' + s, args.flavour)
            if not value:
                continue
            deps = value.split(',')
            for spec in deps:
                dep = re.split('([=><]+)', spec)
                if args.no_version:
                    dep = dep[:1]
                if args.no_subpackages:
                    dep[0] = re.sub('-devel$|-run$', '', dep[0])
                for i, item in enumerate(dep):
                    dep[i] = item.strip()
                if len(dep) == 3:
                    dep_project = re.sub(r'-devel$|-run$', '', dep[0])
                    if args.dont_expand_version_macros and dep_project in args.module:
                        version = dep[2]
                    else:
                        version = get_value(dep_project, 'version', '')
                    if dep[2] == 'VERSION':
                        if args.dont_strip_revision:
                            dep[2] = version
                        else:
                            dep[2] = version.split('-')[0]
                    elif dep[2] == 'VERSION-REVISION':
                        dep[2] = version
                    elif version_pattern.match(dep[2]):
                        # dep[2] = dep[2]
                        pass
                    else:
                        raise Exception("Unknown version specifier in " + spec)
                r.append(' '.join(dep))
    print(args.delimiter.join(r))

def get_libname(names):
    vals = get_modules_from_project_txt(names, 'build', 'libname',
            scope = 1, add_self=False, names_only=True)
    if not vals:
        return ' '.join(names)
    if 'none' in vals:
        vals.remove('none')
    return ' '.join(reversed(vals))

# -L needs to contain more paths than libs linked with -l would require
def get_ldpathflags(names, exclude = []):
    deps = get_modules_from_project_txt(names, 'pkg.requires.jw', 'build',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        if m in exclude:
            continue
        libname = get_libname([m])
        if len(libname):
            r = r + ' -L' + proj_dir(m) + '/lib'
    print(r[1:])

def get_ldflags(names, exclude = [], add_self_ = False):
    #print(names)
    deps = get_modules_from_project_txt(names, 'pkg.requires.jw', 'build',
            scope = 1, add_self=add_self_, names_only=True)
    debug("deps = " + ' '.join(deps))
    #print(deps)
    r = ''
    for m in reversed(deps):
        if m in exclude:
            continue
        libname = get_libname([m])
        if len(libname):
            #r = r + ' -L' + proj_dir(m) + '/lib -l' + libname
            r = r + ' -l' + libname
    if len(r):
        ldpathflags = get_ldpathflags(names, exclude)
        if ldpathflags:
            r = ldpathflags + ' ' + r
        return r[1::]
    return ''

def commands():
    f = open(sys.argv[0])
    cmds = []
    for line in f:
        debug("checking line ", line)
        rr = re.findall('^def  *cmd_([a-z0-9_]+).*', line)
        if len(rr):
            cmds.append(rr[0].replace('_', '-'))
    f.close()
    return ' '.join(cmds)

# --------------------------------------------------------------------- commands

def cmd_commands(args_):
    print(commands())

def cmd_test(args_):
    parser = argparse.ArgumentParser(description='Test')
    parser.add_argument('blah', default='', help='The blah argument')
    args=parser.parse_args(args_)
    print("blah = " + args.blah)

# TODO: seems at least partly redundant to cmd_pkg_requires / pkg_relations
def cmd_requires_pkg(args_):
    parser = argparse.ArgumentParser(description='requires-pkg')
    parser.add_argument('module', nargs='*', help='Modules')
    parser.add_argument('--flavours', help='Dependency flavours', default='build')
    args=parser.parse_args(args_)
    flavours = args.flavours.split()
    debug("flavours = " + args.flavours)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', flavours,
            scope = 2, add_self=True, names_only=True)
    subsecs = os_cascade()
    debug("subsecs = ", subsecs)
    requires = []
    for s in subsecs:
        for f in flavours:
            vals = collect_values(deps, 'pkg.requires.' + s, f)
            if vals:
                requires = requires + vals
    # TODO: add all not in build tree as -devel
    r = ''
    for m in requires:
        r = r + ' ' + m
    print(r[1:])

def cmd_os_cascade(args_):
    print(' '.join(os_cascade()))

def cmd_ldlibpath(args_):
    parser = argparse.ArgumentParser(description='ldlibpath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ],
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/lib'
    print(r[1:])

def cmd_pythonpath(args_):
    parser = argparse.ArgumentParser(description='pythonpath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build' ],
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        pdir = proj_dir(m)
        for subdir in [ 'src/python', 'tools/python' ]:
            cand = pdir + "/" + subdir
            if isdir(cand):
                r = r + ':' + cand
    print(r[1:])

def cmd_exepath(args_):
    parser = argparse.ArgumentParser(description='exepath')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ],
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/bin'
    print(r[1:])

def cmd_libname(args_):
    parser = argparse.ArgumentParser(description='libname')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    print(get_libname(args.module))

def cmd_ldflags(args_):
    parser = argparse.ArgumentParser(description='ldflags')
    parser.add_argument('module', nargs='*', help='Modules')
    parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[])
    parser.add_argument('--add-self', '-s', action='store_true',
        default=False, help='Include libflags of specified modules, too, not only their dependencies')
    args=parser.parse_args(args_)
    print(get_ldflags(args.module, args.exclude, args.add_self))

def cmd_cflags(args_):
    parser = argparse.ArgumentParser(description='cflags')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', 'build',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in reversed(deps):
        r = r + ' -I' + proj_dir(m) + '/include'
    print(r[1:])

def cmd_path(args_):
    parser = argparse.ArgumentParser(description='path')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', 'run',
            scope = 2, add_self=True, names_only=True)
    r = ''
    for m in deps:
        r = r + ':' + proj_dir(m) + '/bin'
    print(r[1:])

# TODO: seems at least partly redundant to cmd_pkg_requires / pkg_relations
def cmd_prereq(args_):
    parser = argparse.ArgumentParser(description='path')
    parser.add_argument('flavour', help='Flavour')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw',
        args.flavour, scope = 2, add_self=False, names_only=True)
    print(' '.join(deps))

def cmd_pkg_requires(args_):
    return pkg_relations("requires", args_)

def cmd_pkg_conflicts(args_):
    return pkg_relations("conflicts", args_)

def cmd_proj_dir(args_):
    parser = argparse.ArgumentParser(description='proj-dir')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        r.append(proj_dir(m))
    print(' '.join(r))

def cmd_htdocs_dir(args_):
    parser = argparse.ArgumentParser(description='htdocs-dir')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        r.append(htdocs_dir(m))
    print(' '.join(r))

def cmd_summary(args_):
    parser = argparse.ArgumentParser(description='summary')
    parser.add_argument('module', nargs='*', help='Modules')
    args=parser.parse_args(args_)
    r = []
    for m in args.module:
        summary = get_value(m, "summary", None)
        if summary is not None:
            r.append(summary)
    print(' '.join(r))

def contains(small, big):
    for i in xrange(len(big)-len(small)+1):
        for j in xrange(len(small)):
            if big[i+j] != small[j]:
                break
        else:
            return i, i+len(small)
    return False

def read_dep_graph(modules, section, graph):
    for m in modules:
        if m in graph:
            continue
        deps = get_modules_from_project_txt([ m ], 'pkg.requires.jw', section,
                scope = 1, add_self=False, names_only=True)
        if not deps is None:
            graph[m] = deps
            for d in deps:
                read_dep_graph([ d ], section, graph)

def flip_graph(graph):
    r = {}
    for m, deps in graph.iteritems():
        for d in deps:
            if not d in r:
                r[d] = Set()
            r[d].add(m)
    return r

def check_circular_deps(module, section, graph, unvisited, temp, path):
    if module in temp:
        debug('found circular dependency at module', module)
        return module
    if not module in unvisited:
        return None
    temp.add(module)
    if module in graph:
        for m in graph[module]:
            last = check_circular_deps(m, section, graph, unvisited, temp, path)
            if last is not None:
                path.insert(0, m)
                return last
    unvisited.remove(module)
    temp.remove(module)

def cmd_check(args_):
    parser = argparse.ArgumentParser(description='check')
    parser.add_argument('module', nargs='*', help='Modules')
    parser.add_argument('--flavour', '-f', nargs='?', default = 'build')
    args=parser.parse_args(args_)

    graph = {}
    path = []
    read_dep_graph(args.module, args.flavour, graph)
    unvisited = graph.keys()
    temp = Set()
    while len(unvisited) is not 0:
        m = unvisited[0]
        debug('checking circular dependency of', m)
        last = check_circular_deps(m, args.flavour, flip_graph(graph), unvisited, temp, path)
        if last is not None:
            debug('found circular dependency below', m, ', last is', last)
            print('found circular dependency in flavour', args.flavour, ':', ' -> '.join(path))
            exit(1)

    print('no circular dependency found for flavour', args.flavour, ' in modules:',
            ' '.join(args.module))
    exit(0)

def cmd_getval(args_):
    parser = argparse.ArgumentParser(description='Get value from project config')
    parser.add_argument('--project', default = top_name, help = 'Project name')
    parser.add_argument('section', default = '', help = 'Config section')
    parser.add_argument('key', default = '', help = 'Config key')
    args=parser.parse_args(args_)
    print(get_value(args.project, args.section, args.key))

# -------------------------------------------------------------------- here we go

global_args = []
cache = {}

skip = 0
for a in sys.argv[1::]:
    global_args.append(a)
    if a in [ '--prefix', '-p', '--topdir', '-t' ]:
        skip = 1
        continue
    if skip > 0:
        skip = skip -1
        continue
    if a[0] != '-':
        break

topdir = None
top_name = None

parser = argparse.ArgumentParser(description='Project metadata evaluation')
parser.add_argument('cmd', default='', help='Command')
parser.add_argument('--debug', '-d', action='store_true',
    default=False, help='Output debug information to stderr')
parser.add_argument('--topdir', '-t', nargs='?', help='Project Path')
parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") +
    '/local/src/jw.dev/proj', help='Projects Path Prefix')
parser.add_argument('arg', nargs='*', help='Command arguments')
args=parser.parse_args(global_args)

debug("----------------------------------------- running ", ' '.join(sys.argv))

projs_root = args.prefix
if args.topdir:
    topdir = args.topdir
    top_name = read_value_cached(topdir + '/make/project.conf', 'build', 'name')
    if not top_name:
        top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))

cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_'))
cmd(sys.argv[(len(global_args) + 1)::])