jw-pkg/src/python/jw/build/App.py

#!/usr/bin/python3 -u
# -*- coding: utf-8 -*-
#
# This source code file is a merge of various build tools and a horrible mess.
#

from __future__ import print_function
import os
import sys
import argparse
import pwd
import time
from os.path import isfile
from os.path import isdir
from os.path import expanduser
from os.path import basename
from os.path import realpath
import subprocess
import re
import platform
import datetime

# meaning of pkg.requires.xxx variables
# build: needs to be built and installed before this can be built
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
# run:   needs to be installed before this-run can be installed, i.e. before this and other packages can run with this

# --------------------------------------------------------------------- Python 2 / 3 compatibility stuff
try:
    basestring
except NameError:
    basestring = str

# --------------------------------------------------------------------- helpers

class ResultCache(object):

    def __init__(self):
        self.__cache = {}

    def run(self, func, args):
        d = self.__cache
        depth = 0
        keys = [ func.__name__ ] + args
        l = len(keys)
        for k in keys:
            if k is None:
                k = 'None'
            else:
                k = str(k)
            depth += 1
            #self.projects.debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
            if k in d:
                if l == depth:
                    return d[k]
                d = d[k]
                continue
            if l == depth:
                r = func(*args)
                d[k] = r
                return r
            d = d[k] = {}
            #d = d[k]
        raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)

# ----------------------------------------------------------------- class App

class App(object):

    def __init__(self):

        self.global_args = []
        self.opt_os = None
        self.top_name = None
        self.glob_os_cascade = None

        self.dep_cache = {}
        self.my_dir = os.path.dirname(os.path.realpath(__file__))
        self.opt_debug = False
        self.res_cache = ResultCache()
        self.topdir = None
        self.projs_root = expanduser("~") + '/local/src/jw.dev/proj'

    def debug(self, *objs):
        if self.opt_debug:
            print("DEBUG: ", *objs, file = sys.stderr)

    def warn(self, *objs):
            print("WARNING: ", *objs, file = sys.stderr)

    def err(self, *objs):
            print("ERR: ", *objs, file = sys.stderr)

    def find_proj_path_unused(name):
        name = name.replace("dspider-", "")
        search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
        for sub in search_path:
            path = self.projs_root + "/" + sub + "/" + name
            if os.path.exists(path):
                return os.path.abspath(path)
        raise Exception("module " + name + " not found below " + self.projs_root)

    def find_proj_path_cached_unused(name):
        return self.res_cache.run(find_proj_path_unused, [ name ])

    def proj_dir(self, name):
        if name == self.top_name:
            return self.topdir
        for d in [ self.projs_root, '/opt' ]:
            r = d + '/' + name
            if os.path.exists(r):
                return r
        if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'):
            # The package exists but does not have a dedicated project directory
            return None
        raise Exception('No project path found for module "{}"'.format(name))

    def re_section(self, name):
        return re.compile('[' + name + ']'
                     '.*?'
                     '(?=[)',
                     re.DOTALL)

    def remove_duplicates(self, seq):
        seen = set()
        seen_add = seen.add
        return [x for x in seq if not (x in seen or seen_add(x))]

    def get_os(self, args = ""):
        for d in [ self.projs_root + '/jw-build/scripts', '/opt/jw-build/bin' ]:
            script = d + '/get-os.sh'
            if isfile(script):
                cmd = '/bin/bash ' + script
                if args:
                    cmd = cmd + ' ' + args
                p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
                (out, rr) = p.communicate()
                if rr:
                    self.err("failed to run ", cmd)
                    continue
                out = re.sub('\n', '', out.decode('utf-8'))
                return out
        return "linux"

    # TODO: add support for customizing this in project.conf
    def htdocs_dir(self, name):
        pd = self.proj_dir(name)
        if pd is None:
            return None
        for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]:
            if isdir(r):
                return r
        return None

    def os_cascade(self):
        if self.glob_os_cascade is not None:
            return self.glob_os_cascade.copy()
        r = [ 'os', platform.system().lower() ]
        os = self.opt_os if self.opt_os is not None else self.res_cache.run(self.get_os, [])
        name = re.sub('-.*', '', os)
        series = os
        while True:
            n = re.sub(r'\.[0-9]+$', '', series)
            if n == series:
                break
            r.append(n)
            series = n
        if not name in r:
            r.append(name)
        if not os in r:
            r.append(os)
        # e.g. os, linux, suse, suse-tumbleweed
        #return [ 'os', platform.system().lower(), name, os ]
        self.glob_os_cascade = r
        return r

    def strip_module_from_spec(self, mod):
        return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())

    def get_section(self, path, section):
        r = ''
        file = open(path)
        pat = '[' + section + ']'
        in_section = False
        for line in file:
            if (line.rstrip() == pat):
                in_section = True
                continue
            if in_section:
                if len(line) and line[0] == '[':
                    break
                r = r + line
        file.close()
        return r.rstrip()

    def read_value(self, path, section, key):

        def scan_section(f, key):
            if key is None:
                r = ''
                for line in f:
                    if len(line) and line[0] == '[':
                        break
                    r += line
                return r if len(r) else None
            lines = []
            cont_line = ''
            for line in f:
                if len(line) and line[0] == '[':
                    break
                cont_line += line.rstrip()
                if len(cont_line) and cont_line[-1] == '\\':
                    cont_line = cont_line[0:-1]
                    continue
                lines.append(cont_line)
                cont_line = ''
            for line in lines:
                #self.debug("  looking for >%s< in line=>%s<" % (key, line))
                rr = re.findall('^ *' + key + ' *= *(.*)', line)
                if len(rr) > 0:
                    return rr[0]
            return None

        def scan_section_debug(f, key):
            rr = scan_section(f, key)
            #self.debug("  returning", rr)
            return rr

        try:
            #self.debug("looking for {}::[{}].{}".format(path, section, key))
            with open(path, 'r') as f:
                if not len(section):
                    rr = scan_section(f, key)
                pat = '[' + section + ']'
                for line in f:
                    if line.rstrip() == pat:
                        return scan_section(f, key)
                return None
        except:
            self.debug(path, "not found")
            # TODO: handle this special case cleaner somewhere up the stack
            if section == 'build' and key == 'libname':
                return 'none'
            return None

    def get_value(self, name, section, key):
        self.debug("getting value [%s].%s for project %s (%s)" %(section, key, name, self.top_name))
        if self.top_name and name == self.top_name:
            proj_root = self.topdir
        else:
            proj_root = self.projs_root + '/' + name
        self.debug("proj_root = " + proj_root)

        if section == 'version':
            proj_version_dirs = [ proj_root ]
            if proj_root != self.topdir:
                proj_version_dirs.append('/usr/share/doc/packages/' + name)
            for d in proj_version_dirs:
                version_path = d + '/VERSION'
                try:
                    with open(version_path) as fd:
                        r = fd.read().replace('\n', '').replace('-dev', '')
                        fd.close()
                        return r
                except EnvironmentError:
                    self.debug("ignoring unreadable file " + version_path)
                    continue
            raise Exception("No version file found for project \"" + name + "\"")

        path = proj_root + '/make/project.conf'
        #print('path = ', path, 'self.top_name = ', self.top_name, 'name = ', name)
        return self.res_cache.run(self.read_value, [path, section, key])

    def collect_values(self, names, section, key):
        r = ""
        for n in names:
            val = self.get_value(n, section, key)
            if val:
                r = r + " " + val
        return self.remove_duplicates([x.strip() for x in r.split(",")])

    # scope 0: no children
    # scope 1: children
    # scope 2: recursive

    def add_modules_from_project_txt_cached(self, buf, visited, spec, section, key, add_self, scope,
            names_only):
        return self.res_cache.run(self.add_modules_from_project_txt, [buf, visited, spec, section, key,
            add_self, scope, names_only])

    def add_modules_from_project_txt(self, buf, visited, spec, section, key, add_self, scope,
            names_only):
        name = self.strip_module_from_spec(spec)
        if names_only:
            spec = name
        if spec in buf:
            return
        if spec in visited:
            if add_self:
                buf.append(spec)
            return
        visited.add(spec)
        deps = self.get_value(name, section, key)
        self.debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited)
        if deps and scope > 0:
            if scope == 1:
                subscope = 0
            else:
                subscope = 2
            deps = deps.split(',')
            for dep in deps:
                dep = dep.strip()
                if not(len(dep)):
                    continue
                self.add_modules_from_project_txt_cached(buf, visited, dep,
                        section, key, add_self=True, scope=subscope,
                        names_only=names_only)
        if add_self:
            buf.append(spec)

    def get_modules_from_project_txt(self, names, sections, keys, add_self, scope,
            names_only = True):
        if isinstance(keys, basestring):
            keys = [ keys ]
        #r = set()
        r = []
        for section in sections:
            for key in keys:
                visited = set()
                for name in names:
                    rr = []
                    self.add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
                            names_only)
                    # TODO: this looks like a performance hogger
                    for m in rr:
                        if not m in r:
                           r.append(m)
        return r

    def pkg_relations_add_arguments(self, parser: argparse.ArgumentParser) -> None:
        parser.add_argument('-S', '--subsections', nargs='?', default=None, help='Subsections to consider, comma-separated')
        parser.add_argument('-d', '--delimiter', nargs='?', default=', ', help='Output words delimiter')
        parser.add_argument('flavour', help='Flavour')
        parser.add_argument('module', nargs='*', help='Modules')
        parser.add_argument('-p', '--no-subpackages', action='store_true',
            default=False, help='Cut -run and -devel from package names')
        parser.add_argument('--no-version', action='store_true',
            default=False, help='Don\'t report version information')
        parser.add_argument('--dont-strip-revision', action='store_true',
            default=False, help='Always treat VERSION macro as VERSION-REVISION')
        parser.add_argument('--recursive', action='store_true',
            default=False, help='Find dependencies recursively')
        parser.add_argument('--dont-expand-version-macros', action='store_true',
            default=False, help='Don\'t expand VERSION and REVISION macros')
        parser.add_argument('--ignore', nargs='?', default='', help='Packages that '
            'should be ignored together with their dependencies')

    def pkg_relations(self, rel_type, args):
        version_pattern=re.compile("[0-9-.]*")
        if args.subsections is None:
            subsecs = self.os_cascade()
            subsecs.append('jw')
        else:
            subsecs = args.subsections.split(',')
        self.debug('flavour = ', args.flavour, ', subsecs = ', ' '.join(subsecs))
        ignore = args.ignore.split(',')
        self.debug("ignore = ", ignore)

        r = []
        flavours = args.flavour.split(',')
        for flavour in flavours:
            for s in subsecs:
                section = 'pkg.' + rel_type + '.' + s
                visited = set()
                modules = args.module.copy()
                while len(modules):
                    m = modules.pop(0)
                    if m in visited or m in ignore:
                        continue
                    visited.add(m)
                    value = self.get_value(m, section, flavour)
                    if not value:
                        continue
                    deps = value.split(',')
                    for spec in deps:
                        dep = re.split('([=><]+)', spec)
                        if args.no_version:
                            dep = dep[:1]
                        dep = list(map(str.strip, dep))
                        dep_name = re.sub('-dev$|-devel$|-run$', '', dep[0])
                        if dep_name in ignore or dep[0] in ignore:
                            continue
                        if args.no_subpackages:
                            dep[0] = dep_name
                        for i, item in enumerate(dep):
                            dep[i] = item.strip()
                        if s == 'jw':
                            if args.recursive and not dep_name in visited and not dep_name in modules:
                                modules.append(dep_name)
                            if len(dep) == 3:
                                if args.dont_expand_version_macros and dep_name in args.module:
                                    version = dep[2]
                                else:
                                    version = self.get_value(dep_name, 'version', '')
                                if dep[2] == 'VERSION':
                                    if args.dont_strip_revision:
                                        dep[2] = version
                                    else:
                                        dep[2] = version.split('-')[0]
                                elif dep[2] == 'VERSION-REVISION':
                                    dep[2] = version
                                elif version_pattern.match(dep[2]):
                                    # dep[2] = dep[2]
                                    pass
                                else:
                                    raise Exception("Unknown version specifier in " + spec)
                        cleaned_dep = ' '.join(dep)
                        if not cleaned_dep in r:
                            self.debug("appending", cleaned_dep)
                            r.append(cleaned_dep)
        return args.delimiter.join(r)

    def print_pkg_relations(self, rel_type, args_):
        print(self.pkg_relations(rel_type, args_))

    def get_libname(self, names):
        vals = self.get_modules_from_project_txt(names, ['build'], 'libname',
                scope = 1, add_self=False, names_only=True)
        if not vals:
            return ' '.join(names)
        if 'none' in vals:
            vals.remove('none')
        return ' '.join(reversed(vals))

    def is_excluded_from_build(self, module):
        self.debug("checking if module " + module + " is excluded from build")
        exclude = self.get_modules_from_project_txt([ module ], ['build'], 'exclude',
                scope = 1, add_self=False, names_only=True)
        cascade = self.os_cascade() + [ 'all' ]
        for p1 in exclude:
            for p2 in cascade:
                if p1 == p2:
                    return p1
        return None

    # -L needs to contain more paths than libs linked with -l would require
    def get_ldpathflags(self, names, exclude = []):
        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
                scope = 2, add_self=True, names_only=True)
        r = ''
        for m in deps:
            if m in exclude:
                continue
            libname = self.get_libname([m])
            if len(libname):
                r = r + ' -L' + self.proj_dir(m) + '/lib'
        print(r[1:])

    def get_ldflags(self, names, exclude = [], add_self_ = False):
        #print(names)
        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
                scope = 1, add_self=add_self_, names_only=True)
        self.debug("deps = " + ' '.join(deps))
        #print(deps)
        r = ''
        for m in reversed(deps):
            if m in exclude:
                continue
            libname = self.get_libname([m])
            if len(libname):
                #r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname
                r = r + ' -l' + libname
        if len(r):
            ldpathflags = self.get_ldpathflags(names, exclude)
            if ldpathflags:
                r = ldpathflags + ' ' + r
            return r[1::]
        return ''

    # ----------------------------------------------------------------- commands

    def contains(self, small, big):
        for i in xrange(len(big)-len(small)+1):
            for j in xrange(len(small)):
                if big[i+j] != small[j]:
                    break
            else:
                return i, i+len(small)
        return False

    def read_dep_graph(self, modules, section, graph):
        for m in modules:
            if m in graph:
                continue
            deps = self.get_modules_from_project_txt([ m ], ['pkg.requires.jw'], section,
                    scope = 1, add_self=False, names_only=True)
            if not deps is None:
                graph[m] = deps
                for d in deps:
                    self.read_dep_graph([ d ], section, graph)

    def flip_graph(self, graph):
        r = {}
        for m, deps in graph.items():
            for d in deps:
                if not d in r:
                    r[d] = set()
                r[d].add(m)
        return r

    def check_circular_deps(self, module, section, graph, unvisited, temp, path):
        if module in temp:
            self.debug('found circular dependency at module', module)
            return module
        if not module in unvisited:
            return None
        temp.add(module)
        if module in graph:
            for m in graph[module]:
                last = self.check_circular_deps(m, section, graph, unvisited, temp, path)
                if last is not None:
                    path.insert(0, m)
                    return last
        unvisited.remove(module)
        temp.remove(module)

    def run_from_cmd_module(self, name: str, args_) -> None:
        import importlib
        name = name.replace('-', '_')
        cc_name = 'Cmd' + ''.join(x.capitalize() for x in name.lower().split("_"))
        module = importlib.import_module('jw.build.cmds.' + cc_name)
        cmd = getattr(module, cc_name)()
        cmd.app = self
        parser = argparse.ArgumentParser(description=name)
        cmd.add_arguments(parser)
        args = parser.parse_args(args_)
        return cmd.run(args)

    # -------------------------------------------------------------------- here we go
    def run(self):

        skip = 0
        for a in sys.argv[1::]:
            self.global_args.append(a)
            if a in [ '-p', '--prefix', '-t', '--topdir', '-O', '--os' ]:
                skip = 1
                continue
            if skip > 0:
                skip = skip -1
                continue
            if a[0] != '-':
                break

        # -- defaults
        self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"

        parser = argparse.ArgumentParser(description='Project metadata evaluation')
        parser.add_argument('-d', '--debug', action='store_true',
            default=False, help='Output debug information to stderr')
        parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path')
        parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ], help='App Path Prefix')
        parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system')
        parser.add_argument('cmd', default='', help='Command, run "{sys.argv[0]} commands" for a list of supported commands')
        parser.add_argument('arg', nargs='*', help='Command arguments')
        args = parser.parse_args(self.global_args)

        self.opt_debug = args.debug
        if len(args.os):
            self.opt_os = args.os[0]

        self.debug("----------------------------------------- running ", ' '.join(sys.argv))

        self.projs_root = args.prefix[0]
        self.debug("projs_root = ", self.projs_root)
        if len(args.topdir):
            self.topdir = args.topdir[0]

        if self.topdir:
            self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name'])
            if not self.top_name:
                self.top_name = re.sub('-[0-9.-]*$', '', basename(realpath(self.topdir)))

        try:
            return self.run_from_cmd_module(args.cmd, sys.argv[(len(self.global_args) + 1)::])
        except:
            raise
            pass
        cmd_name = 'cmd_' + args.cmd.replace('-', '_')
        cmd = getattr(self, cmd_name)
        cmd(sys.argv[(len(self.global_args) + 1)::])

if __name__ == "__main__":
    projects = App()
    projects.run()