jw-pkg/src/python/jw/pkg/App.py

# -*- coding: utf-8 -*-
#
# This source code file is a merge of various build tools and a horrible mess.
#

import os, sys, argparse, pwd, re

# meaning of pkg.requires.xxx variables
# build: needs to be built and installed before this can be built
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
# run:   needs to be installed before this-run can be installed, i.e. before this and other packages can run with this

# --------------------------------------------------------------------- helpers

class ResultCache(object):

    def __init__(self):
        self.__cache = {}

    def run(self, func, args):
        d = self.__cache
        depth = 0
        keys = [ func.__name__ ] + args
        l = len(keys)
        for k in keys:
            if k is None:
                k = 'None'
            else:
                k = str(k)
            depth += 1
            #self.projects.debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
            if k in d:
                if l == depth:
                    return d[k]
                d = d[k]
                continue
            if l == depth:
                r = func(*args)
                d[k] = r
                return r
            d = d[k] = {}
            #d = d[k]
        raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)

# ----------------------------------------------------------------- class App

class App(object):

    def __init__(self):

        self.global_args = []
        self.opt_os = None
        self.top_name = None
        self.glob_os_cascade = None

        self.dep_cache = {}
        self.my_dir = os.path.dirname(os.path.realpath(__file__))
        self.opt_debug = False
        self.res_cache = ResultCache()
        self.topdir = None
        self.projs_root = os.path.expanduser("~") + '/local/src/jw.dev/proj'

    def debug(self, *objs):
        if self.opt_debug:
            print("DEBUG: ", *objs, file = sys.stderr)

    def warn(self, *objs):
            print("WARNING: ", *objs, file = sys.stderr)

    def err(self, *objs):
            print("ERR: ", *objs, file = sys.stderr)

    def proj_dir(self, name):
        if name == self.top_name:
            return self.topdir
        for d in [ self.projs_root, '/opt' ]:
            r = d + '/' + name
            if os.path.exists(r):
                return r
        if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'):
            # The package exists but does not have a dedicated project directory
            return None
        raise Exception('No project path found for module "{}"'.format(name))

    def re_section(self, name):
        return re.compile('[' + name + ']'
                     '.*?'
                     '(?=[)',
                     re.DOTALL)

    def remove_duplicates(self, seq):
        seen = set()
        seen_add = seen.add
        return [x for x in seq if not (x in seen or seen_add(x))]

    def get_os(self, args = ""):
        import subprocess
        for d in [ self.projs_root + '/jw-pkg/scripts', '/opt/jw-pkg/bin' ]:
            script = d + '/get-os.sh'
            if os.path.isfile(script):
                cmd = '/bin/bash ' + script
                if args:
                    cmd = cmd + ' ' + args
                p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
                (out, rr) = p.communicate()
                if rr:
                    self.err("failed to run ", cmd)
                    continue
                out = re.sub('\n', '', out.decode('utf-8'))
                return out
        return "linux"

    # TODO: add support for customizing this in project.conf
    def htdocs_dir(self, name):
        pd = self.proj_dir(name)
        if pd is None:
            return None
        for r in [ pd + "/src/html/htdocs", pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]:
            if os.path.isdir(r):
                return r
        return None

    # TODO: add support for customizing this in project.conf
    def tmpl_dir(self, name):
        pd = self.proj_dir(name)
        if pd is None:
            return None
        for r in [ pd + "/tmpl", "/opt/" + name + "/share/tmpl" ]:
            if os.path.isdir(r):
                return r
        return None

    def os_cascade(self):
        import platform
        if self.glob_os_cascade is not None:
            return self.glob_os_cascade.copy()
        r = [ 'os', platform.system().lower() ]
        os = self.opt_os if self.opt_os is not None else self.res_cache.run(self.get_os, [])
        name = re.sub('-.*', '', os)
        series = os
        while True:
            n = re.sub(r'\.[0-9]+$', '', series)
            if n == series:
                break
            r.append(n)
            series = n
        if not name in r:
            r.append(name)
        if not os in r:
            r.append(os)
        # e.g. os, linux, suse, suse-tumbleweed
        #return [ 'os', platform.system().lower(), name, os ]
        self.glob_os_cascade = r
        return r

    def strip_module_from_spec(self, mod):
        return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())

    def get_section(self, path, section):
        r = ''
        file = open(path)
        pat = '[' + section + ']'
        in_section = False
        for line in file:
            if (line.rstrip() == pat):
                in_section = True
                continue
            if in_section:
                if len(line) and line[0] == '[':
                    break
                r = r + line
        file.close()
        return r.rstrip()

    def read_value(self, path, section, key):

        def scan_section(f, key):
            if key is None:
                r = ''
                for line in f:
                    if len(line) and line[0] == '[':
                        break
                    r += line
                return r if len(r) else None
            lines = []
            cont_line = ''
            for line in f:
                if len(line) and line[0] == '[':
                    break
                cont_line += line.rstrip()
                if len(cont_line) and cont_line[-1] == '\\':
                    cont_line = cont_line[0:-1]
                    continue
                lines.append(cont_line)
                cont_line = ''
            for line in lines:
                #self.debug("  looking for >%s< in line=>%s<" % (key, line))
                rr = re.findall('^ *' + key + ' *= *(.*)', line)
                if len(rr) > 0:
                    return rr[0]
            return None

        def scan_section_debug(f, key):
            rr = scan_section(f, key)
            #self.debug("  returning", rr)
            return rr

        try:
            #self.debug("looking for {}::[{}].{}".format(path, section, key))
            with open(path, 'r') as f:
                if not len(section):
                    rr = scan_section(f, key)
                pat = '[' + section + ']'
                for line in f:
                    if line.rstrip() == pat:
                        return scan_section(f, key)
                return None
        except:
            self.debug(path, "not found")
            # TODO: handle this special case cleaner somewhere up the stack
            if section == 'build' and key == 'libname':
                return 'none'
            return None

    def get_value(self, name, section, key):
        self.debug("getting value [%s].%s for project %s (%s)" %(section, key, name, self.top_name))
        if self.top_name and name == self.top_name:
            proj_root = self.topdir
        else:
            proj_root = self.projs_root + '/' + name
        self.debug("proj_root = " + proj_root)

        if section == 'version':
            proj_version_dirs = [ proj_root ]
            if proj_root != self.topdir:
                proj_version_dirs.append('/usr/share/doc/packages/' + name)
            for d in proj_version_dirs:
                version_path = d + '/VERSION'
                try:
                    with open(version_path) as fd:
                        r = fd.read().replace('\n', '').replace('-dev', '')
                        fd.close()
                        return r
                except EnvironmentError:
                    self.debug("ignoring unreadable file " + version_path)
                    continue
            raise Exception("No version file found for project \"" + name + "\"")

        path = proj_root + '/make/project.conf'
        #print('path = ', path, 'self.top_name = ', self.top_name, 'name = ', name)
        return self.res_cache.run(self.read_value, [path, section, key])

    def collect_values(self, names, section, key):
        r = ""
        for n in names:
            val = self.get_value(n, section, key)
            if val:
                r = r + " " + val
        return self.remove_duplicates([x.strip() for x in r.split(",")])

    # scope 0: no children
    # scope 1: children
    # scope 2: recursive

    def add_modules_from_project_txt_cached(self, buf, visited, spec, section, key, add_self, scope,
            names_only):
        return self.res_cache.run(self.add_modules_from_project_txt, [buf, visited, spec, section, key,
            add_self, scope, names_only])

    def add_modules_from_project_txt(self, buf, visited, spec, section, key, add_self, scope,
            names_only):
        name = self.strip_module_from_spec(spec)
        if names_only:
            spec = name
        if spec in buf:
            return
        if spec in visited:
            if add_self:
                buf.append(spec)
            return
        visited.add(spec)
        deps = self.get_value(name, section, key)
        self.debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited)
        if deps and scope > 0:
            if scope == 1:
                subscope = 0
            else:
                subscope = 2
            deps = deps.split(',')
            for dep in deps:
                dep = dep.strip()
                if not(len(dep)):
                    continue
                self.add_modules_from_project_txt_cached(buf, visited, dep,
                        section, key, add_self=True, scope=subscope,
                        names_only=names_only)
        if add_self:
            buf.append(spec)

    def get_modules_from_project_txt(self, names, sections, keys, add_self, scope,
            names_only = True):
        if isinstance(keys, str):
            keys = [ keys ]
        #r = set()
        r = []
        for section in sections:
            for key in keys:
                visited = set()
                for name in names:
                    rr = []
                    self.add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
                            names_only)
                    # TODO: this looks like a performance hogger
                    for m in rr:
                        if not m in r:
                           r.append(m)
        return r

    def get_libname(self, names):
        vals = self.get_modules_from_project_txt(names, ['build'], 'libname',
                scope = 1, add_self=False, names_only=True)
        if not vals:
            return ' '.join(names)
        if 'none' in vals:
            vals.remove('none')
        return ' '.join(reversed(vals))

    def is_excluded_from_build(self, module):
        self.debug("checking if module " + module + " is excluded from build")
        exclude = self.get_modules_from_project_txt([ module ], ['build'], 'exclude',
                scope = 1, add_self=False, names_only=True)
        cascade = self.os_cascade() + [ 'all' ]
        for p1 in exclude:
            for p2 in cascade:
                if p1 == p2:
                    return p1
        return None

    # -L needs to contain more paths than libs linked with -l would require
    def get_ldpathflags(self, names, exclude = []):
        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
                scope = 2, add_self=True, names_only=True)
        r = ''
        for m in deps:
            if m in exclude:
                continue
            libname = self.get_libname([m])
            if len(libname):
                r = r + ' -L' + self.proj_dir(m) + '/lib'
        print(r[1:])

    def get_ldflags(self, names, exclude = [], add_self_ = False):
        #print(names)
        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
                scope = 1, add_self=add_self_, names_only=True)
        self.debug("deps = " + ' '.join(deps))
        #print(deps)
        r = ''
        for m in reversed(deps):
            if m in exclude:
                continue
            libname = self.get_libname([m])
            if len(libname):
                #r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname
                r = r + ' -l' + libname
        if len(r):
            ldpathflags = self.get_ldpathflags(names, exclude)
            if ldpathflags:
                r = ldpathflags + ' ' + r
            return r[1::]
        return ''

    def contains(self, small, big):
        for i in xrange(len(big)-len(small)+1):
            for j in xrange(len(small)):
                if big[i+j] != small[j]:
                    break
            else:
                return i, i+len(small)
        return False

    def read_dep_graph(self, modules, section, graph):
        for m in modules:
            if m in graph:
                continue
            deps = self.get_modules_from_project_txt([ m ], ['pkg.requires.jw'], section,
                    scope = 1, add_self=False, names_only=True)
            if not deps is None:
                graph[m] = deps
                for d in deps:
                    self.read_dep_graph([ d ], section, graph)

    def flip_graph(self, graph):
        r = {}
        for m, deps in graph.items():
            for d in deps:
                if not d in r:
                    r[d] = set()
                r[d].add(m)
        return r

    def check_circular_deps(self, module, section, graph, unvisited, temp, path):
        if module in temp:
            self.debug('found circular dependency at module', module)
            return module
        if not module in unvisited:
            return None
        temp.add(module)
        if module in graph:
            for m in graph[module]:
                last = self.check_circular_deps(m, section, graph, unvisited, temp, path)
                if last is not None:
                    path.insert(0, m)
                    return last
        unvisited.remove(module)
        temp.remove(module)

    # -------------------------------------------------------------------- here we go
    def run(self):

        skip = 0
        for a in sys.argv[1::]:
            self.global_args.append(a)
            if a in [ '-p', '--prefix', '-t', '--topdir', '-O', '--os' ]:
                skip = 1
                continue
            if skip > 0:
                skip = skip -1
                continue
            if a[0] != '-':
                break

        # -- defaults
        self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"

        parser = argparse.ArgumentParser(description='Project metadata evaluation')
        parser.add_argument('-d', '--debug', action='store_true',
            default=False, help='Output debug information to stderr')
        parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path')
        parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ],
            help='Parent directory of project source directories')
        parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system')
        parser.add_argument('cmd', default='', help=f'Command, run "{sys.argv[0]} commands" for a list of supported commands')
        parser.add_argument('arg', nargs='*', help='Command arguments')
        args = parser.parse_args(self.global_args)

        self.opt_debug = args.debug
        if len(args.os):
            self.opt_os = args.os[0]

        self.debug("----------------------------------------- running ", ' '.join(sys.argv))

        self.projs_root = args.prefix[0]
        self.debug("projs_root = ", self.projs_root)
        if len(args.topdir):
            self.topdir = args.topdir[0]

        if self.topdir:
            self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name'])
            if not self.top_name:
                self.top_name = re.sub('-[0-9.-]*$', '', os.path.basename(os.path.realpath(self.topdir)))

        import importlib
        cmd_name = args.cmd.replace('-', '_')
        cc_cmd_name = 'Cmd' + ''.join(x.capitalize() for x in cmd_name.lower().split("_"))
        module = importlib.import_module('jw.pkg.build.cmds.' + cc_cmd_name)
        cmd = getattr(module, cc_cmd_name)()
        cmd.app = self
        subparser = argparse.ArgumentParser(description=cmd_name)
        cmd.add_arguments(subparser)
        args = subparser.parse_args(sys.argv[(len(self.global_args) + 1)::])
        try:
            return cmd.run(args)
        except Exception as e:
            self.err('Failed to run >{}<: {}'.format(' '.join(sys.argv), e))
            raise
            sys.exit(1)