diff --git a/make/projects-dir.mk b/make/projects-dir.mk index ee889764..cf734b89 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -131,7 +131,7 @@ EDITOR ?= xdg-open ifeq ($(TIME),) TIME = $(shell which time) -p endif -PROJECTS_PY = python3 $(JWB_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS) +PROJECTS_PY = PYTHONPATH=$(realpath $(JWBDIR)/src/python) $(TIME) python3 $(JWB_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS) PROJECTS_PY_BUILD = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS) PGIT_SH = /bin/bash $(JWB_SCRIPT_DIR)/pgit.sh PURGE_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/purge-stale-projects.sh $(JW_BUILD_BINDIR)/purge-stale-projects.sh) purge-not-found) @@ -154,7 +154,7 @@ CVS_PROJECTS = $(patsubst %/,%,$(dir $(wildcard $(addsuffix /CVS # --- mandatory targets all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) - $(TIME) $(PROJECTS_PY_BUILD) $@ $(TARGET_PROJECTS) + $(PROJECTS_PY_BUILD) $@ $(TARGET_PROJECTS) clean: clean-dirs distclean: clean-all-dirs done.clean install: diff --git a/make/projects.mk b/make/projects.mk index b904474a..cf71f918 100644 --- a/make/projects.mk +++ b/make/projects.mk @@ -30,7 +30,7 @@ endif include $(JWBDIR)/make/py-version.mk ifneq ($(TOPDIR),) - proj_query_cmd = $(PYTHON) $(JWB_SCRIPT_DIR)/projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_ARGS) + proj_query_cmd = PYTHONPATH=$(JWBDIR)/src/python $(PYTHON) $(JWB_SCRIPT_DIR)/projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_ARGS) proj_query = $(shell $(proj_query_cmd) $(1)) proj_dir = $(call proj_query,proj-dir $(1)) htdocs_dir = $(call proj_query,htdocs-dir $(1)) diff --git a/scripts/projects.py b/scripts/projects.py index 9a9fb881..08d0eceb 100644 --- a/scripts/projects.py +++ b/scripts/projects.py @@ -1,1012 +1,5 @@ -#!/usr/bin/python3 -u # -*- coding: utf-8 -*- -# -# This source code file is a merge of various build tools and a horrible mess. -# -from __future__ import print_function -import os -import sys -import argparse -import pwd -import time -from os.path import isfile -from os.path import isdir -from os.path import expanduser -from os.path import basename -from os.path import realpath -import subprocess -import re -import platform -import datetime +from jw.build.App import App -# meaning of pkg.requires.xxx variables -# build: needs to be built and installed before this can be built -# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this -# run: needs to be installed before this-run can be installed, i.e. before this and other packages can run with this - -# --------------------------------------------------------------------- Python 2 / 3 compatibility stuff -try: - basestring -except NameError: - basestring = str - -# --------------------------------------------------------------------- helpers - -class ResultCache(object): - - def __init__(self): - self.__cache = {} - - def run(self, func, args): - d = self.__cache - depth = 0 - keys = [ func.__name__ ] + args - l = len(keys) - for k in keys: - if k is None: - k = 'None' - else: - k = str(k) - depth += 1 - #self.projects.debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) - if k in d: - if l == depth: - return d[k] - d = d[k] - continue - if l == depth: - r = func(*args) - d[k] = r - return r - d = d[k] = {} - #d = d[k] - raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) - -# ----------------------------------------------------------------- class Projects - -class Projects(object): - - def __init__(self): - - self.global_args = [] - self.opt_os = None - self.top_name = None - self.glob_os_cascade = None - - self.dep_cache = {} - self.my_dir = os.path.dirname(os.path.realpath(__file__)) - self.opt_debug = False - self.res_cache = ResultCache() - self.topdir = None - self.projs_root = expanduser("~") + '/local/src/jw.dev/proj' - - def debug(self, *objs): - if self.opt_debug: - print("DEBUG: ", *objs, file = sys.stderr) - - def warn(self, *objs): - print("WARNING: ", *objs, file = sys.stderr) - - def err(self, *objs): - print("ERR: ", *objs, file = sys.stderr) - - def find_proj_path_unused(name): - name = name.replace("dspider-", "") - search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] - for sub in search_path: - path = self.projs_root + "/" + sub + "/" + name - if os.path.exists(path): - return os.path.abspath(path) - raise Exception("module " + name + " not found below " + self.projs_root) - - def find_proj_path_cached_unused(name): - return self.res_cache.run(find_proj_path_unused, [ name ]) - - def proj_dir(self, name): - if name == self.top_name: - return self.topdir - for d in [ self.projs_root, '/opt' ]: - r = d + '/' + name - if os.path.exists(r): - return r - if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'): - # The package exists but does not have a dedicated project directory - return None - raise Exception('No project path found for module "{}"'.format(name)) - - def re_section(self, name): - return re.compile('[' + name + ']' - '.*?' - '(?=[)', - re.DOTALL) - - def remove_duplicates(self, seq): - seen = set() - seen_add = seen.add - return [x for x in seq if not (x in seen or seen_add(x))] - - def get_os(self, args = ""): - for d in [ self.projs_root + '/jw-build/scripts', '/opt/jw-build/bin' ]: - script = d + '/get-os.sh' - if isfile(script): - cmd = '/bin/bash ' + script - if args: - cmd = cmd + ' ' + args - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - (out, rr) = p.communicate() - if rr: - self.err("failed to run ", cmd) - continue - out = re.sub('\n', '', out.decode('utf-8')) - return out - return "linux" - - # TODO: add support for customizing this in project.conf - def htdocs_dir(self, name): - pd = self.proj_dir(name) - if pd is None: - return None - for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]: - if isdir(r): - return r - return None - - def os_cascade(self): - if self.glob_os_cascade is not None: - return self.glob_os_cascade.copy() - r = [ 'os', platform.system().lower() ] - os = self.opt_os if self.opt_os is not None else self.res_cache.run(self.get_os, []) - name = re.sub('-.*', '', os) - series = os - while True: - n = re.sub(r'\.[0-9]+$', '', series) - if n == series: - break - r.append(n) - series = n - if not name in r: - r.append(name) - if not os in r: - r.append(os) - # e.g. os, linux, suse, suse-tumbleweed - #return [ 'os', platform.system().lower(), name, os ] - self.glob_os_cascade = r - return r - - def strip_module_from_spec(self, mod): - return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip()) - - def get_section(self, path, section): - r = '' - file = open(path) - pat = '[' + section + ']' - in_section = False - for line in file: - if (line.rstrip() == pat): - in_section = True - continue - if in_section: - if len(line) and line[0] == '[': - break - r = r + line - file.close() - return r.rstrip() - - def read_value(self, path, section, key): - - def scan_section(f, key): - if key is None: - r = '' - for line in f: - if len(line) and line[0] == '[': - break - r += line - return r if len(r) else None - lines = [] - cont_line = '' - for line in f: - if len(line) and line[0] == '[': - break - cont_line += line.rstrip() - if len(cont_line) and cont_line[-1] == '\\': - cont_line = cont_line[0:-1] - continue - lines.append(cont_line) - cont_line = '' - for line in lines: - #self.debug(" looking for >%s< in line=>%s<" % (key, line)) - rr = re.findall('^ *' + key + ' *= *(.*)', line) - if len(rr) > 0: - return rr[0] - return None - - def scan_section_debug(f, key): - rr = scan_section(f, key) - #self.debug(" returning", rr) - return rr - - try: - #self.debug("looking for {}::[{}].{}".format(path, section, key)) - with open(path, 'r') as f: - if not len(section): - rr = scan_section(f, key) - pat = '[' + section + ']' - for line in f: - if line.rstrip() == pat: - return scan_section(f, key) - return None - except: - self.debug(path, "not found") - # TODO: handle this special case cleaner somewhere up the stack - if section == 'build' and key == 'libname': - return 'none' - return None - - def get_value(self, name, section, key): - self.debug("getting value [%s].%s for project %s (%s)" %(section, key, name, self.top_name)) - if self.top_name and name == self.top_name: - proj_root = self.topdir - else: - proj_root = self.projs_root + '/' + name - self.debug("proj_root = " + proj_root) - - if section == 'version': - proj_version_dirs = [ proj_root ] - if proj_root != self.topdir: - proj_version_dirs.append('/usr/share/doc/packages/' + name) - for d in proj_version_dirs: - version_path = d + '/VERSION' - try: - with open(version_path) as fd: - r = fd.read().replace('\n', '').replace('-dev', '') - fd.close() - return r - except EnvironmentError: - self.debug("ignoring unreadable file " + version_path) - continue - raise Exception("No version file found for project \"" + name + "\"") - - path = proj_root + '/make/project.conf' - #print('path = ', path, 'self.top_name = ', self.top_name, 'name = ', name) - return self.res_cache.run(self.read_value, [path, section, key]) - - def collect_values(self, names, section, key): - r = "" - for n in names: - val = self.get_value(n, section, key) - if val: - r = r + " " + val - return self.remove_duplicates([x.strip() for x in r.split(",")]) - - # scope 0: no children - # scope 1: children - # scope 2: recursive - - def add_modules_from_project_txt_cached(self, buf, visited, spec, section, key, add_self, scope, - names_only): - return self.res_cache.run(self.add_modules_from_project_txt, [buf, visited, spec, section, key, - add_self, scope, names_only]) - - def add_modules_from_project_txt(self, buf, visited, spec, section, key, add_self, scope, - names_only): - name = self.strip_module_from_spec(spec) - if names_only: - spec = name - if spec in buf: - return - if spec in visited: - if add_self: - buf.append(spec) - return - visited.add(spec) - deps = self.get_value(name, section, key) - self.debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited) - if deps and scope > 0: - if scope == 1: - subscope = 0 - else: - subscope = 2 - deps = deps.split(',') - for dep in deps: - dep = dep.strip() - if not(len(dep)): - continue - self.add_modules_from_project_txt_cached(buf, visited, dep, - section, key, add_self=True, scope=subscope, - names_only=names_only) - if add_self: - buf.append(spec) - - def get_modules_from_project_txt(self, names, sections, keys, add_self, scope, - names_only = True): - if isinstance(keys, basestring): - keys = [ keys ] - #r = set() - r = [] - for section in sections: - for key in keys: - visited = set() - for name in names: - rr = [] - self.add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope, - names_only) - # TODO: this looks like a performance hogger - for m in rr: - if not m in r: - r.append(m) - return r - - def pkg_relations(self, rel_type, args_): - parser = argparse.ArgumentParser(description='pkg-' + rel_type) - # TODO: implement Vendor evaluation - - parser.add_argument('-S', '--subsections', nargs='?', default=None, help='Subsections to consider, comma-separated') - parser.add_argument('-d', '--delimiter', nargs='?', default=', ', help='Output words delimiter') - parser.add_argument('flavour', help='Flavour') - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('-p', '--no-subpackages', action='store_true', - default=False, help='Cut -run and -devel from package names') - parser.add_argument('--no-version', action='store_true', - default=False, help='Don\'t report version information') - parser.add_argument('--dont-strip-revision', action='store_true', - default=False, help='Always treat VERSION macro as VERSION-REVISION') - parser.add_argument('--recursive', action='store_true', - default=False, help='Find dependencies recursively') - parser.add_argument('--dont-expand-version-macros', action='store_true', - default=False, help='Don\'t expand VERSION and REVISION macros') - parser.add_argument('--ignore', nargs='?', default='', help='Packages that ' - 'should be ignored together with their dependencies') - args = parser.parse_args(args_) - version_pattern=re.compile("[0-9-.]*") - if args.subsections is None: - subsecs = self.os_cascade() - subsecs.append('jw') - else: - subsecs = args.subsections.split(',') - self.debug('flavour = ', args.flavour, ', subsecs = ', ' '.join(subsecs)) - ignore = args.ignore.split(',') - self.debug("ignore = ", ignore) - - r = [] - flavours = args.flavour.split(',') - for flavour in flavours: - for s in subsecs: - section = 'pkg.' + rel_type + '.' + s - visited = set() - modules = args.module.copy() - while len(modules): - m = modules.pop(0) - if m in visited or m in ignore: - continue - visited.add(m) - value = self.get_value(m, section, flavour) - if not value: - continue - deps = value.split(',') - for spec in deps: - dep = re.split('([=><]+)', spec) - if args.no_version: - dep = dep[:1] - dep = list(map(str.strip, dep)) - dep_name = re.sub('-dev$|-devel$|-run$', '', dep[0]) - if dep_name in ignore or dep[0] in ignore: - continue - if args.no_subpackages: - dep[0] = dep_name - for i, item in enumerate(dep): - dep[i] = item.strip() - if s == 'jw': - if args.recursive and not dep_name in visited and not dep_name in modules: - modules.append(dep_name) - if len(dep) == 3: - if args.dont_expand_version_macros and dep_name in args.module: - version = dep[2] - else: - version = self.get_value(dep_name, 'version', '') - if dep[2] == 'VERSION': - if args.dont_strip_revision: - dep[2] = version - else: - dep[2] = version.split('-')[0] - elif dep[2] == 'VERSION-REVISION': - dep[2] = version - elif version_pattern.match(dep[2]): - # dep[2] = dep[2] - pass - else: - raise Exception("Unknown version specifier in " + spec) - cleaned_dep = ' '.join(dep) - if not cleaned_dep in r: - self.debug("appending", cleaned_dep) - r.append(cleaned_dep) - return args.delimiter.join(r) - - def print_pkg_relations(self, rel_type, args_): - print(self.pkg_relations(rel_type, args_)) - - def get_libname(self, names): - vals = self.get_modules_from_project_txt(names, ['build'], 'libname', - scope = 1, add_self=False, names_only=True) - if not vals: - return ' '.join(names) - if 'none' in vals: - vals.remove('none') - return ' '.join(reversed(vals)) - - def is_excluded_from_build(self, module): - self.debug("checking if module " + module + " is excluded from build") - exclude = self.get_modules_from_project_txt([ module ], ['build'], 'exclude', - scope = 1, add_self=False, names_only=True) - cascade = self.os_cascade() + [ 'all' ] - for p1 in exclude: - for p2 in cascade: - if p1 == p2: - return p1 - return None - - # -L needs to contain more paths than libs linked with -l would require - def get_ldpathflags(self, names, exclude = []): - deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build', - scope = 2, add_self=True, names_only=True) - r = '' - for m in deps: - if m in exclude: - continue - libname = self.get_libname([m]) - if len(libname): - r = r + ' -L' + self.proj_dir(m) + '/lib' - print(r[1:]) - - def get_ldflags(self, names, exclude = [], add_self_ = False): - #print(names) - deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build', - scope = 1, add_self=add_self_, names_only=True) - self.debug("deps = " + ' '.join(deps)) - #print(deps) - r = '' - for m in reversed(deps): - if m in exclude: - continue - libname = self.get_libname([m]) - if len(libname): - #r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname - r = r + ' -l' + libname - if len(r): - ldpathflags = self.get_ldpathflags(names, exclude) - if ldpathflags: - r = ldpathflags + ' ' + r - return r[1::] - return '' - - def commands(self): - f = open(sys.argv[0]) - cmds = [] - for line in f: - self.debug("checking line ", line) - rr = re.findall('^ def *cmd_([a-z0-9_]+).*', line) - if len(rr): - cmds.append(rr[0].replace('_', '-')) - f.close() - return ' '.join(cmds) - - # ----------------------------------------------------------------- commands - - def cmd_commands(self, args_): - print(self.commands()) - - def cmd_modules(self, args_): - proj_root = self.projs_root - self.debug("proj_root = " + proj_root) - path = pathlib.Path(self.projs_root) - - parser = argparse.ArgumentParser(description='Query existing janware packages') - - parser.add_argument('-F', '--filter', nargs='?', default=None, help='Key-value pairs, seperated by commas, to be searched for in project.conf') - args = parser.parse_args(args_) - modules = [p.parents[1].name for p in path.glob('*/make/project.conf')] - self.debug("modules = ", modules) - out = [] - filters = None if args.filter is None else [re.split("=", f) for f in re.split(",", args.filter)] - for m in modules: - if filters: - for f in filters: - path = f[0].rsplit('.') - if len(path) > 1: - sec = path[0] - key = path[1] - else: - sec = None - key = path[0] - val = self.get_value(m, sec, key) - self.debug('Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val)) - if val and val == f[1]: - out.append(m) - break - else: - out.append(m) - print(' '.join(out)) - - def cmd_build(self, args_): - - def read_deps(cur, prereq_type): - # dep cache doesn't make a difference at all - if prereq_type in self.dep_cache: - if cur in self.dep_cache[prereq_type]: - return self.dep_cache[prereq_type][cur] - else: - self.dep_cache[prereq_type] = {} - - r = self.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'], - prereq_type, scope = 2, add_self=False, names_only=True) - self.debug('prerequisites = ' + ' '.join(r)) - if cur in r: - r.remove(cur) - self.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) - self.dep_cache[prereq_type][cur] = r - return r - - def read_deps_cached(cur, prereq_type): - return self.res_cache.run(read_deps, [ cur, prereq_type ]) - - def add_dep_tree(cur, prereq_types, tree, all_deps): - self.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) - if cur in all_deps: - self.debug('already handled module ' + cur) - return 0 - deps = set() - all_deps.add(cur) - for t in prereq_types: - self.debug("checking prereqisites of type " + t) - deps.update(read_deps_cached(cur, t)) - for d in deps: - add_dep_tree(d, prereq_types, tree, all_deps) - tree[cur] = deps - return len(deps) - - def calculate_order(order, modules, prereq_types): - all_deps = set() - dep_tree = {} - for m in modules: - self.debug("--- adding dependency tree of module " + m) - add_dep_tree(m, prereq_types, dep_tree, all_deps) - while len(all_deps): - # Find any leaf - for d in all_deps: - if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself - break # found - else: # no Leaf found - print(all_deps) - raise Exception("fatal: the dependencies between these modules are unresolvable") - order.append(d) # do it - # bookkeep it - all_deps.remove(d) - for k in dep_tree.keys(): - if d in dep_tree[k]: - dep_tree[k].remove(d) - return 1 - - def run_make(module, target, cur_project, num_projects): - #make_cmd = "make " + target + " 2>&1" - make_cmd = [ "make", target ] - path = self.proj_dir(module) - delim_len = 120 - delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path) - delim = delim + '-' * (delim_len - len(delim)) - - print(',' + delim + ' >') - - patt = self.is_excluded_from_build(module) - if patt is not None: - print('| Configured to skip build on platform >' + patt + '<') - print('`' + delim + ' <') - return - - os.chdir(path) - p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) - for line in iter(p.stdout.readline, b''): - line = line.decode(sys.stdout.encoding) - sys.stdout.write('| ' + line) # avoid extra newlines from print() - sys.stdout.flush() - p.wait() - print('`' + delim + ' <') - if p.returncode: - print(' '.join(make_cmd) + ' failed') - raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projs_root) - - def run_make_on_modules(modules, order, target): - cur_project = 0 - num_projects = len(order) - if target in ["clean", "distclean"]: - for m in reversed(order): - cur_project += 1 - run_make(m, target, cur_project, num_projects) - if m in modules: - modules.remove(m) - if not len(modules): - print("all modules cleaned") - return - else: - for m in order: - cur_project += 1 - run_make(m, target, cur_project, num_projects) - - def run(args_): - - # -- parse command line - parser = argparse.ArgumentParser(description='janware software project build tool') - parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') - parser.add_argument('-n', '--dry-run', action='store_true', - default=False, help='Don\'t build anything, just print what would be done.') - parser.add_argument('-O', '--build-order', action='store_true', - default=False, help='Don\'t build anything, just print the build order.') - parser.add_argument('-I', '--ignore-deps', action='store_true', - default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') - parser.add_argument('target', default='all', help='Build target') - parser.add_argument('modules', nargs='+', default='', help='Modules to be built') - - args = parser.parse_args(args_) - - self.debug("----------------------------------------- running ", ' '.join(args_)) - - modules = args.modules - exclude = args.exclude.split() - target = args.target - - env_exclude = os.getenv('BUILD_EXCLUDE', '') - if len(env_exclude): - print("exluding modules from environment: " + env_exclude) - exclude += " " + env_exclude - - # -- build - order = [] - - glob_prereq_types = [ "build" ] - if re.match("pkg-.*", target) is not None: - glob_prereq_types = [ "build", "run", "release", "devel" ] - - if target != 'order' and not args.build_order: - print("using prerequisite types " + ' '.join(glob_prereq_types)) - print("calculating order for modules ... ") - - calculate_order(order, modules, glob_prereq_types) - if args.ignore_deps: - order = [m for m in order if m in args.modules] - order = [m for m in order if m not in exclude] - if target == 'order' or args.build_order: - print(' '.join(order)) - exit(0) - - cur_project = 0 - print("Building target %s in %d projects:" % (target, len(order))) - for m in order: - cur_project += 1 - print(" %3d %s" % (cur_project, m)) - - if args.dry_run: - exit(0) - - run_make_on_modules(modules, order, target) - - print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - run(args_) - - def cmd_test(self, args_): - parser = argparse.ArgumentParser(description='Test') - parser.add_argument('blah', default='', help='The blah argument') - args = parser.parse_args(args_) - print("blah = " + args.blah) - - def cmd_os_cascade(self, args_): - print(' '.join(self.os_cascade())) - - def cmd_ldlibpath(self, args_): - parser = argparse.ArgumentParser(description='ldlibpath') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], - scope = 2, add_self=True, names_only=True) - r = '' - for m in deps: - pd = self.proj_dir(m) - if pd is None: - continue - r = r + ':' + pd + '/lib' - print(r[1:]) - - def cmd_pythonpath(self, args_): - parser = argparse.ArgumentParser(description='pythonpath') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ], - scope = 2, add_self=True, names_only=True) - r = '' - for m in deps: - pd = self.proj_dir(m) - if pd is None: - continue - for subdir in [ 'src/python', 'tools/python' ]: - cand = pd + "/" + subdir - if isdir(cand): - r = r + ':' + cand - print(r[1:]) - - def cmd_exepath(self, args_): - parser = argparse.ArgumentParser(description='exepath') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], - scope = 2, add_self=True, names_only=True) - self.debug('deps = ', deps) - r = '' - for m in deps: - pd = self.proj_dir(m) - if pd is None: - continue - r = r + ':' + pd + '/bin' - print(r[1:]) - - def cmd_libname(self, args_): - parser = argparse.ArgumentParser(description='libname') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - print(self.get_libname(args.module)) - - def cmd_ldflags(self, args_): - parser = argparse.ArgumentParser(description='ldflags') - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[]) - parser.add_argument('-s', '--add-self', action='store_true', - default=False, help='Include libflags of specified modules, too, not only their dependencies') - args = parser.parse_args(args_) - print(self.get_ldflags(args.module, args.exclude, args.add_self)) - - def cmd_cflags(self, args_): - parser = argparse.ArgumentParser(description='cflags') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build', - scope = 2, add_self=True, names_only=True) - r = '' - for m in reversed(deps): - try: - pd = self.proj_dir(m) - if pd is None: - continue - r = r + ' -I' + pd + '/include' - except Exception as e: - self.warn(f'No include path for module "{m}", ignoring: {e}') - print(r[1:]) - - def cmd_path(self, args_): - parser = argparse.ArgumentParser(description='path') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'run', - scope = 2, add_self=True, names_only=True) - r = '' - for m in deps: - pd = self.proj_dir(m) - if pd is None: - continue - r = r + ':' + pd + '/bin' - print(r[1:]) - - # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations - def cmd_prereq(self, args_): - parser = argparse.ArgumentParser(description='path') - parser.add_argument('flavour', help='Flavour') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], - args.flavour, scope = 2, add_self=False, names_only=True) - print(' '.join(deps)) - - # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations - def cmd_required_os_pkg(self, args_): - parser = argparse.ArgumentParser(description='required-os-pkg') - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('--flavours', help='Dependency flavours', default='build') - parser.add_argument('--skip-excluded', action='store_true', default=False, - help='Output empty prerequisite list if module is excluded') - args = parser.parse_args(args_) - modules = args.module - flavours = args.flavours.split() - if 'build' in flavours and not 'run' in flavours: - # TODO: This adds too much. Only the run dependencies of the build dependencies would be needed. - flavours.append('run') - self.debug("flavours = " + args.flavours) - deps = self.get_modules_from_project_txt(modules, ['pkg.requires.jw'], flavours, - scope = 2, add_self=True, names_only=True) - if args.skip_excluded: - for d in deps: - if self.is_excluded_from_build(d) is not None: - deps.remove(d) - subsecs = self.os_cascade() - self.debug("subsecs = ", subsecs) - requires = [] - for s in subsecs: - for f in flavours: - vals = self.collect_values(deps, 'pkg.requires.' + s, f) - if vals: - requires = requires + vals - # TODO: add all not in build tree as -devel - r = '' - for m in requires: - r = r + ' ' + m - print(r[1:]) - - def cmd_pkg_requires(self, args_): - return self.print_pkg_relations("requires", args_) - - def cmd_pkg_conflicts(self, args_): - return self.print_pkg_relations("conflicts", args_) - - def cmd_pkg_provides(self, args_): - return self.print_pkg_relations("provides", args_) - - def cmd_proj_dir(self, args_): - parser = argparse.ArgumentParser(description='proj-dir') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - r = [] - for m in args.module: - try: - pd = self.proj_dir(m) - if pd is None: - continue - r.append(pd) - except Exception as e: - self.warn(f'No project directory for module "{m}: {e}') - continue - print(' '.join(r)) - - def cmd_htdocs_dir(self, args_): - parser = argparse.ArgumentParser(description='htdocs-dir') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - r = [] - for m in args.module: - r.append(self.htdocs_dir(m)) - print(' '.join(r)) - - def cmd_summary(self, args_): - parser = argparse.ArgumentParser(description='summary') - parser.add_argument('module', nargs='*', help='Modules') - args = parser.parse_args(args_) - r = [] - for m in args.module: - summary = self.get_value(m, "summary", None) - if summary is not None: - r.append(summary) - print(' '.join(r)) - - def contains(self, small, big): - for i in xrange(len(big)-len(small)+1): - for j in xrange(len(small)): - if big[i+j] != small[j]: - break - else: - return i, i+len(small) - return False - - def read_dep_graph(self, modules, section, graph): - for m in modules: - if m in graph: - continue - deps = self.get_modules_from_project_txt([ m ], ['pkg.requires.jw'], section, - scope = 1, add_self=False, names_only=True) - if not deps is None: - graph[m] = deps - for d in deps: - self.read_dep_graph([ d ], section, graph) - - def flip_graph(self, graph): - r = {} - for m, deps in graph.items(): - for d in deps: - if not d in r: - r[d] = set() - r[d].add(m) - return r - - def check_circular_deps(self, module, section, graph, unvisited, temp, path): - if module in temp: - self.debug('found circular dependency at module', module) - return module - if not module in unvisited: - return None - temp.add(module) - if module in graph: - for m in graph[module]: - last = self.check_circular_deps(m, section, graph, unvisited, temp, path) - if last is not None: - path.insert(0, m) - return last - unvisited.remove(module) - temp.remove(module) - - def cmd_check(self, args_): - parser = argparse.ArgumentParser(description='check') - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('-f', '--flavour', nargs='?', default = 'build') - args = parser.parse_args(args_) - - graph = {} - path = [] - self.read_dep_graph(args.module, args.flavour, graph) - unvisited = graph.keys() - temp = set() - while len(unvisited) != 0: - m = unvisited[0] - self.debug('checking circular dependency of', m) - last = self.check_circular_deps(m, args.flavour, self.flip_graph(graph), unvisited, temp, path) - if last is not None: - self.debug('found circular dependency below', m, ', last is', last) - print('found circular dependency in flavour', args.flavour, ':', ' -> '.join(path)) - exit(1) - - print('no circular dependency found for flavour', args.flavour, ' in modules:', - ' '.join(args.module)) - exit(0) - - def cmd_getval(self, args_): - parser = argparse.ArgumentParser(description='Get value from project config') - parser.add_argument('--project', default = self.top_name, help = 'Project name') - parser.add_argument('section', default = '', help = 'Config section') - parser.add_argument('key', default = '', help = 'Config key') - args = parser.parse_args(args_) - print(self.get_value(args.project, args.section, args.key)) - - # -------------------------------------------------------------------- here we go - def run(self): - - skip = 0 - for a in sys.argv[1::]: - self.global_args.append(a) - if a in [ '-p', '--prefix', '-t', '--topdir', '-O', '--os' ]: - skip = 1 - continue - if skip > 0: - skip = skip -1 - continue - if a[0] != '-': - break - - # -- defaults - self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" - - parser = argparse.ArgumentParser(description='Project metadata evaluation') - parser.add_argument('-d', '--debug', action='store_true', - default=False, help='Output debug information to stderr') - parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path') - parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ], help='Projects Path Prefix') - parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system') - parser.add_argument('cmd', default='', help='Command, one of: ' + self.commands()) - parser.add_argument('arg', nargs='*', help='Command arguments') - args = parser.parse_args(self.global_args) - - self.opt_debug = args.debug - if len(args.os): - self.opt_os = args.os[0] - - self.debug("----------------------------------------- running ", ' '.join(sys.argv)) - - self.projs_root = args.prefix[0] - self.debug("projs_root = ", self.projs_root) - if len(args.topdir): - self.topdir = args.topdir[0] - - if self.topdir: - self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name']) - if not self.top_name: - self.top_name = re.sub('-[0-9.-]*$', '', basename(realpath(self.topdir))) - - cmd_name = 'cmd_' + args.cmd.replace('-', '_') - cmd = getattr(self, cmd_name) - cmd(sys.argv[(len(self.global_args) + 1)::]) - -if __name__ == "__main__": - projects = Projects() - projects.run() +App().run() diff --git a/src/Makefile b/src/Makefile new file mode 100644 index 00000000..a9018b59 --- /dev/null +++ b/src/Makefile @@ -0,0 +1,4 @@ +TOPDIR = .. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/src/python/Makefile b/src/python/Makefile new file mode 100644 index 00000000..de51571d --- /dev/null +++ b/src/python/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-mods.mk diff --git a/src/python/jw/Makefile b/src/python/jw/Makefile new file mode 100644 index 00000000..62d684a7 --- /dev/null +++ b/src/python/jw/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/src/python/jw/build/App.py b/src/python/jw/build/App.py new file mode 100644 index 00000000..39d298b5 --- /dev/null +++ b/src/python/jw/build/App.py @@ -0,0 +1,1012 @@ +#!/usr/bin/python3 -u +# -*- coding: utf-8 -*- +# +# This source code file is a merge of various build tools and a horrible mess. +# + +from __future__ import print_function +import os +import sys +import argparse +import pwd +import time +from os.path import isfile +from os.path import isdir +from os.path import expanduser +from os.path import basename +from os.path import realpath +import subprocess +import re +import platform +import datetime + +# meaning of pkg.requires.xxx variables +# build: needs to be built and installed before this can be built +# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this +# run: needs to be installed before this-run can be installed, i.e. before this and other packages can run with this + +# --------------------------------------------------------------------- Python 2 / 3 compatibility stuff +try: + basestring +except NameError: + basestring = str + +# --------------------------------------------------------------------- helpers + +class ResultCache(object): + + def __init__(self): + self.__cache = {} + + def run(self, func, args): + d = self.__cache + depth = 0 + keys = [ func.__name__ ] + args + l = len(keys) + for k in keys: + if k is None: + k = 'None' + else: + k = str(k) + depth += 1 + #self.projects.debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) + if k in d: + if l == depth: + return d[k] + d = d[k] + continue + if l == depth: + r = func(*args) + d[k] = r + return r + d = d[k] = {} + #d = d[k] + raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) + +# ----------------------------------------------------------------- class App + +class App(object): + + def __init__(self): + + self.global_args = [] + self.opt_os = None + self.top_name = None + self.glob_os_cascade = None + + self.dep_cache = {} + self.my_dir = os.path.dirname(os.path.realpath(__file__)) + self.opt_debug = False + self.res_cache = ResultCache() + self.topdir = None + self.projs_root = expanduser("~") + '/local/src/jw.dev/proj' + + def debug(self, *objs): + if self.opt_debug: + print("DEBUG: ", *objs, file = sys.stderr) + + def warn(self, *objs): + print("WARNING: ", *objs, file = sys.stderr) + + def err(self, *objs): + print("ERR: ", *objs, file = sys.stderr) + + def find_proj_path_unused(name): + name = name.replace("dspider-", "") + search_path = [".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ] + for sub in search_path: + path = self.projs_root + "/" + sub + "/" + name + if os.path.exists(path): + return os.path.abspath(path) + raise Exception("module " + name + " not found below " + self.projs_root) + + def find_proj_path_cached_unused(name): + return self.res_cache.run(find_proj_path_unused, [ name ]) + + def proj_dir(self, name): + if name == self.top_name: + return self.topdir + for d in [ self.projs_root, '/opt' ]: + r = d + '/' + name + if os.path.exists(r): + return r + if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'): + # The package exists but does not have a dedicated project directory + return None + raise Exception('No project path found for module "{}"'.format(name)) + + def re_section(self, name): + return re.compile('[' + name + ']' + '.*?' + '(?=[)', + re.DOTALL) + + def remove_duplicates(self, seq): + seen = set() + seen_add = seen.add + return [x for x in seq if not (x in seen or seen_add(x))] + + def get_os(self, args = ""): + for d in [ self.projs_root + '/jw-build/scripts', '/opt/jw-build/bin' ]: + script = d + '/get-os.sh' + if isfile(script): + cmd = '/bin/bash ' + script + if args: + cmd = cmd + ' ' + args + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + (out, rr) = p.communicate() + if rr: + self.err("failed to run ", cmd) + continue + out = re.sub('\n', '', out.decode('utf-8')) + return out + return "linux" + + # TODO: add support for customizing this in project.conf + def htdocs_dir(self, name): + pd = self.proj_dir(name) + if pd is None: + return None + for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]: + if isdir(r): + return r + return None + + def os_cascade(self): + if self.glob_os_cascade is not None: + return self.glob_os_cascade.copy() + r = [ 'os', platform.system().lower() ] + os = self.opt_os if self.opt_os is not None else self.res_cache.run(self.get_os, []) + name = re.sub('-.*', '', os) + series = os + while True: + n = re.sub(r'\.[0-9]+$', '', series) + if n == series: + break + r.append(n) + series = n + if not name in r: + r.append(name) + if not os in r: + r.append(os) + # e.g. os, linux, suse, suse-tumbleweed + #return [ 'os', platform.system().lower(), name, os ] + self.glob_os_cascade = r + return r + + def strip_module_from_spec(self, mod): + return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip()) + + def get_section(self, path, section): + r = '' + file = open(path) + pat = '[' + section + ']' + in_section = False + for line in file: + if (line.rstrip() == pat): + in_section = True + continue + if in_section: + if len(line) and line[0] == '[': + break + r = r + line + file.close() + return r.rstrip() + + def read_value(self, path, section, key): + + def scan_section(f, key): + if key is None: + r = '' + for line in f: + if len(line) and line[0] == '[': + break + r += line + return r if len(r) else None + lines = [] + cont_line = '' + for line in f: + if len(line) and line[0] == '[': + break + cont_line += line.rstrip() + if len(cont_line) and cont_line[-1] == '\\': + cont_line = cont_line[0:-1] + continue + lines.append(cont_line) + cont_line = '' + for line in lines: + #self.debug(" looking for >%s< in line=>%s<" % (key, line)) + rr = re.findall('^ *' + key + ' *= *(.*)', line) + if len(rr) > 0: + return rr[0] + return None + + def scan_section_debug(f, key): + rr = scan_section(f, key) + #self.debug(" returning", rr) + return rr + + try: + #self.debug("looking for {}::[{}].{}".format(path, section, key)) + with open(path, 'r') as f: + if not len(section): + rr = scan_section(f, key) + pat = '[' + section + ']' + for line in f: + if line.rstrip() == pat: + return scan_section(f, key) + return None + except: + self.debug(path, "not found") + # TODO: handle this special case cleaner somewhere up the stack + if section == 'build' and key == 'libname': + return 'none' + return None + + def get_value(self, name, section, key): + self.debug("getting value [%s].%s for project %s (%s)" %(section, key, name, self.top_name)) + if self.top_name and name == self.top_name: + proj_root = self.topdir + else: + proj_root = self.projs_root + '/' + name + self.debug("proj_root = " + proj_root) + + if section == 'version': + proj_version_dirs = [ proj_root ] + if proj_root != self.topdir: + proj_version_dirs.append('/usr/share/doc/packages/' + name) + for d in proj_version_dirs: + version_path = d + '/VERSION' + try: + with open(version_path) as fd: + r = fd.read().replace('\n', '').replace('-dev', '') + fd.close() + return r + except EnvironmentError: + self.debug("ignoring unreadable file " + version_path) + continue + raise Exception("No version file found for project \"" + name + "\"") + + path = proj_root + '/make/project.conf' + #print('path = ', path, 'self.top_name = ', self.top_name, 'name = ', name) + return self.res_cache.run(self.read_value, [path, section, key]) + + def collect_values(self, names, section, key): + r = "" + for n in names: + val = self.get_value(n, section, key) + if val: + r = r + " " + val + return self.remove_duplicates([x.strip() for x in r.split(",")]) + + # scope 0: no children + # scope 1: children + # scope 2: recursive + + def add_modules_from_project_txt_cached(self, buf, visited, spec, section, key, add_self, scope, + names_only): + return self.res_cache.run(self.add_modules_from_project_txt, [buf, visited, spec, section, key, + add_self, scope, names_only]) + + def add_modules_from_project_txt(self, buf, visited, spec, section, key, add_self, scope, + names_only): + name = self.strip_module_from_spec(spec) + if names_only: + spec = name + if spec in buf: + return + if spec in visited: + if add_self: + buf.append(spec) + return + visited.add(spec) + deps = self.get_value(name, section, key) + self.debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited) + if deps and scope > 0: + if scope == 1: + subscope = 0 + else: + subscope = 2 + deps = deps.split(',') + for dep in deps: + dep = dep.strip() + if not(len(dep)): + continue + self.add_modules_from_project_txt_cached(buf, visited, dep, + section, key, add_self=True, scope=subscope, + names_only=names_only) + if add_self: + buf.append(spec) + + def get_modules_from_project_txt(self, names, sections, keys, add_self, scope, + names_only = True): + if isinstance(keys, basestring): + keys = [ keys ] + #r = set() + r = [] + for section in sections: + for key in keys: + visited = set() + for name in names: + rr = [] + self.add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope, + names_only) + # TODO: this looks like a performance hogger + for m in rr: + if not m in r: + r.append(m) + return r + + def pkg_relations(self, rel_type, args_): + parser = argparse.ArgumentParser(description='pkg-' + rel_type) + # TODO: implement Vendor evaluation + + parser.add_argument('-S', '--subsections', nargs='?', default=None, help='Subsections to consider, comma-separated') + parser.add_argument('-d', '--delimiter', nargs='?', default=', ', help='Output words delimiter') + parser.add_argument('flavour', help='Flavour') + parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('-p', '--no-subpackages', action='store_true', + default=False, help='Cut -run and -devel from package names') + parser.add_argument('--no-version', action='store_true', + default=False, help='Don\'t report version information') + parser.add_argument('--dont-strip-revision', action='store_true', + default=False, help='Always treat VERSION macro as VERSION-REVISION') + parser.add_argument('--recursive', action='store_true', + default=False, help='Find dependencies recursively') + parser.add_argument('--dont-expand-version-macros', action='store_true', + default=False, help='Don\'t expand VERSION and REVISION macros') + parser.add_argument('--ignore', nargs='?', default='', help='Packages that ' + 'should be ignored together with their dependencies') + args = parser.parse_args(args_) + version_pattern=re.compile("[0-9-.]*") + if args.subsections is None: + subsecs = self.os_cascade() + subsecs.append('jw') + else: + subsecs = args.subsections.split(',') + self.debug('flavour = ', args.flavour, ', subsecs = ', ' '.join(subsecs)) + ignore = args.ignore.split(',') + self.debug("ignore = ", ignore) + + r = [] + flavours = args.flavour.split(',') + for flavour in flavours: + for s in subsecs: + section = 'pkg.' + rel_type + '.' + s + visited = set() + modules = args.module.copy() + while len(modules): + m = modules.pop(0) + if m in visited or m in ignore: + continue + visited.add(m) + value = self.get_value(m, section, flavour) + if not value: + continue + deps = value.split(',') + for spec in deps: + dep = re.split('([=><]+)', spec) + if args.no_version: + dep = dep[:1] + dep = list(map(str.strip, dep)) + dep_name = re.sub('-dev$|-devel$|-run$', '', dep[0]) + if dep_name in ignore or dep[0] in ignore: + continue + if args.no_subpackages: + dep[0] = dep_name + for i, item in enumerate(dep): + dep[i] = item.strip() + if s == 'jw': + if args.recursive and not dep_name in visited and not dep_name in modules: + modules.append(dep_name) + if len(dep) == 3: + if args.dont_expand_version_macros and dep_name in args.module: + version = dep[2] + else: + version = self.get_value(dep_name, 'version', '') + if dep[2] == 'VERSION': + if args.dont_strip_revision: + dep[2] = version + else: + dep[2] = version.split('-')[0] + elif dep[2] == 'VERSION-REVISION': + dep[2] = version + elif version_pattern.match(dep[2]): + # dep[2] = dep[2] + pass + else: + raise Exception("Unknown version specifier in " + spec) + cleaned_dep = ' '.join(dep) + if not cleaned_dep in r: + self.debug("appending", cleaned_dep) + r.append(cleaned_dep) + return args.delimiter.join(r) + + def print_pkg_relations(self, rel_type, args_): + print(self.pkg_relations(rel_type, args_)) + + def get_libname(self, names): + vals = self.get_modules_from_project_txt(names, ['build'], 'libname', + scope = 1, add_self=False, names_only=True) + if not vals: + return ' '.join(names) + if 'none' in vals: + vals.remove('none') + return ' '.join(reversed(vals)) + + def is_excluded_from_build(self, module): + self.debug("checking if module " + module + " is excluded from build") + exclude = self.get_modules_from_project_txt([ module ], ['build'], 'exclude', + scope = 1, add_self=False, names_only=True) + cascade = self.os_cascade() + [ 'all' ] + for p1 in exclude: + for p2 in cascade: + if p1 == p2: + return p1 + return None + + # -L needs to contain more paths than libs linked with -l would require + def get_ldpathflags(self, names, exclude = []): + deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build', + scope = 2, add_self=True, names_only=True) + r = '' + for m in deps: + if m in exclude: + continue + libname = self.get_libname([m]) + if len(libname): + r = r + ' -L' + self.proj_dir(m) + '/lib' + print(r[1:]) + + def get_ldflags(self, names, exclude = [], add_self_ = False): + #print(names) + deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build', + scope = 1, add_self=add_self_, names_only=True) + self.debug("deps = " + ' '.join(deps)) + #print(deps) + r = '' + for m in reversed(deps): + if m in exclude: + continue + libname = self.get_libname([m]) + if len(libname): + #r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname + r = r + ' -l' + libname + if len(r): + ldpathflags = self.get_ldpathflags(names, exclude) + if ldpathflags: + r = ldpathflags + ' ' + r + return r[1::] + return '' + + def commands(self): + f = open(sys.argv[0]) + cmds = [] + for line in f: + self.debug("checking line ", line) + rr = re.findall('^ def *cmd_([a-z0-9_]+).*', line) + if len(rr): + cmds.append(rr[0].replace('_', '-')) + f.close() + return ' '.join(cmds) + + # ----------------------------------------------------------------- commands + + def cmd_commands(self, args_): + print(self.commands()) + + def cmd_modules(self, args_): + proj_root = self.projs_root + self.debug("proj_root = " + proj_root) + path = pathlib.Path(self.projs_root) + + parser = argparse.ArgumentParser(description='Query existing janware packages') + + parser.add_argument('-F', '--filter', nargs='?', default=None, help='Key-value pairs, seperated by commas, to be searched for in project.conf') + args = parser.parse_args(args_) + modules = [p.parents[1].name for p in path.glob('*/make/project.conf')] + self.debug("modules = ", modules) + out = [] + filters = None if args.filter is None else [re.split("=", f) for f in re.split(",", args.filter)] + for m in modules: + if filters: + for f in filters: + path = f[0].rsplit('.') + if len(path) > 1: + sec = path[0] + key = path[1] + else: + sec = None + key = path[0] + val = self.get_value(m, sec, key) + self.debug('Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val)) + if val and val == f[1]: + out.append(m) + break + else: + out.append(m) + print(' '.join(out)) + + def cmd_build(self, args_): + + def read_deps(cur, prereq_type): + # dep cache doesn't make a difference at all + if prereq_type in self.dep_cache: + if cur in self.dep_cache[prereq_type]: + return self.dep_cache[prereq_type][cur] + else: + self.dep_cache[prereq_type] = {} + + r = self.get_modules_from_project_txt([ cur ], ['pkg.requires.jw'], + prereq_type, scope = 2, add_self=False, names_only=True) + self.debug('prerequisites = ' + ' '.join(r)) + if cur in r: + r.remove(cur) + self.debug('inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(r)) + self.dep_cache[prereq_type][cur] = r + return r + + def read_deps_cached(cur, prereq_type): + return self.res_cache.run(read_deps, [ cur, prereq_type ]) + + def add_dep_tree(cur, prereq_types, tree, all_deps): + self.debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) + if cur in all_deps: + self.debug('already handled module ' + cur) + return 0 + deps = set() + all_deps.add(cur) + for t in prereq_types: + self.debug("checking prereqisites of type " + t) + deps.update(read_deps_cached(cur, t)) + for d in deps: + add_dep_tree(d, prereq_types, tree, all_deps) + tree[cur] = deps + return len(deps) + + def calculate_order(order, modules, prereq_types): + all_deps = set() + dep_tree = {} + for m in modules: + self.debug("--- adding dependency tree of module " + m) + add_dep_tree(m, prereq_types, dep_tree, all_deps) + while len(all_deps): + # Find any leaf + for d in all_deps: + if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself + break # found + else: # no Leaf found + print(all_deps) + raise Exception("fatal: the dependencies between these modules are unresolvable") + order.append(d) # do it + # bookkeep it + all_deps.remove(d) + for k in dep_tree.keys(): + if d in dep_tree[k]: + dep_tree[k].remove(d) + return 1 + + def run_make(module, target, cur_project, num_projects): + #make_cmd = "make " + target + " 2>&1" + make_cmd = [ "make", target ] + path = self.proj_dir(module) + delim_len = 120 + delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path) + delim = delim + '-' * (delim_len - len(delim)) + + print(',' + delim + ' >') + + patt = self.is_excluded_from_build(module) + if patt is not None: + print('| Configured to skip build on platform >' + patt + '<') + print('`' + delim + ' <') + return + + os.chdir(path) + p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True) + for line in iter(p.stdout.readline, b''): + line = line.decode(sys.stdout.encoding) + sys.stdout.write('| ' + line) # avoid extra newlines from print() + sys.stdout.flush() + p.wait() + print('`' + delim + ' <') + if p.returncode: + print(' '.join(make_cmd) + ' failed') + raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.projs_root) + + def run_make_on_modules(modules, order, target): + cur_project = 0 + num_projects = len(order) + if target in ["clean", "distclean"]: + for m in reversed(order): + cur_project += 1 + run_make(m, target, cur_project, num_projects) + if m in modules: + modules.remove(m) + if not len(modules): + print("all modules cleaned") + return + else: + for m in order: + cur_project += 1 + run_make(m, target, cur_project, num_projects) + + def run(args_): + + # -- parse command line + parser = argparse.ArgumentParser(description='janware software project build tool') + parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') + parser.add_argument('-n', '--dry-run', action='store_true', + default=False, help='Don\'t build anything, just print what would be done.') + parser.add_argument('-O', '--build-order', action='store_true', + default=False, help='Don\'t build anything, just print the build order.') + parser.add_argument('-I', '--ignore-deps', action='store_true', + default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') + parser.add_argument('target', default='all', help='Build target') + parser.add_argument('modules', nargs='+', default='', help='Modules to be built') + + args = parser.parse_args(args_) + + self.debug("----------------------------------------- running ", ' '.join(args_)) + + modules = args.modules + exclude = args.exclude.split() + target = args.target + + env_exclude = os.getenv('BUILD_EXCLUDE', '') + if len(env_exclude): + print("exluding modules from environment: " + env_exclude) + exclude += " " + env_exclude + + # -- build + order = [] + + glob_prereq_types = [ "build" ] + if re.match("pkg-.*", target) is not None: + glob_prereq_types = [ "build", "run", "release", "devel" ] + + if target != 'order' and not args.build_order: + print("using prerequisite types " + ' '.join(glob_prereq_types)) + print("calculating order for modules ... ") + + calculate_order(order, modules, glob_prereq_types) + if args.ignore_deps: + order = [m for m in order if m in args.modules] + order = [m for m in order if m not in exclude] + if target == 'order' or args.build_order: + print(' '.join(order)) + exit(0) + + cur_project = 0 + print("Building target %s in %d projects:" % (target, len(order))) + for m in order: + cur_project += 1 + print(" %3d %s" % (cur_project, m)) + + if args.dry_run: + exit(0) + + run_make_on_modules(modules, order, target) + + print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + run(args_) + + def cmd_test(self, args_): + parser = argparse.ArgumentParser(description='Test') + parser.add_argument('blah', default='', help='The blah argument') + args = parser.parse_args(args_) + print("blah = " + args.blah) + + def cmd_os_cascade(self, args_): + print(' '.join(self.os_cascade())) + + def cmd_ldlibpath(self, args_): + parser = argparse.ArgumentParser(description='ldlibpath') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], + scope = 2, add_self=True, names_only=True) + r = '' + for m in deps: + pd = self.proj_dir(m) + if pd is None: + continue + r = r + ':' + pd + '/lib' + print(r[1:]) + + def cmd_pythonpath(self, args_): + parser = argparse.ArgumentParser(description='pythonpath') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ], + scope = 2, add_self=True, names_only=True) + r = '' + for m in deps: + pd = self.proj_dir(m) + if pd is None: + continue + for subdir in [ 'src/python', 'tools/python' ]: + cand = pd + "/" + subdir + if isdir(cand): + r = r + ':' + cand + print(r[1:]) + + def cmd_exepath(self, args_): + parser = argparse.ArgumentParser(description='exepath') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], + scope = 2, add_self=True, names_only=True) + self.debug('deps = ', deps) + r = '' + for m in deps: + pd = self.proj_dir(m) + if pd is None: + continue + r = r + ':' + pd + '/bin' + print(r[1:]) + + def cmd_libname(self, args_): + parser = argparse.ArgumentParser(description='libname') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + print(self.get_libname(args.module)) + + def cmd_ldflags(self, args_): + parser = argparse.ArgumentParser(description='ldflags') + parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[]) + parser.add_argument('-s', '--add-self', action='store_true', + default=False, help='Include libflags of specified modules, too, not only their dependencies') + args = parser.parse_args(args_) + print(self.get_ldflags(args.module, args.exclude, args.add_self)) + + def cmd_cflags(self, args_): + parser = argparse.ArgumentParser(description='cflags') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build', + scope = 2, add_self=True, names_only=True) + r = '' + for m in reversed(deps): + try: + pd = self.proj_dir(m) + if pd is None: + continue + r = r + ' -I' + pd + '/include' + except Exception as e: + self.warn(f'No include path for module "{m}", ignoring: {e}') + print(r[1:]) + + def cmd_path(self, args_): + parser = argparse.ArgumentParser(description='path') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'run', + scope = 2, add_self=True, names_only=True) + r = '' + for m in deps: + pd = self.proj_dir(m) + if pd is None: + continue + r = r + ':' + pd + '/bin' + print(r[1:]) + + # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations + def cmd_prereq(self, args_): + parser = argparse.ArgumentParser(description='path') + parser.add_argument('flavour', help='Flavour') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + deps = self.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], + args.flavour, scope = 2, add_self=False, names_only=True) + print(' '.join(deps)) + + # TODO: seems at least partly redundant to cmd_pkg_requires / print_pkg_relations + def cmd_required_os_pkg(self, args_): + parser = argparse.ArgumentParser(description='required-os-pkg') + parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('--flavours', help='Dependency flavours', default='build') + parser.add_argument('--skip-excluded', action='store_true', default=False, + help='Output empty prerequisite list if module is excluded') + args = parser.parse_args(args_) + modules = args.module + flavours = args.flavours.split() + if 'build' in flavours and not 'run' in flavours: + # TODO: This adds too much. Only the run dependencies of the build dependencies would be needed. + flavours.append('run') + self.debug("flavours = " + args.flavours) + deps = self.get_modules_from_project_txt(modules, ['pkg.requires.jw'], flavours, + scope = 2, add_self=True, names_only=True) + if args.skip_excluded: + for d in deps: + if self.is_excluded_from_build(d) is not None: + deps.remove(d) + subsecs = self.os_cascade() + self.debug("subsecs = ", subsecs) + requires = [] + for s in subsecs: + for f in flavours: + vals = self.collect_values(deps, 'pkg.requires.' + s, f) + if vals: + requires = requires + vals + # TODO: add all not in build tree as -devel + r = '' + for m in requires: + r = r + ' ' + m + print(r[1:]) + + def cmd_pkg_requires(self, args_): + return self.print_pkg_relations("requires", args_) + + def cmd_pkg_conflicts(self, args_): + return self.print_pkg_relations("conflicts", args_) + + def cmd_pkg_provides(self, args_): + return self.print_pkg_relations("provides", args_) + + def cmd_proj_dir(self, args_): + parser = argparse.ArgumentParser(description='proj-dir') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + r = [] + for m in args.module: + try: + pd = self.proj_dir(m) + if pd is None: + continue + r.append(pd) + except Exception as e: + self.warn(f'No project directory for module "{m}: {e}') + continue + print(' '.join(r)) + + def cmd_htdocs_dir(self, args_): + parser = argparse.ArgumentParser(description='htdocs-dir') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + r = [] + for m in args.module: + r.append(self.htdocs_dir(m)) + print(' '.join(r)) + + def cmd_summary(self, args_): + parser = argparse.ArgumentParser(description='summary') + parser.add_argument('module', nargs='*', help='Modules') + args = parser.parse_args(args_) + r = [] + for m in args.module: + summary = self.get_value(m, "summary", None) + if summary is not None: + r.append(summary) + print(' '.join(r)) + + def contains(self, small, big): + for i in xrange(len(big)-len(small)+1): + for j in xrange(len(small)): + if big[i+j] != small[j]: + break + else: + return i, i+len(small) + return False + + def read_dep_graph(self, modules, section, graph): + for m in modules: + if m in graph: + continue + deps = self.get_modules_from_project_txt([ m ], ['pkg.requires.jw'], section, + scope = 1, add_self=False, names_only=True) + if not deps is None: + graph[m] = deps + for d in deps: + self.read_dep_graph([ d ], section, graph) + + def flip_graph(self, graph): + r = {} + for m, deps in graph.items(): + for d in deps: + if not d in r: + r[d] = set() + r[d].add(m) + return r + + def check_circular_deps(self, module, section, graph, unvisited, temp, path): + if module in temp: + self.debug('found circular dependency at module', module) + return module + if not module in unvisited: + return None + temp.add(module) + if module in graph: + for m in graph[module]: + last = self.check_circular_deps(m, section, graph, unvisited, temp, path) + if last is not None: + path.insert(0, m) + return last + unvisited.remove(module) + temp.remove(module) + + def cmd_check(self, args_): + parser = argparse.ArgumentParser(description='check') + parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('-f', '--flavour', nargs='?', default = 'build') + args = parser.parse_args(args_) + + graph = {} + path = [] + self.read_dep_graph(args.module, args.flavour, graph) + unvisited = graph.keys() + temp = set() + while len(unvisited) != 0: + m = unvisited[0] + self.debug('checking circular dependency of', m) + last = self.check_circular_deps(m, args.flavour, self.flip_graph(graph), unvisited, temp, path) + if last is not None: + self.debug('found circular dependency below', m, ', last is', last) + print('found circular dependency in flavour', args.flavour, ':', ' -> '.join(path)) + exit(1) + + print('no circular dependency found for flavour', args.flavour, ' in modules:', + ' '.join(args.module)) + exit(0) + + def cmd_getval(self, args_): + parser = argparse.ArgumentParser(description='Get value from project config') + parser.add_argument('--project', default = self.top_name, help = 'Project name') + parser.add_argument('section', default = '', help = 'Config section') + parser.add_argument('key', default = '', help = 'Config key') + args = parser.parse_args(args_) + print(self.get_value(args.project, args.section, args.key)) + + # -------------------------------------------------------------------- here we go + def run(self): + + skip = 0 + for a in sys.argv[1::]: + self.global_args.append(a) + if a in [ '-p', '--prefix', '-t', '--topdir', '-O', '--os' ]: + skip = 1 + continue + if skip > 0: + skip = skip -1 + continue + if a[0] != '-': + break + + # -- defaults + self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" + + parser = argparse.ArgumentParser(description='Project metadata evaluation') + parser.add_argument('-d', '--debug', action='store_true', + default=False, help='Output debug information to stderr') + parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path') + parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ], help='App Path Prefix') + parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system') + parser.add_argument('cmd', default='', help='Command, one of: ' + self.commands()) + parser.add_argument('arg', nargs='*', help='Command arguments') + args = parser.parse_args(self.global_args) + + self.opt_debug = args.debug + if len(args.os): + self.opt_os = args.os[0] + + self.debug("----------------------------------------- running ", ' '.join(sys.argv)) + + self.projs_root = args.prefix[0] + self.debug("projs_root = ", self.projs_root) + if len(args.topdir): + self.topdir = args.topdir[0] + + if self.topdir: + self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name']) + if not self.top_name: + self.top_name = re.sub('-[0-9.-]*$', '', basename(realpath(self.topdir))) + + cmd_name = 'cmd_' + args.cmd.replace('-', '_') + cmd = getattr(self, cmd_name) + cmd(sys.argv[(len(self.global_args) + 1)::]) + +if __name__ == "__main__": + projects = App() + projects.run() diff --git a/src/python/jw/build/Makefile b/src/python/jw/build/Makefile new file mode 100644 index 00000000..59b3ac1f --- /dev/null +++ b/src/python/jw/build/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-mod.mk diff --git a/src/python/jw/build/__init__.py b/src/python/jw/build/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python/jw/build/cmds/__init__.py b/src/python/jw/build/cmds/__init__.py new file mode 100644 index 00000000..e69de29b