mirror of
ssh://git.janware.com/srv/git/janware/proj/jw-pkg
synced 2026-02-01 09:18:40 +01:00
Everywhere: Rename package "jw-build" to "jw-pkg"
jw-build doesn't stop at building software, packaging it afterwards is also a core feature, so this commit gives the package a better name. The commit replaces strings s/jw-build/jw-pkg/ in text files and file names. Fallout to the functionality is fixed, variable names are left as they are, though. To be adjusted by later commits. Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
parent
ca95a9d6b2
commit
9217d38964
50 changed files with 36 additions and 36 deletions
|
|
@ -1,466 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# This source code file is a merge of various build tools and a horrible mess.
|
||||
#
|
||||
|
||||
import os, sys, argparse, pwd, re
|
||||
|
||||
# meaning of pkg.requires.xxx variables
|
||||
# build: needs to be built and installed before this can be built
|
||||
# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this
|
||||
# run: needs to be installed before this-run can be installed, i.e. before this and other packages can run with this
|
||||
|
||||
# --------------------------------------------------------------------- helpers
|
||||
|
||||
class ResultCache(object):
|
||||
|
||||
def __init__(self):
|
||||
self.__cache = {}
|
||||
|
||||
def run(self, func, args):
|
||||
d = self.__cache
|
||||
depth = 0
|
||||
keys = [ func.__name__ ] + args
|
||||
l = len(keys)
|
||||
for k in keys:
|
||||
if k is None:
|
||||
k = 'None'
|
||||
else:
|
||||
k = str(k)
|
||||
depth += 1
|
||||
#self.projects.debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
|
||||
if k in d:
|
||||
if l == depth:
|
||||
return d[k]
|
||||
d = d[k]
|
||||
continue
|
||||
if l == depth:
|
||||
r = func(*args)
|
||||
d[k] = r
|
||||
return r
|
||||
d = d[k] = {}
|
||||
#d = d[k]
|
||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||
|
||||
# ----------------------------------------------------------------- class App
|
||||
|
||||
class App(object):
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.global_args = []
|
||||
self.opt_os = None
|
||||
self.top_name = None
|
||||
self.glob_os_cascade = None
|
||||
|
||||
self.dep_cache = {}
|
||||
self.my_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
self.opt_debug = False
|
||||
self.res_cache = ResultCache()
|
||||
self.topdir = None
|
||||
self.projs_root = os.path.expanduser("~") + '/local/src/jw.dev/proj'
|
||||
|
||||
def debug(self, *objs):
|
||||
if self.opt_debug:
|
||||
print("DEBUG: ", *objs, file = sys.stderr)
|
||||
|
||||
def warn(self, *objs):
|
||||
print("WARNING: ", *objs, file = sys.stderr)
|
||||
|
||||
def err(self, *objs):
|
||||
print("ERR: ", *objs, file = sys.stderr)
|
||||
|
||||
def proj_dir(self, name):
|
||||
if name == self.top_name:
|
||||
return self.topdir
|
||||
for d in [ self.projs_root, '/opt' ]:
|
||||
r = d + '/' + name
|
||||
if os.path.exists(r):
|
||||
return r
|
||||
if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'):
|
||||
# The package exists but does not have a dedicated project directory
|
||||
return None
|
||||
raise Exception('No project path found for module "{}"'.format(name))
|
||||
|
||||
def re_section(self, name):
|
||||
return re.compile('[' + name + ']'
|
||||
'.*?'
|
||||
'(?=[)',
|
||||
re.DOTALL)
|
||||
|
||||
def remove_duplicates(self, seq):
|
||||
seen = set()
|
||||
seen_add = seen.add
|
||||
return [x for x in seq if not (x in seen or seen_add(x))]
|
||||
|
||||
def get_os(self, args = ""):
|
||||
import subprocess
|
||||
for d in [ self.projs_root + '/jw-build/scripts', '/opt/jw-build/bin' ]:
|
||||
script = d + '/get-os.sh'
|
||||
if os.path.isfile(script):
|
||||
cmd = '/bin/bash ' + script
|
||||
if args:
|
||||
cmd = cmd + ' ' + args
|
||||
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
|
||||
(out, rr) = p.communicate()
|
||||
if rr:
|
||||
self.err("failed to run ", cmd)
|
||||
continue
|
||||
out = re.sub('\n', '', out.decode('utf-8'))
|
||||
return out
|
||||
return "linux"
|
||||
|
||||
# TODO: add support for customizing this in project.conf
|
||||
def htdocs_dir(self, name):
|
||||
pd = self.proj_dir(name)
|
||||
if pd is None:
|
||||
return None
|
||||
for r in [ pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]:
|
||||
if os.path.isdir(r):
|
||||
return r
|
||||
return None
|
||||
|
||||
def os_cascade(self):
|
||||
import platform
|
||||
if self.glob_os_cascade is not None:
|
||||
return self.glob_os_cascade.copy()
|
||||
r = [ 'os', platform.system().lower() ]
|
||||
os = self.opt_os if self.opt_os is not None else self.res_cache.run(self.get_os, [])
|
||||
name = re.sub('-.*', '', os)
|
||||
series = os
|
||||
while True:
|
||||
n = re.sub(r'\.[0-9]+$', '', series)
|
||||
if n == series:
|
||||
break
|
||||
r.append(n)
|
||||
series = n
|
||||
if not name in r:
|
||||
r.append(name)
|
||||
if not os in r:
|
||||
r.append(os)
|
||||
# e.g. os, linux, suse, suse-tumbleweed
|
||||
#return [ 'os', platform.system().lower(), name, os ]
|
||||
self.glob_os_cascade = r
|
||||
return r
|
||||
|
||||
def strip_module_from_spec(self, mod):
|
||||
return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())
|
||||
|
||||
def get_section(self, path, section):
|
||||
r = ''
|
||||
file = open(path)
|
||||
pat = '[' + section + ']'
|
||||
in_section = False
|
||||
for line in file:
|
||||
if (line.rstrip() == pat):
|
||||
in_section = True
|
||||
continue
|
||||
if in_section:
|
||||
if len(line) and line[0] == '[':
|
||||
break
|
||||
r = r + line
|
||||
file.close()
|
||||
return r.rstrip()
|
||||
|
||||
def read_value(self, path, section, key):
|
||||
|
||||
def scan_section(f, key):
|
||||
if key is None:
|
||||
r = ''
|
||||
for line in f:
|
||||
if len(line) and line[0] == '[':
|
||||
break
|
||||
r += line
|
||||
return r if len(r) else None
|
||||
lines = []
|
||||
cont_line = ''
|
||||
for line in f:
|
||||
if len(line) and line[0] == '[':
|
||||
break
|
||||
cont_line += line.rstrip()
|
||||
if len(cont_line) and cont_line[-1] == '\\':
|
||||
cont_line = cont_line[0:-1]
|
||||
continue
|
||||
lines.append(cont_line)
|
||||
cont_line = ''
|
||||
for line in lines:
|
||||
#self.debug(" looking for >%s< in line=>%s<" % (key, line))
|
||||
rr = re.findall('^ *' + key + ' *= *(.*)', line)
|
||||
if len(rr) > 0:
|
||||
return rr[0]
|
||||
return None
|
||||
|
||||
def scan_section_debug(f, key):
|
||||
rr = scan_section(f, key)
|
||||
#self.debug(" returning", rr)
|
||||
return rr
|
||||
|
||||
try:
|
||||
#self.debug("looking for {}::[{}].{}".format(path, section, key))
|
||||
with open(path, 'r') as f:
|
||||
if not len(section):
|
||||
rr = scan_section(f, key)
|
||||
pat = '[' + section + ']'
|
||||
for line in f:
|
||||
if line.rstrip() == pat:
|
||||
return scan_section(f, key)
|
||||
return None
|
||||
except:
|
||||
self.debug(path, "not found")
|
||||
# TODO: handle this special case cleaner somewhere up the stack
|
||||
if section == 'build' and key == 'libname':
|
||||
return 'none'
|
||||
return None
|
||||
|
||||
def get_value(self, name, section, key):
|
||||
self.debug("getting value [%s].%s for project %s (%s)" %(section, key, name, self.top_name))
|
||||
if self.top_name and name == self.top_name:
|
||||
proj_root = self.topdir
|
||||
else:
|
||||
proj_root = self.projs_root + '/' + name
|
||||
self.debug("proj_root = " + proj_root)
|
||||
|
||||
if section == 'version':
|
||||
proj_version_dirs = [ proj_root ]
|
||||
if proj_root != self.topdir:
|
||||
proj_version_dirs.append('/usr/share/doc/packages/' + name)
|
||||
for d in proj_version_dirs:
|
||||
version_path = d + '/VERSION'
|
||||
try:
|
||||
with open(version_path) as fd:
|
||||
r = fd.read().replace('\n', '').replace('-dev', '')
|
||||
fd.close()
|
||||
return r
|
||||
except EnvironmentError:
|
||||
self.debug("ignoring unreadable file " + version_path)
|
||||
continue
|
||||
raise Exception("No version file found for project \"" + name + "\"")
|
||||
|
||||
path = proj_root + '/make/project.conf'
|
||||
#print('path = ', path, 'self.top_name = ', self.top_name, 'name = ', name)
|
||||
return self.res_cache.run(self.read_value, [path, section, key])
|
||||
|
||||
def collect_values(self, names, section, key):
|
||||
r = ""
|
||||
for n in names:
|
||||
val = self.get_value(n, section, key)
|
||||
if val:
|
||||
r = r + " " + val
|
||||
return self.remove_duplicates([x.strip() for x in r.split(",")])
|
||||
|
||||
# scope 0: no children
|
||||
# scope 1: children
|
||||
# scope 2: recursive
|
||||
|
||||
def add_modules_from_project_txt_cached(self, buf, visited, spec, section, key, add_self, scope,
|
||||
names_only):
|
||||
return self.res_cache.run(self.add_modules_from_project_txt, [buf, visited, spec, section, key,
|
||||
add_self, scope, names_only])
|
||||
|
||||
def add_modules_from_project_txt(self, buf, visited, spec, section, key, add_self, scope,
|
||||
names_only):
|
||||
name = self.strip_module_from_spec(spec)
|
||||
if names_only:
|
||||
spec = name
|
||||
if spec in buf:
|
||||
return
|
||||
if spec in visited:
|
||||
if add_self:
|
||||
buf.append(spec)
|
||||
return
|
||||
visited.add(spec)
|
||||
deps = self.get_value(name, section, key)
|
||||
self.debug("name = ", name, "section = ", section, "key = ", key, "deps = ", deps, "scope = ", scope, "visited = ", visited)
|
||||
if deps and scope > 0:
|
||||
if scope == 1:
|
||||
subscope = 0
|
||||
else:
|
||||
subscope = 2
|
||||
deps = deps.split(',')
|
||||
for dep in deps:
|
||||
dep = dep.strip()
|
||||
if not(len(dep)):
|
||||
continue
|
||||
self.add_modules_from_project_txt_cached(buf, visited, dep,
|
||||
section, key, add_self=True, scope=subscope,
|
||||
names_only=names_only)
|
||||
if add_self:
|
||||
buf.append(spec)
|
||||
|
||||
def get_modules_from_project_txt(self, names, sections, keys, add_self, scope,
|
||||
names_only = True):
|
||||
if isinstance(keys, str):
|
||||
keys = [ keys ]
|
||||
#r = set()
|
||||
r = []
|
||||
for section in sections:
|
||||
for key in keys:
|
||||
visited = set()
|
||||
for name in names:
|
||||
rr = []
|
||||
self.add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
|
||||
names_only)
|
||||
# TODO: this looks like a performance hogger
|
||||
for m in rr:
|
||||
if not m in r:
|
||||
r.append(m)
|
||||
return r
|
||||
|
||||
def get_libname(self, names):
|
||||
vals = self.get_modules_from_project_txt(names, ['build'], 'libname',
|
||||
scope = 1, add_self=False, names_only=True)
|
||||
if not vals:
|
||||
return ' '.join(names)
|
||||
if 'none' in vals:
|
||||
vals.remove('none')
|
||||
return ' '.join(reversed(vals))
|
||||
|
||||
def is_excluded_from_build(self, module):
|
||||
self.debug("checking if module " + module + " is excluded from build")
|
||||
exclude = self.get_modules_from_project_txt([ module ], ['build'], 'exclude',
|
||||
scope = 1, add_self=False, names_only=True)
|
||||
cascade = self.os_cascade() + [ 'all' ]
|
||||
for p1 in exclude:
|
||||
for p2 in cascade:
|
||||
if p1 == p2:
|
||||
return p1
|
||||
return None
|
||||
|
||||
# -L needs to contain more paths than libs linked with -l would require
|
||||
def get_ldpathflags(self, names, exclude = []):
|
||||
deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
|
||||
scope = 2, add_self=True, names_only=True)
|
||||
r = ''
|
||||
for m in deps:
|
||||
if m in exclude:
|
||||
continue
|
||||
libname = self.get_libname([m])
|
||||
if len(libname):
|
||||
r = r + ' -L' + self.proj_dir(m) + '/lib'
|
||||
print(r[1:])
|
||||
|
||||
def get_ldflags(self, names, exclude = [], add_self_ = False):
|
||||
#print(names)
|
||||
deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
|
||||
scope = 1, add_self=add_self_, names_only=True)
|
||||
self.debug("deps = " + ' '.join(deps))
|
||||
#print(deps)
|
||||
r = ''
|
||||
for m in reversed(deps):
|
||||
if m in exclude:
|
||||
continue
|
||||
libname = self.get_libname([m])
|
||||
if len(libname):
|
||||
#r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname
|
||||
r = r + ' -l' + libname
|
||||
if len(r):
|
||||
ldpathflags = self.get_ldpathflags(names, exclude)
|
||||
if ldpathflags:
|
||||
r = ldpathflags + ' ' + r
|
||||
return r[1::]
|
||||
return ''
|
||||
|
||||
def contains(self, small, big):
|
||||
for i in xrange(len(big)-len(small)+1):
|
||||
for j in xrange(len(small)):
|
||||
if big[i+j] != small[j]:
|
||||
break
|
||||
else:
|
||||
return i, i+len(small)
|
||||
return False
|
||||
|
||||
def read_dep_graph(self, modules, section, graph):
|
||||
for m in modules:
|
||||
if m in graph:
|
||||
continue
|
||||
deps = self.get_modules_from_project_txt([ m ], ['pkg.requires.jw'], section,
|
||||
scope = 1, add_self=False, names_only=True)
|
||||
if not deps is None:
|
||||
graph[m] = deps
|
||||
for d in deps:
|
||||
self.read_dep_graph([ d ], section, graph)
|
||||
|
||||
def flip_graph(self, graph):
|
||||
r = {}
|
||||
for m, deps in graph.items():
|
||||
for d in deps:
|
||||
if not d in r:
|
||||
r[d] = set()
|
||||
r[d].add(m)
|
||||
return r
|
||||
|
||||
def check_circular_deps(self, module, section, graph, unvisited, temp, path):
|
||||
if module in temp:
|
||||
self.debug('found circular dependency at module', module)
|
||||
return module
|
||||
if not module in unvisited:
|
||||
return None
|
||||
temp.add(module)
|
||||
if module in graph:
|
||||
for m in graph[module]:
|
||||
last = self.check_circular_deps(m, section, graph, unvisited, temp, path)
|
||||
if last is not None:
|
||||
path.insert(0, m)
|
||||
return last
|
||||
unvisited.remove(module)
|
||||
temp.remove(module)
|
||||
|
||||
# -------------------------------------------------------------------- here we go
|
||||
def run(self):
|
||||
|
||||
skip = 0
|
||||
for a in sys.argv[1::]:
|
||||
self.global_args.append(a)
|
||||
if a in [ '-p', '--prefix', '-t', '--topdir', '-O', '--os' ]:
|
||||
skip = 1
|
||||
continue
|
||||
if skip > 0:
|
||||
skip = skip -1
|
||||
continue
|
||||
if a[0] != '-':
|
||||
break
|
||||
|
||||
# -- defaults
|
||||
self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
|
||||
|
||||
parser = argparse.ArgumentParser(description='Project metadata evaluation')
|
||||
parser.add_argument('-d', '--debug', action='store_true',
|
||||
default=False, help='Output debug information to stderr')
|
||||
parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path')
|
||||
parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ], help='App Path Prefix')
|
||||
parser.add_argument('-O', '--os', nargs=1, default = [], help='Target operating system')
|
||||
parser.add_argument('cmd', default='', help=f'Command, run "{sys.argv[0]} commands" for a list of supported commands')
|
||||
parser.add_argument('arg', nargs='*', help='Command arguments')
|
||||
args = parser.parse_args(self.global_args)
|
||||
|
||||
self.opt_debug = args.debug
|
||||
if len(args.os):
|
||||
self.opt_os = args.os[0]
|
||||
|
||||
self.debug("----------------------------------------- running ", ' '.join(sys.argv))
|
||||
|
||||
self.projs_root = args.prefix[0]
|
||||
self.debug("projs_root = ", self.projs_root)
|
||||
if len(args.topdir):
|
||||
self.topdir = args.topdir[0]
|
||||
|
||||
if self.topdir:
|
||||
self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name'])
|
||||
if not self.top_name:
|
||||
self.top_name = re.sub('-[0-9.-]*$', '', os.path.basename(os.path.realpath(self.topdir)))
|
||||
|
||||
import importlib
|
||||
cmd_name = args.cmd.replace('-', '_')
|
||||
cc_cmd_name = 'Cmd' + ''.join(x.capitalize() for x in cmd_name.lower().split("_"))
|
||||
module = importlib.import_module('jw.build.cmds.' + cc_cmd_name)
|
||||
cmd = getattr(module, cc_cmd_name)()
|
||||
cmd.app = self
|
||||
subparser = argparse.ArgumentParser(description=cmd_name)
|
||||
cmd.add_arguments(subparser)
|
||||
args = subparser.parse_args(sys.argv[(len(self.global_args) + 1)::])
|
||||
try:
|
||||
return cmd.run(args)
|
||||
except Exception as e:
|
||||
self.err('Failed to run >{}<: {}'.format(' '.join(sys.argv), e))
|
||||
raise
|
||||
sys.exit(1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue