mirror of
ssh://git.janware.com/srv/git/janware/proj/jw-pkg
synced 2026-01-15 03:53:32 +01:00
build.py: Preparare build.py for merge into projects.py
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
parent
5b8c7fe738
commit
49ec96eebb
2 changed files with 132 additions and 89 deletions
|
|
@ -96,7 +96,7 @@ endif
|
|||
# ------------ external programs II
|
||||
|
||||
PGIT = CLONE_PROJECTS="$(PROJECTS)" /bin/bash $(MOD_SCRIPT_DIR)/pgit.sh
|
||||
BUILD_PY = python2 $(MOD_SCRIPT_DIR)/build.py -b $(shell pwd) $(BUILD_PY_EXTRA_ARGS)
|
||||
BUILD_PY = python2 $(MOD_SCRIPT_DIR)/build.py --prefix $(shell pwd) build $(BUILD_PY_EXTRA_ARGS)
|
||||
PROJECTS_PY = python2 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_ARGS)
|
||||
PURGE_SH = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/purge-stale-projects.sh /opt/jw-build/bin/purge-stale-projects.sh) purge-not-found)
|
||||
RELEASE_SH = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/build-release.sh /opt/jw-build/bin/build-release.sh) release-not-found)
|
||||
|
|
|
|||
219
scripts/build.py
219
scripts/build.py
|
|
@ -15,14 +15,15 @@ import argparse
|
|||
import time
|
||||
import datetime
|
||||
import re
|
||||
from os.path import expanduser
|
||||
|
||||
def debug(*objs):
|
||||
if args.debug:
|
||||
print("DEBUG: ", *objs, file=sys.stderr)
|
||||
|
||||
def cache_func(func, args):
|
||||
global cache
|
||||
d = cache
|
||||
def build_cache_func(func, args):
|
||||
global build_cache
|
||||
d = build_cache
|
||||
depth = 0
|
||||
keys = [ func.__name__ ] + args
|
||||
l = len(keys)
|
||||
|
|
@ -44,18 +45,19 @@ def cache_func(func, args):
|
|||
d = d[k]
|
||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||
|
||||
def find_proj_path(name):
|
||||
def build_find_proj_path(name):
|
||||
name=name.replace("dspider-", "")
|
||||
search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
|
||||
for sub in search_path:
|
||||
path=proj_base + "/" + sub + "/" + name
|
||||
path=projs_root + "/" + sub + "/" + name
|
||||
if os.path.exists(path):
|
||||
return os.path.abspath(path)
|
||||
raise Exception("module " + name + " not found below " + proj_base)
|
||||
raise Exception("module " + name + " not found below " + projs_root)
|
||||
|
||||
def find_proj_path_cached(name):
|
||||
return cache_func(find_proj_path, [ name ])
|
||||
def build_find_proj_path_cached(name):
|
||||
return build_cache_func(build_find_proj_path, [ name ])
|
||||
|
||||
def read_deps(cur, prereq_type):
|
||||
def build_read_deps(cur, prereq_type):
|
||||
# dep cache doesn't make a difference at all
|
||||
if prereq_type in dep_cache:
|
||||
if cur in dep_cache[prereq_type]:
|
||||
|
|
@ -82,10 +84,10 @@ def read_deps(cur, prereq_type):
|
|||
dep_cache[prereq_type][cur] = r
|
||||
return r
|
||||
|
||||
def read_deps_cached(cur, prereq_type):
|
||||
return cache_func(read_deps, [ cur, prereq_type ])
|
||||
def build_read_deps_cached(cur, prereq_type):
|
||||
return build_cache_func(build_read_deps, [ cur, prereq_type ])
|
||||
|
||||
def add_tree(cur, prereq_types):
|
||||
def build_add_tree(cur, prereq_types, all_deps):
|
||||
debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
|
||||
if cur in all_deps:
|
||||
debug('already handled module ' + cur)
|
||||
|
|
@ -95,16 +97,16 @@ def add_tree(cur, prereq_types):
|
|||
all_deps.add(cur)
|
||||
for t in prereq_types:
|
||||
debug("checking prereqisites of type " + t)
|
||||
deps.update(read_deps_cached(cur, t))
|
||||
deps.update(build_read_deps_cached(cur, t))
|
||||
for d in deps:
|
||||
add_tree(d, prereq_types)
|
||||
build_add_tree(d, prereq_types, all_deps)
|
||||
dep_tree[cur] = deps
|
||||
return len(deps)
|
||||
|
||||
def calculate_order(order, modules, prereq_types):
|
||||
def build_calculate_order(order, modules, prereq_types, all_deps):
|
||||
for m in modules:
|
||||
debug("--- adding dependency tree of module " + m)
|
||||
add_tree(m, prereq_types)
|
||||
build_add_tree(m, prereq_types, all_deps)
|
||||
while len(all_deps):
|
||||
for d in all_deps:
|
||||
if not len(dep_tree[d]):
|
||||
|
|
@ -119,14 +121,12 @@ def calculate_order(order, modules, prereq_types):
|
|||
dep_tree[k].remove(d)
|
||||
return 1
|
||||
|
||||
def run_make(module, target):
|
||||
global cur_project
|
||||
cur_project=cur_project+1
|
||||
def build_run_make(module, target, cur_project, num_projects):
|
||||
#make_cmd = "make " + target + " 2>&1"
|
||||
make_cmd = [ "make", target ]
|
||||
path = find_proj_path_cached(module)
|
||||
path = build_find_proj_path_cached(module)
|
||||
delim_len=120
|
||||
delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
|
||||
delim='---- %d/%d: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
|
||||
delim = delim + '-' * (delim_len - len(delim))
|
||||
|
||||
print(',' + delim + ' >')
|
||||
|
|
@ -139,12 +139,15 @@ def run_make(module, target):
|
|||
print('`' + delim + ' <')
|
||||
if p.returncode:
|
||||
print(make_cmd + ' failed')
|
||||
raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + proj_base)
|
||||
raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + projs_root)
|
||||
|
||||
def build(modules, order, target):
|
||||
def build_run_make_on_modules(modules, order, target):
|
||||
cur_project = 0
|
||||
num_projects = len(order)
|
||||
if target in ["clean", "distclean"]:
|
||||
for m in reversed(order):
|
||||
run_make(m, target)
|
||||
cur_project += 1
|
||||
build_run_make(m, target, cur_project, num_projects)
|
||||
if m in modules:
|
||||
modules.remove(m)
|
||||
if not len(modules):
|
||||
|
|
@ -152,78 +155,118 @@ def build(modules, order, target):
|
|||
return
|
||||
else:
|
||||
for m in order:
|
||||
run_make(m, target)
|
||||
cur_project += 1
|
||||
build_run_make(m, target, cur_project, num_projects)
|
||||
|
||||
# ------------ here we go
|
||||
def cmd_build(args_):
|
||||
all_deps = Set()
|
||||
visited = {}
|
||||
glob_order = []
|
||||
projs_root=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
|
||||
|
||||
all_deps = Set()
|
||||
cache = {}
|
||||
visited = {}
|
||||
dep_tree = {}
|
||||
glob_order = []
|
||||
proj_base=pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
|
||||
search_path=[".", "dspc/src", "dspc/src/dspcd-plugins", "dspc/src/io" ]
|
||||
dep_cache = {}
|
||||
my_dir=os.path.dirname(os.path.realpath(__file__))
|
||||
# -- parse command line
|
||||
parser = argparse.ArgumentParser(description='janware software project build tool')
|
||||
parser.add_argument('--base', '-b', nargs='?', default=projs_root, help='Project base directory')
|
||||
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
|
||||
parser.add_argument('--debug', '-d', action='store_true',
|
||||
default=False, help='Output debug information to stderr')
|
||||
parser.add_argument('--dry-run', '-n', action='store_true',
|
||||
default=False, help='Don\'t build anything, just print what would be done.')
|
||||
parser.add_argument('--ignore-deps', '-I', action='store_true',
|
||||
default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
|
||||
parser.add_argument('target', default='all', help='Build target')
|
||||
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
|
||||
|
||||
# -- parse command line
|
||||
parser = argparse.ArgumentParser(description='janware software project build tool')
|
||||
parser.add_argument('--base', '-b', nargs='?', default=proj_base, help='Project base directory')
|
||||
parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
|
||||
args=parser.parse_args(args_)
|
||||
|
||||
debug("----------------------------------------- running ", ' '.join(args_))
|
||||
|
||||
modules=args.modules
|
||||
exclude=args.exclude.split()
|
||||
target=args.target
|
||||
|
||||
env_exclude=os.getenv('BUILD_EXCLUDE', '')
|
||||
if len(env_exclude):
|
||||
print("exluding modules from environment: " + env_exclude)
|
||||
exclude += " " + env_exclude
|
||||
|
||||
# -- build
|
||||
if target != 'order':
|
||||
print("calculating order for modules ... ")
|
||||
order = []
|
||||
|
||||
glob_prereq_types = [ "build" ]
|
||||
if re.match("pkg-.*", target) is not None:
|
||||
glob_prereq_types = [ "build", "run", "release", "devel" ]
|
||||
print("using prerequisite types " + ' '.join(glob_prereq_types))
|
||||
|
||||
build_calculate_order(order, modules, glob_prereq_types, all_deps)
|
||||
if args.ignore_deps:
|
||||
order = [m for m in order if m in args.modules]
|
||||
order = [m for m in order if m not in exclude]
|
||||
if target == 'order':
|
||||
print(' '.join(order))
|
||||
exit(0)
|
||||
|
||||
cur_project = 0
|
||||
print("Building target %s in %d projects:" % (target, len(order)))
|
||||
for m in order:
|
||||
cur_project += 1
|
||||
print(" %3d %s" % (cur_project, m))
|
||||
|
||||
if args.dry_run:
|
||||
exit(0)
|
||||
|
||||
build_run_make_on_modules(modules, order, target)
|
||||
|
||||
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
|
||||
# -------------------------------------------------------------------- here we go
|
||||
|
||||
global_args = []
|
||||
#cache = ResultCache()
|
||||
|
||||
skip = 0
|
||||
for a in sys.argv[1::]:
|
||||
global_args.append(a)
|
||||
if a in [ '--prefix', '-p', '--topdir', '-t' ]:
|
||||
skip = 1
|
||||
continue
|
||||
if skip > 0:
|
||||
skip = skip -1
|
||||
continue
|
||||
if a[0] != '-':
|
||||
break
|
||||
|
||||
topdir = None
|
||||
top_name = None
|
||||
|
||||
parser = argparse.ArgumentParser(description='Project metadata evaluation')
|
||||
parser.add_argument('cmd', default='', help='Command')
|
||||
parser.add_argument('--debug', '-d', action='store_true',
|
||||
default=False, help='Output debug information to stderr')
|
||||
parser.add_argument('--dry-run', '-n', action='store_true',
|
||||
default=False, help='Don\'t build anything, just print what would be done.')
|
||||
parser.add_argument('--ignore-deps', '-I', action='store_true',
|
||||
default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
|
||||
parser.add_argument('target', default='all', help='Build target')
|
||||
parser.add_argument('modules', nargs='+', default='', help='Modules to be built')
|
||||
|
||||
args=parser.parse_args()
|
||||
parser.add_argument('--topdir', '-t', nargs='?', help='Project Path')
|
||||
parser.add_argument('--prefix', '-p', nargs='?', default = expanduser("~") +
|
||||
'/local/src/jw.dev/proj', help='Projects Path Prefix')
|
||||
parser.add_argument('arg', nargs='*', help='Command arguments')
|
||||
args=parser.parse_args(global_args)
|
||||
|
||||
debug("----------------------------------------- running ", ' '.join(sys.argv))
|
||||
|
||||
modules=args.modules
|
||||
exclude=args.exclude.split()
|
||||
proj_base=args.base
|
||||
target=args.target
|
||||
projs_root = args.prefix
|
||||
|
||||
projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
|
||||
if args.topdir:
|
||||
topdir = args.topdir
|
||||
top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
|
||||
if not top_name:
|
||||
top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))
|
||||
|
||||
env_exclude=os.getenv('BUILD_EXCLUDE', '')
|
||||
if len(env_exclude):
|
||||
print("exluding modules from environment: " + env_exclude)
|
||||
exclude += " " + env_exclude
|
||||
|
||||
# -- build
|
||||
if target != 'order':
|
||||
print("calculating order for modules ... ")
|
||||
order = []
|
||||
|
||||
glob_prereq_types = [ "build" ]
|
||||
if re.match("pkg-.*", target) is not None:
|
||||
glob_prereq_types = [ "build", "run", "release", "devel" ]
|
||||
print("using prerequisite types " + ' '.join(glob_prereq_types))
|
||||
|
||||
calculate_order(order, modules, glob_prereq_types)
|
||||
if args.ignore_deps:
|
||||
order = [m for m in order if m in args.modules]
|
||||
order = [m for m in order if m not in exclude]
|
||||
if target == 'order':
|
||||
print(' '.join(order))
|
||||
exit(0)
|
||||
|
||||
cur_project=0
|
||||
print("Building target %s in %d projects:" % (target, len(order)))
|
||||
for m in order:
|
||||
cur_project=cur_project+1
|
||||
print(" %3d %s" % (cur_project, m))
|
||||
cur_project=0
|
||||
|
||||
if args.dry_run:
|
||||
exit(0)
|
||||
|
||||
build(modules, order, target)
|
||||
|
||||
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
# ---- global variables
|
||||
build_cache = {}
|
||||
dep_cache = {}
|
||||
my_dir=os.path.dirname(os.path.realpath(__file__))
|
||||
projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
|
||||
dep_tree = {}
|
||||
|
||||
cmd = getattr(sys.modules[__name__], 'cmd_' + args.cmd.replace('-', '_'))
|
||||
cmd(sys.argv[(len(global_args) + 1)::])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue