janware/jw-pkg
1
mirror of ssh://git.janware.com/srv/git/janware/proj/jw-pkg synced 2026-01-17 12:50:47 +01:00
Code Activity

build.py and projects.py: Cache results queried from file system

Browse source 
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2017-08-31 18:57:13 +00:00
commit 378feb60ec
2 changed files with 78 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

39
scripts/build.py
View file

@ -20,6 +20,30 @@ def debug(*objs):
if args.debug:
print("DEBUG: ", *objs, file=sys.stderr)
def cache_func(func, args):
global cache
d = cache
depth = 0
keys = [ func.__name__ ] + args
l = len(keys)
for k in keys:
if k is None:
k = 'None'
depth += 1
#debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
if k in d:
if l == depth:
return d[k]
d = d[k]
continue
if l == depth:
r = func(*args)
d[k] = r
return r
d[k] = {}
d = d[k]
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
def find_proj_path(name):
name=name.replace("dspider-", "")
for sub in search_path:
@ -28,6 +52,9 @@ def find_proj_path(name):
return os.path.abspath(path)
raise Exception("module " + name + " not found below " + proj_base)
def find_proj_path_cached(name):
return cache_func(find_proj_path, [ name ])
def read_deps(cur, prereq_type):
# dep cache doesn't make a difference at all
if prereq_type in dep_cache:
@ -36,6 +63,7 @@ def read_deps(cur, prereq_type):
else:
dep_cache[prereq_type] = {}
cmd = projects_py + " prereq " + prereq_type + " " + cur
debug('running', cmd)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
p.wait()
if p.returncode:
@ -54,6 +82,9 @@ def read_deps(cur, prereq_type):
dep_cache[prereq_type][cur] = r
return r
def read_deps_cached(cur, prereq_type):
return cache_func(read_deps, [ cur, prereq_type ])
def add_tree(cur, prereq_types):
debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
if cur in all_deps:
@ -64,7 +95,7 @@ def add_tree(cur, prereq_types):
all_deps.add(cur)
for t in prereq_types:
debug("checking prereqisites of type " + t)
deps.update(read_deps(cur, t))
deps.update(read_deps_cached(cur, t))
for d in deps:
add_tree(d, prereq_types)
dep_tree[cur] = deps
@ -92,7 +123,7 @@ def run_make(module, target):
global cur_project
cur_project=cur_project+1
make_cmd = "make " + target + " 2>&1"
path = find_proj_path(module)
path = find_proj_path_cached(module)
delim_len=120
delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
delim = delim + '-' * (delim_len - len(delim))
@ -125,6 +156,7 @@ def build(modules, order, target):
# ------------ here we go
all_deps = Set()
cache = {}
visited = {}
dep_tree = {}
glob_order = []
@ -154,7 +186,8 @@ modules=args.modules
exclude=args.exclude.split()
proj_base=args.base
target=args.target
projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base
projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
env_exclude=os.getenv('BUILD_EXCLUDE', '')
if len(env_exclude):