From 378feb60ecc54d5528c4cbdd838e831ec46f07ed Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 31 Aug 2017 18:57:13 +0000 Subject: [PATCH] build.py and projects.py: Cache results queried from file system Signed-off-by: Jan Lindemann --- scripts/build.py | 39 +++++++++++++++++++-- scripts/projects.py | 82 +++++++++++++++++++++++---------------------- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index 8b049f0b..6c00d5a7 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -20,6 +20,30 @@ def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) +def cache_func(func, args): + global cache + d = cache + depth = 0 + keys = [ func.__name__ ] + args + l = len(keys) + for k in keys: + if k is None: + k = 'None' + depth += 1 + #debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) + if k in d: + if l == depth: + return d[k] + d = d[k] + continue + if l == depth: + r = func(*args) + d[k] = r + return r + d[k] = {} + d = d[k] + raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) + def find_proj_path(name): name=name.replace("dspider-", "") for sub in search_path: @@ -28,6 +52,9 @@ def find_proj_path(name): return os.path.abspath(path) raise Exception("module " + name + " not found below " + proj_base) +def find_proj_path_cached(name): + return cache_func(find_proj_path, [ name ]) + def read_deps(cur, prereq_type): # dep cache doesn't make a difference at all if prereq_type in dep_cache: @@ -36,6 +63,7 @@ def read_deps(cur, prereq_type): else: dep_cache[prereq_type] = {} cmd = projects_py + " prereq " + prereq_type + " " + cur + debug('running', cmd) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p.wait() if p.returncode: @@ -54,6 +82,9 @@ def read_deps(cur, prereq_type): dep_cache[prereq_type][cur] = r return r +def read_deps_cached(cur, prereq_type): + return cache_func(read_deps, [ cur, prereq_type ]) + def add_tree(cur, prereq_types): debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) if cur in all_deps: @@ -64,7 +95,7 @@ def add_tree(cur, prereq_types): all_deps.add(cur) for t in prereq_types: debug("checking prereqisites of type " + t) - deps.update(read_deps(cur, t)) + deps.update(read_deps_cached(cur, t)) for d in deps: add_tree(d, prereq_types) dep_tree[cur] = deps @@ -92,7 +123,7 @@ def run_make(module, target): global cur_project cur_project=cur_project+1 make_cmd = "make " + target + " 2>&1" - path = find_proj_path(module) + path = find_proj_path_cached(module) delim_len=120 delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path) delim = delim + '-' * (delim_len - len(delim)) @@ -125,6 +156,7 @@ def build(modules, order, target): # ------------ here we go all_deps = Set() +cache = {} visited = {} dep_tree = {} glob_order = [] @@ -154,7 +186,8 @@ modules=args.modules exclude=args.exclude.split() proj_base=args.base target=args.target -projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base + +projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "") env_exclude=os.getenv('BUILD_EXCLUDE', '') if len(env_exclude): diff --git a/scripts/projects.py b/scripts/projects.py index aa5a3fcb..c808f7f1 100644 --- a/scripts/projects.py +++ b/scripts/projects.py @@ -20,6 +20,36 @@ import platform # --------------------------------------------------------------------- helpers +class ResultCache(object): + + def __init__(self): + self.__cache = {} + + def run(self, func, args): + d = self.__cache + depth = 0 + keys = [ func.__name__ ] + args + l = len(keys) + for k in keys: + if k is None: + k = 'None' + else: + k = str(k) + depth += 1 + #debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) + if k in d: + if l == depth: + return d[k] + d = d[k] + continue + if l == depth: + r = func(*args) + d[k] = r + return r + d = d[k] = {} + #d = d[k] + raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) + def debug(*objs): if args.debug: print("DEBUG: ", *objs, file=sys.stderr) @@ -27,29 +57,6 @@ def debug(*objs): def err(*objs): print("ERR: ", *objs, file=sys.stderr) -def cache_func(func, args): - d = cache - depth = 0 - keys = [ func.__name__ ] + args - l = len(keys) - for k in keys: - if k is None: - k = 'None' - depth += 1 - #debug('depth = ', depth, 'key = ', k, 'd = ', str(d)) - if k in d: - if l == depth: - return d[k] - d = d[k] - continue - if l == depth: - r = func(*args) - d[k] = r - return r - d[k] = {} - d = d[k] - raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth) - def proj_dir(name): if name == top_name: return topdir @@ -82,11 +89,6 @@ def get_os(args = ""): return out return "linux" -def get_os_cached(args = ""): - if 'os' in cache: - return cache['os'] - return get_os(args) - # TODO: add support for customizing this in project.conf def htdocs_dir(name): pd = proj_dir(name) @@ -97,7 +99,7 @@ def htdocs_dir(name): return None def os_cascade(): - os = get_os_cached() + os = cache.run(get_os, []) name = re.sub('-.*', '', os) # e.g. os, linux, suse, suse-tumbleweed return [ 'os', platform.system().lower(), name, os ] @@ -121,9 +123,6 @@ def get_section(path, section): file.close() return r.rstrip() -def get_section_cached(path, section): - return cache_func(get_section, [ path, section ]) - def read_value(path, section, key): debug("opening ", path) try: @@ -162,9 +161,6 @@ def read_value(path, section, key): return r[0] return None -def read_value_cached(path, section, key): - return cache_func(read_value, [path, section, key]) - def get_value(name, section, key): debug("getting value [%s].%s for project %s (%s)" %(section, key, name, top_name)) if top_name and name == top_name: @@ -181,7 +177,7 @@ def get_value(name, section, key): path = proj_root + '/make/project.conf' #print('path = ', path, 'top_name = ', top_name, 'name = ', name) - return read_value_cached(path, section, key) + return cache.run(read_value, [path, section, key]) def collect_values(names, section, key): r = "" @@ -195,6 +191,11 @@ def collect_values(names, section, key): # scope 1: children # scope 2: recursive +def add_modules_from_project_txt_cached(buf, visited, spec, section, key, add_self, scope, + names_only): + return cache.run(add_modules_from_project_txt, [buf, visited, spec, section, key, + add_self, scope, names_only]) + def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope, names_only): name = strip_module_from_spec(spec) @@ -216,7 +217,7 @@ def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, sco subscope = 2 deps = deps.split(',') for dep in deps: - add_modules_from_project_txt(buf, visited, dep, + add_modules_from_project_txt_cached(buf, visited, dep, section, key, add_self=True, scope=subscope, names_only=names_only) if add_self: @@ -232,7 +233,7 @@ def get_modules_from_project_txt(names, section, keys, add_self, scope, visited = Set() for name in names: rr = [] - add_modules_from_project_txt(rr, visited, name, section, key, add_self, scope, + add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope, names_only) # TODO: this looks like a performance hogger for m in rr: @@ -424,6 +425,7 @@ def cmd_exepath(args_): args=parser.parse_args(args_) deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ], scope = 2, add_self=True, names_only=True) + debug('deps = ', deps) r = '' for m in deps: r = r + ':' + proj_dir(m) + '/bin' @@ -591,7 +593,7 @@ def cmd_getval(args_): # -------------------------------------------------------------------- here we go global_args = [] -cache = {} +cache = ResultCache() skip = 0 for a in sys.argv[1::]: @@ -623,7 +625,7 @@ debug("----------------------------------------- running ", ' '.join(sys.argv)) projs_root = args.prefix if args.topdir: topdir = args.topdir - top_name = read_value_cached(topdir + '/make/project.conf', 'build', 'name') + top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name']) if not top_name: top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))