mirror of
ssh://git.janware.com/srv/git/janware/proj/jw-pkg
synced 2026-01-15 03:53:32 +01:00
build.py and projects.py: Cache results queried from file system
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
parent
b26d6c943f
commit
378feb60ec
2 changed files with 78 additions and 43 deletions
|
|
@ -20,6 +20,30 @@ def debug(*objs):
|
|||
if args.debug:
|
||||
print("DEBUG: ", *objs, file=sys.stderr)
|
||||
|
||||
def cache_func(func, args):
|
||||
global cache
|
||||
d = cache
|
||||
depth = 0
|
||||
keys = [ func.__name__ ] + args
|
||||
l = len(keys)
|
||||
for k in keys:
|
||||
if k is None:
|
||||
k = 'None'
|
||||
depth += 1
|
||||
#debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
|
||||
if k in d:
|
||||
if l == depth:
|
||||
return d[k]
|
||||
d = d[k]
|
||||
continue
|
||||
if l == depth:
|
||||
r = func(*args)
|
||||
d[k] = r
|
||||
return r
|
||||
d[k] = {}
|
||||
d = d[k]
|
||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||
|
||||
def find_proj_path(name):
|
||||
name=name.replace("dspider-", "")
|
||||
for sub in search_path:
|
||||
|
|
@ -28,6 +52,9 @@ def find_proj_path(name):
|
|||
return os.path.abspath(path)
|
||||
raise Exception("module " + name + " not found below " + proj_base)
|
||||
|
||||
def find_proj_path_cached(name):
|
||||
return cache_func(find_proj_path, [ name ])
|
||||
|
||||
def read_deps(cur, prereq_type):
|
||||
# dep cache doesn't make a difference at all
|
||||
if prereq_type in dep_cache:
|
||||
|
|
@ -36,6 +63,7 @@ def read_deps(cur, prereq_type):
|
|||
else:
|
||||
dep_cache[prereq_type] = {}
|
||||
cmd = projects_py + " prereq " + prereq_type + " " + cur
|
||||
debug('running', cmd)
|
||||
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
|
||||
p.wait()
|
||||
if p.returncode:
|
||||
|
|
@ -54,6 +82,9 @@ def read_deps(cur, prereq_type):
|
|||
dep_cache[prereq_type][cur] = r
|
||||
return r
|
||||
|
||||
def read_deps_cached(cur, prereq_type):
|
||||
return cache_func(read_deps, [ cur, prereq_type ])
|
||||
|
||||
def add_tree(cur, prereq_types):
|
||||
debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
|
||||
if cur in all_deps:
|
||||
|
|
@ -64,7 +95,7 @@ def add_tree(cur, prereq_types):
|
|||
all_deps.add(cur)
|
||||
for t in prereq_types:
|
||||
debug("checking prereqisites of type " + t)
|
||||
deps.update(read_deps(cur, t))
|
||||
deps.update(read_deps_cached(cur, t))
|
||||
for d in deps:
|
||||
add_tree(d, prereq_types)
|
||||
dep_tree[cur] = deps
|
||||
|
|
@ -92,7 +123,7 @@ def run_make(module, target):
|
|||
global cur_project
|
||||
cur_project=cur_project+1
|
||||
make_cmd = "make " + target + " 2>&1"
|
||||
path = find_proj_path(module)
|
||||
path = find_proj_path_cached(module)
|
||||
delim_len=120
|
||||
delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
|
||||
delim = delim + '-' * (delim_len - len(delim))
|
||||
|
|
@ -125,6 +156,7 @@ def build(modules, order, target):
|
|||
# ------------ here we go
|
||||
|
||||
all_deps = Set()
|
||||
cache = {}
|
||||
visited = {}
|
||||
dep_tree = {}
|
||||
glob_order = []
|
||||
|
|
@ -154,7 +186,8 @@ modules=args.modules
|
|||
exclude=args.exclude.split()
|
||||
proj_base=args.base
|
||||
target=args.target
|
||||
projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base
|
||||
|
||||
projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
|
||||
|
||||
env_exclude=os.getenv('BUILD_EXCLUDE', '')
|
||||
if len(env_exclude):
|
||||
|
|
|
|||
|
|
@ -20,6 +20,36 @@ import platform
|
|||
|
||||
# --------------------------------------------------------------------- helpers
|
||||
|
||||
class ResultCache(object):
|
||||
|
||||
def __init__(self):
|
||||
self.__cache = {}
|
||||
|
||||
def run(self, func, args):
|
||||
d = self.__cache
|
||||
depth = 0
|
||||
keys = [ func.__name__ ] + args
|
||||
l = len(keys)
|
||||
for k in keys:
|
||||
if k is None:
|
||||
k = 'None'
|
||||
else:
|
||||
k = str(k)
|
||||
depth += 1
|
||||
#debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
|
||||
if k in d:
|
||||
if l == depth:
|
||||
return d[k]
|
||||
d = d[k]
|
||||
continue
|
||||
if l == depth:
|
||||
r = func(*args)
|
||||
d[k] = r
|
||||
return r
|
||||
d = d[k] = {}
|
||||
#d = d[k]
|
||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||
|
||||
def debug(*objs):
|
||||
if args.debug:
|
||||
print("DEBUG: ", *objs, file=sys.stderr)
|
||||
|
|
@ -27,29 +57,6 @@ def debug(*objs):
|
|||
def err(*objs):
|
||||
print("ERR: ", *objs, file=sys.stderr)
|
||||
|
||||
def cache_func(func, args):
|
||||
d = cache
|
||||
depth = 0
|
||||
keys = [ func.__name__ ] + args
|
||||
l = len(keys)
|
||||
for k in keys:
|
||||
if k is None:
|
||||
k = 'None'
|
||||
depth += 1
|
||||
#debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
|
||||
if k in d:
|
||||
if l == depth:
|
||||
return d[k]
|
||||
d = d[k]
|
||||
continue
|
||||
if l == depth:
|
||||
r = func(*args)
|
||||
d[k] = r
|
||||
return r
|
||||
d[k] = {}
|
||||
d = d[k]
|
||||
raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
|
||||
|
||||
def proj_dir(name):
|
||||
if name == top_name:
|
||||
return topdir
|
||||
|
|
@ -82,11 +89,6 @@ def get_os(args = ""):
|
|||
return out
|
||||
return "linux"
|
||||
|
||||
def get_os_cached(args = ""):
|
||||
if 'os' in cache:
|
||||
return cache['os']
|
||||
return get_os(args)
|
||||
|
||||
# TODO: add support for customizing this in project.conf
|
||||
def htdocs_dir(name):
|
||||
pd = proj_dir(name)
|
||||
|
|
@ -97,7 +99,7 @@ def htdocs_dir(name):
|
|||
return None
|
||||
|
||||
def os_cascade():
|
||||
os = get_os_cached()
|
||||
os = cache.run(get_os, [])
|
||||
name = re.sub('-.*', '', os)
|
||||
# e.g. os, linux, suse, suse-tumbleweed
|
||||
return [ 'os', platform.system().lower(), name, os ]
|
||||
|
|
@ -121,9 +123,6 @@ def get_section(path, section):
|
|||
file.close()
|
||||
return r.rstrip()
|
||||
|
||||
def get_section_cached(path, section):
|
||||
return cache_func(get_section, [ path, section ])
|
||||
|
||||
def read_value(path, section, key):
|
||||
debug("opening ", path)
|
||||
try:
|
||||
|
|
@ -162,9 +161,6 @@ def read_value(path, section, key):
|
|||
return r[0]
|
||||
return None
|
||||
|
||||
def read_value_cached(path, section, key):
|
||||
return cache_func(read_value, [path, section, key])
|
||||
|
||||
def get_value(name, section, key):
|
||||
debug("getting value [%s].%s for project %s (%s)" %(section, key, name, top_name))
|
||||
if top_name and name == top_name:
|
||||
|
|
@ -181,7 +177,7 @@ def get_value(name, section, key):
|
|||
|
||||
path = proj_root + '/make/project.conf'
|
||||
#print('path = ', path, 'top_name = ', top_name, 'name = ', name)
|
||||
return read_value_cached(path, section, key)
|
||||
return cache.run(read_value, [path, section, key])
|
||||
|
||||
def collect_values(names, section, key):
|
||||
r = ""
|
||||
|
|
@ -195,6 +191,11 @@ def collect_values(names, section, key):
|
|||
# scope 1: children
|
||||
# scope 2: recursive
|
||||
|
||||
def add_modules_from_project_txt_cached(buf, visited, spec, section, key, add_self, scope,
|
||||
names_only):
|
||||
return cache.run(add_modules_from_project_txt, [buf, visited, spec, section, key,
|
||||
add_self, scope, names_only])
|
||||
|
||||
def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope,
|
||||
names_only):
|
||||
name = strip_module_from_spec(spec)
|
||||
|
|
@ -216,7 +217,7 @@ def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, sco
|
|||
subscope = 2
|
||||
deps = deps.split(',')
|
||||
for dep in deps:
|
||||
add_modules_from_project_txt(buf, visited, dep,
|
||||
add_modules_from_project_txt_cached(buf, visited, dep,
|
||||
section, key, add_self=True, scope=subscope,
|
||||
names_only=names_only)
|
||||
if add_self:
|
||||
|
|
@ -232,7 +233,7 @@ def get_modules_from_project_txt(names, section, keys, add_self, scope,
|
|||
visited = Set()
|
||||
for name in names:
|
||||
rr = []
|
||||
add_modules_from_project_txt(rr, visited, name, section, key, add_self, scope,
|
||||
add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
|
||||
names_only)
|
||||
# TODO: this looks like a performance hogger
|
||||
for m in rr:
|
||||
|
|
@ -424,6 +425,7 @@ def cmd_exepath(args_):
|
|||
args=parser.parse_args(args_)
|
||||
deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ],
|
||||
scope = 2, add_self=True, names_only=True)
|
||||
debug('deps = ', deps)
|
||||
r = ''
|
||||
for m in deps:
|
||||
r = r + ':' + proj_dir(m) + '/bin'
|
||||
|
|
@ -591,7 +593,7 @@ def cmd_getval(args_):
|
|||
# -------------------------------------------------------------------- here we go
|
||||
|
||||
global_args = []
|
||||
cache = {}
|
||||
cache = ResultCache()
|
||||
|
||||
skip = 0
|
||||
for a in sys.argv[1::]:
|
||||
|
|
@ -623,7 +625,7 @@ debug("----------------------------------------- running ", ' '.join(sys.argv))
|
|||
projs_root = args.prefix
|
||||
if args.topdir:
|
||||
topdir = args.topdir
|
||||
top_name = read_value_cached(topdir + '/make/project.conf', 'build', 'name')
|
||||
top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
|
||||
if not top_name:
|
||||
top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue