build.py and projects.py: Cache results queried from file system

Signed-off-by: Jan Lindemann <jan@janware.com>

scripts/projects.py

@@ -20,6 +20,36 @@ import platform
 
 # --------------------------------------------------------------------- helpers
 
+class ResultCache(object):
+
+    def __init__(self):
+        self.__cache = {}
+
+    def run(self, func, args):
+        d = self.__cache
+        depth = 0
+        keys = [ func.__name__ ] + args
+        l = len(keys)
+        for k in keys:
+            if k is None:
+                k = 'None'
+            else:
+                k = str(k)
+            depth += 1
+            #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
+            if k in d:
+                if l == depth:
+                    return d[k]
+                d = d[k]
+                continue
+            if l == depth:
+                r = func(*args)
+                d[k] = r
+                return r
+            d = d[k] = {}
+            #d = d[k]
+        raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
+
 def debug(*objs):
     if args.debug:
         print("DEBUG: ", *objs, file=sys.stderr)
@@ -27,29 +57,6 @@ def debug(*objs):
 def err(*objs):
         print("ERR: ", *objs, file=sys.stderr)
 
-def cache_func(func, args):
-    d = cache
-    depth = 0
-    keys = [ func.__name__ ] + args
-    l = len(keys)
-    for k in keys:
-        if k is None:
-            k = 'None'
-        depth += 1
-        #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
-        if k in d:
-            if l == depth:
-                return d[k]
-            d = d[k]
-            continue
-        if l == depth:
-            r = func(*args)
-            d[k] = r
-            return r
-        d[k] = {}
-        d = d[k]
-    raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
-
 def proj_dir(name):
     if name == top_name:
         return topdir
@@ -82,11 +89,6 @@ def get_os(args = ""):
             return out
     return "linux"
 
-def get_os_cached(args = ""):
-    if 'os' in cache:
-        return cache['os']
-    return get_os(args)
-
 # TODO: add support for customizing this in project.conf
 def htdocs_dir(name):
     pd = proj_dir(name)
@@ -97,7 +99,7 @@ def htdocs_dir(name):
     return None
 
 def os_cascade():
-    os = get_os_cached()
+    os = cache.run(get_os, [])
     name = re.sub('-.*', '', os)
     # e.g. os, linux, suse, suse-tumbleweed
     return [ 'os', platform.system().lower(), name, os ]
@@ -121,9 +123,6 @@ def get_section(path, section):
     file.close()
     return r.rstrip()
 
-def get_section_cached(path, section):
-    return cache_func(get_section, [ path, section ])
-
 def read_value(path, section, key):
     debug("opening ", path)
     try:
@@ -162,9 +161,6 @@ def read_value(path, section, key):
         return r[0]
     return None
 
-def read_value_cached(path, section, key):
-    return cache_func(read_value, [path, section, key])
-
 def get_value(name, section, key):
     debug("getting value [%s].%s for project %s (%s)" %(section, key, name, top_name))
     if top_name and name == top_name:
@@ -181,7 +177,7 @@ def get_value(name, section, key):
 
     path = proj_root + '/make/project.conf'
     #print('path = ', path, 'top_name = ', top_name, 'name = ', name)
-    return read_value_cached(path, section, key)
+    return cache.run(read_value, [path, section, key])
 
 def collect_values(names, section, key):
     r = ""
@@ -195,6 +191,11 @@ def collect_values(names, section, key):
 # scope 1: children
 # scope 2: recursive
 
+def add_modules_from_project_txt_cached(buf, visited, spec, section, key, add_self, scope,
+        names_only):
+    return cache.run(add_modules_from_project_txt, [buf, visited, spec, section, key,
+        add_self, scope, names_only])
+
 def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope,
         names_only):
     name = strip_module_from_spec(spec)
@@ -216,7 +217,7 @@ def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, sco
             subscope = 2
         deps = deps.split(',')
         for dep in deps:
-            add_modules_from_project_txt(buf, visited, dep,
+            add_modules_from_project_txt_cached(buf, visited, dep,
                     section, key, add_self=True, scope=subscope,
                     names_only=names_only)
     if add_self:
@@ -232,7 +233,7 @@ def get_modules_from_project_txt(names, section, keys, add_self, scope,
         visited = Set()
         for name in names:
             rr = []
-            add_modules_from_project_txt(rr, visited, name, section, key, add_self, scope,
+            add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
                     names_only)
             # TODO: this looks like a performance hogger
             for m in rr:
@@ -424,6 +425,7 @@ def cmd_exepath(args_):
     args=parser.parse_args(args_)
     deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ],
             scope = 2, add_self=True, names_only=True)
+    debug('deps = ', deps)
     r = ''
     for m in deps:
         r = r + ':' + proj_dir(m) + '/bin'
@@ -591,7 +593,7 @@ def cmd_getval(args_):
 # -------------------------------------------------------------------- here we go
 
 global_args = []
-cache = {}
+cache = ResultCache()
 
 skip = 0
 for a in sys.argv[1::]:
@@ -623,7 +625,7 @@ debug("----------------------------------------- running ", ' '.join(sys.argv))
 projs_root = args.prefix
 if args.topdir:
     topdir = args.topdir
-    top_name = read_value_cached(topdir + '/make/project.conf', 'build', 'name')
+    top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
     if not top_name:
         top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))