build.py and projects.py: Cache results queried from file system

Signed-off-by: Jan Lindemann <jan@janware.com>

scripts/build.py

@@ -20,6 +20,30 @@ def debug(*objs):
     if args.debug:
         print("DEBUG: ", *objs, file=sys.stderr)
 
+def cache_func(func, args):
+    global cache
+    d = cache
+    depth = 0
+    keys = [ func.__name__ ] + args
+    l = len(keys)
+    for k in keys:
+        if k is None:
+            k = 'None'
+        depth += 1
+        #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
+        if k in d:
+            if l == depth:
+                return d[k]
+            d = d[k]
+            continue
+        if l == depth:
+            r = func(*args)
+            d[k] = r
+            return r
+        d[k] = {}
+        d = d[k]
+    raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
+
 def find_proj_path(name):
     name=name.replace("dspider-", "")
     for sub in search_path:
@@ -28,6 +52,9 @@ def find_proj_path(name):
             return os.path.abspath(path)
     raise Exception("module " + name + " not found below " + proj_base)
 
+def find_proj_path_cached(name):
+    return cache_func(find_proj_path, [ name ])
+
 def read_deps(cur, prereq_type):
     # dep cache doesn't make a difference at all
     if prereq_type in dep_cache:
@@ -36,6 +63,7 @@ def read_deps(cur, prereq_type):
     else:
         dep_cache[prereq_type] = {}
     cmd = projects_py + " prereq " + prereq_type + " " + cur
+    debug('running', cmd)
     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
     p.wait()
     if p.returncode:
@@ -54,6 +82,9 @@ def read_deps(cur, prereq_type):
     dep_cache[prereq_type][cur] = r
     return r
 
+def read_deps_cached(cur, prereq_type):
+    return cache_func(read_deps, [ cur, prereq_type ])
+
 def add_tree(cur, prereq_types):
     debug("adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
     if cur in all_deps:
@@ -64,7 +95,7 @@ def add_tree(cur, prereq_types):
     all_deps.add(cur)
     for t in prereq_types:
         debug("checking prereqisites of type " + t)
-        deps.update(read_deps(cur, t))
+        deps.update(read_deps_cached(cur, t))
     for d in deps:
         add_tree(d, prereq_types)
     dep_tree[cur] = deps
@@ -92,7 +123,7 @@ def run_make(module, target):
     global cur_project
     cur_project=cur_project+1
     make_cmd = "make " + target + " 2>&1"
-    path = find_proj_path(module)
+    path = find_proj_path_cached(module)
     delim_len=120
     delim='---- %d/%d: running %s in %s -' % (cur_project, len(order), make_cmd, path)
     delim = delim + '-' * (delim_len - len(delim))
@@ -125,6 +156,7 @@ def build(modules, order, target):
 # ------------ here we go
 
 all_deps = Set()
+cache = {}
 visited = {}
 dep_tree = {}
 glob_order = []
@@ -154,7 +186,8 @@ modules=args.modules
 exclude=args.exclude.split()
 proj_base=args.base
 target=args.target
-projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base
+
+projects_py="/usr/bin/python " + my_dir + "/projects.py --prefix " + proj_base + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
 
 env_exclude=os.getenv('BUILD_EXCLUDE', '')
 if len(env_exclude):

scripts/projects.py

@@ -20,6 +20,36 @@ import platform
 
 # --------------------------------------------------------------------- helpers
 
+class ResultCache(object):
+
+    def __init__(self):
+        self.__cache = {}
+
+    def run(self, func, args):
+        d = self.__cache
+        depth = 0
+        keys = [ func.__name__ ] + args
+        l = len(keys)
+        for k in keys:
+            if k is None:
+                k = 'None'
+            else:
+                k = str(k)
+            depth += 1
+            #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
+            if k in d:
+                if l == depth:
+                    return d[k]
+                d = d[k]
+                continue
+            if l == depth:
+                r = func(*args)
+                d[k] = r
+                return r
+            d = d[k] = {}
+            #d = d[k]
+        raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
+
 def debug(*objs):
     if args.debug:
         print("DEBUG: ", *objs, file=sys.stderr)
@@ -27,29 +57,6 @@ def debug(*objs):
 def err(*objs):
         print("ERR: ", *objs, file=sys.stderr)
 
-def cache_func(func, args):
-    d = cache
-    depth = 0
-    keys = [ func.__name__ ] + args
-    l = len(keys)
-    for k in keys:
-        if k is None:
-            k = 'None'
-        depth += 1
-        #debug('depth = ', depth, 'key = ', k, 'd = ', str(d))
-        if k in d:
-            if l == depth:
-                return d[k]
-            d = d[k]
-            continue
-        if l == depth:
-            r = func(*args)
-            d[k] = r
-            return r
-        d[k] = {}
-        d = d[k]
-    raise Exception("cache algorithm failed for function", func.__name__, "in depth", depth)
-
 def proj_dir(name):
     if name == top_name:
         return topdir
@@ -82,11 +89,6 @@ def get_os(args = ""):
             return out
     return "linux"
 
-def get_os_cached(args = ""):
-    if 'os' in cache:
-        return cache['os']
-    return get_os(args)
-
 # TODO: add support for customizing this in project.conf
 def htdocs_dir(name):
     pd = proj_dir(name)
@@ -97,7 +99,7 @@ def htdocs_dir(name):
     return None
 
 def os_cascade():
-    os = get_os_cached()
+    os = cache.run(get_os, [])
     name = re.sub('-.*', '', os)
     # e.g. os, linux, suse, suse-tumbleweed
     return [ 'os', platform.system().lower(), name, os ]
@@ -121,9 +123,6 @@ def get_section(path, section):
     file.close()
     return r.rstrip()
 
-def get_section_cached(path, section):
-    return cache_func(get_section, [ path, section ])
-
 def read_value(path, section, key):
     debug("opening ", path)
     try:
@@ -162,9 +161,6 @@ def read_value(path, section, key):
         return r[0]
     return None
 
-def read_value_cached(path, section, key):
-    return cache_func(read_value, [path, section, key])
-
 def get_value(name, section, key):
     debug("getting value [%s].%s for project %s (%s)" %(section, key, name, top_name))
     if top_name and name == top_name:
@@ -181,7 +177,7 @@ def get_value(name, section, key):
 
     path = proj_root + '/make/project.conf'
     #print('path = ', path, 'top_name = ', top_name, 'name = ', name)
-    return read_value_cached(path, section, key)
+    return cache.run(read_value, [path, section, key])
 
 def collect_values(names, section, key):
     r = ""
@@ -195,6 +191,11 @@ def collect_values(names, section, key):
 # scope 1: children
 # scope 2: recursive
 
+def add_modules_from_project_txt_cached(buf, visited, spec, section, key, add_self, scope,
+        names_only):
+    return cache.run(add_modules_from_project_txt, [buf, visited, spec, section, key,
+        add_self, scope, names_only])
+
 def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, scope,
         names_only):
     name = strip_module_from_spec(spec)
@@ -216,7 +217,7 @@ def add_modules_from_project_txt(buf, visited, spec, section, key, add_self, sco
             subscope = 2
         deps = deps.split(',')
         for dep in deps:
-            add_modules_from_project_txt(buf, visited, dep,
+            add_modules_from_project_txt_cached(buf, visited, dep,
                     section, key, add_self=True, scope=subscope,
                     names_only=names_only)
     if add_self:
@@ -232,7 +233,7 @@ def get_modules_from_project_txt(names, section, keys, add_self, scope,
         visited = Set()
         for name in names:
             rr = []
-            add_modules_from_project_txt(rr, visited, name, section, key, add_self, scope,
+            add_modules_from_project_txt_cached(rr, visited, name, section, key, add_self, scope,
                     names_only)
             # TODO: this looks like a performance hogger
             for m in rr:
@@ -424,6 +425,7 @@ def cmd_exepath(args_):
     args=parser.parse_args(args_)
     deps = get_modules_from_project_txt(args.module, 'pkg.requires.jw', [ 'run', 'build', 'devel' ],
             scope = 2, add_self=True, names_only=True)
+    debug('deps = ', deps)
     r = ''
     for m in deps:
         r = r + ':' + proj_dir(m) + '/bin'
@@ -591,7 +593,7 @@ def cmd_getval(args_):
 # -------------------------------------------------------------------- here we go
 
 global_args = []
-cache = {}
+cache = ResultCache()
 
 skip = 0
 for a in sys.argv[1::]:
@@ -623,7 +625,7 @@ debug("----------------------------------------- running ", ' '.join(sys.argv))
 projs_root = args.prefix
 if args.topdir:
     topdir = args.topdir
-    top_name = read_value_cached(topdir + '/make/project.conf', 'build', 'name')
+    top_name = cache.run(read_value, [topdir + '/make/project.conf', 'build', 'name'])
     if not top_name:
         top_name = re.sub('-[0-9.-]*$', '', basename(realpath(topdir)))