jw.pkg.App: Support --topdir-format

Add support for --topdir-format. The option supports several
different values, affecting the console output of App wherever it
knows that the output contains a reference to the projects' toplevel
directory.

  - "unaltered" will have it print the toplevel directory in the same
    format as passed to the commandline

  - "absolute" will try to resolve it to an absolute path before
    printing

  - make:XXX will return the make-varible $(XXX) instead

To implement this, the proj_dir() member function is turned into the
private member function __proj_dir(), and a new member function
find_dir() is supplied, with two additional parameters:
search_subdirs and search_absdirs, which will try to find an existing
directory relative to the toplevel directory of the given module, or
in the search_absdirs list, respectively.

Command modules in cmds.projects have been updated to use the new
function.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/projects.mk

@@ -30,7 +30,7 @@ endif
 include $(JWBDIR)/make/py-version.mk
 
 ifneq ($(TOPDIR),)
-  proj_query_cmd             = PYTHONPATH=$(JWBDIR)/src/python $(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_OPTS) projects
+  proj_query_cmd             = PYTHONPATH=$(JWBDIR)/src/python $(PYTHON) $(JWB_SCRIPT_DIR)/jw-projects.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format absolute $(PROJECTS_PY_EXTRA_OPTS) projects
   proj_query                 = $(shell $(proj_query_cmd) $(1))
   proj_dir                   = $(call proj_query,proj-dir $(1))
   htdocs_dir                 = $(call proj_query,htdocs-dir $(1))

src/python/jw/pkg/App.py

@@ -49,6 +49,64 @@ class ResultCache(object):
 
 class App(Base):
 
+    def __format_topdir(self, topdir: None|str, fmt: str) -> str:
+        if topdir is None:
+            return None
+        match fmt:
+            case 'unaltered':
+                return topdir
+            case None | 'absolute':
+                return os.path.abspath(self.topdir)
+            case _:
+                m = re.search(r'^make:(\S+)$', fmt)
+                if m is None:
+                    raise Exception(f'Can\'t interpret "{fmt}" as valid topdir ' +
+                                    'reference, expecting "unaltered", "absolute", or "make:<variable-name>"')
+                return '$(' + m.group(1) + ')'
+
+    def __proj_dir(self, name: str, pretty) -> str:
+        if name == self.top_name:
+            if pretty:
+                return self.__pretty_topdir
+            return self.topdir
+        for d in [ self.projs_root, '/opt' ]:
+            ret = d + '/' + name
+            if os.path.exists(ret):
+                return ret
+        if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'):
+            # The package exists but does not have a dedicated project directory
+            return None
+        raise Exception('No project path found for module "{}"'.format(name))
+
+    def __find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True):
+        def format_pd(name: str, pd: str, pretty: bool):
+            if not pretty:
+                return pd
+            if self.__topdir_fmt == 'absolute':
+                return os.path.abspath(pd)
+            if self.__topdir_fmt == 'unaltered':
+                return pd
+            if name == self.top_name:
+                return self.__pretty_topdir
+            raise NotImplementedError(f'Tried to pretty-format directory {pd}, not implemented')
+        pd = self.__proj_dir(name, False)
+        if pd is None:
+            return None
+        if not search_subdirs and not search_absdirs:
+            return format_pd(name, pd, pretty)
+        for sd in search_subdirs:
+            path = pd + '/' + sd
+            if os.path.isdir(path):
+                ret = format_pd(name, pd, pretty)
+                if sd and sd[0] != '/':
+                    ret += '/'
+                ret += sd
+                return ret
+        for ret in search_absdirs:
+            if os.path.isdir(ret):
+                return ret
+        return None
+
     def __init__(self):
         super().__init__("jw-pkg swiss army knife", modules=["jw.pkg.cmds"])
 
@@ -60,15 +118,20 @@ class App(Base):
         self.dep_cache = {}
         self.my_dir = os.path.dirname(os.path.realpath(__file__))
         self.res_cache = ResultCache()
+        self.__topdir_fmt = 'absolute'
         self.topdir = None
+        self.__pretty_topdir = None
 
         # -- defaults
         self.projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
+        self.__pretty_projs_root = None
 
     def _add_arguments(self, parser):
         super()._add_arguments(parser)
-        parser.add_argument('-t', '--topdir', nargs=1, default = [], help='Project Path')
+        parser.add_argument('-t', '--topdir', default = None, help='Project Path')
-        parser.add_argument('-p', '--prefix', nargs=1, default = [ self.projs_root ],
+        parser.add_argument('--topdir-format', default = 'absolute', help='Output references to topdir as '
+            + 'one of "make:<var-name>", "unaltered", "absolute". Absolute topdir by default')
+        parser.add_argument('-p', '--prefix', default = None,
             help='Parent directory of project source directories')
         parser.add_argument('-O', '--os', default = None, help='Target operating system')
 
@@ -76,18 +139,27 @@ class App(Base):
 
         self.opt_os = args.os
 
-        if args.prefix is not None:
-            self.projs_root = args.prefix
-        self.debug("projs_root = ", self.projs_root)
         self.topdir = args.topdir
-
+        self.__pretty_topdir = self.__format_topdir(self.topdir, args.topdir_format)
+        self.__topdir_fmt = args.topdir_format
         if self.topdir is not None:
             self.top_name = self.res_cache.run(self.read_value, [self.topdir + '/make/project.conf', 'build', 'name'])
             if not self.top_name:
                 self.top_name = re.sub('-[0-9.-]*$', '', os.path.basename(os.path.realpath(self.topdir)))
 
+        if args.prefix is not None:
+            self.projs_root = args.prefix
+            self.__pretty_projs_root = args.prefix
+        #if self.__topdir is not None:
+        #    self.
+
+        log(DEBUG, "projs_root = ", self.projs_root)
+
         return await super()._run(args)
 
+    def find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True):
+        return self.__find_dir(name, search_subdirs, search_absdirs, pretty)
+
     def debug(self, *objs):
         log(DEBUG, *objs)
 
@@ -97,18 +169,6 @@ class App(Base):
     def err(self, *objs):
         log(ERR, *objs)
 
-    def proj_dir(self, name):
-        if name == self.top_name:
-            return os.path.abspath(self.topdir)
-        for d in [ self.projs_root, '/opt' ]:
-            r = d + '/' + name
-            if os.path.exists(r):
-                return os.path.abspath(r)
-        if os.path.exists(f'/usr/share/doc/packages/{name}/VERSION'):
-            # The package exists but does not have a dedicated project directory
-            return None
-        raise Exception('No project path found for module "{}"'.format(name))
-
     def re_section(self, name):
         return re.compile('[' + name + ']'
                      '.*?'
@@ -138,24 +198,13 @@ class App(Base):
         return "linux"
 
     # TODO: add support for customizing this in project.conf
-    def htdocs_dir(self, name):
+    def htdocs_dir(self, name: str) -> str:
-        pd = self.proj_dir(name)
+        return self.find_dir(name,["/src/html/htdocs", "/tools/html/htdocs", "/htdocs"],
-        if pd is None:
+                                  ["/srv/www/proj/" + name])
-            return None
-        for r in [ pd + "/src/html/htdocs", pd + "/tools/html/htdocs", pd + "/htdocs", "/srv/www/proj/" + name ]:
-            if os.path.isdir(r):
-                return r
-        return None
 
     # TODO: add support for customizing this in project.conf
-    def tmpl_dir(self, name):
+    def tmpl_dir(self, name: str) -> str:
-        pd = self.proj_dir(name)
+        return self.find_dir(name, ["/tmpl"],  ["/opt/" + name + "/share/tmpl"])
-        if pd is None:
-            return None
-        for r in [ pd + "/tmpl", "/opt/" + name + "/share/tmpl" ]:
-            if os.path.isdir(r):
-                return r
-        return None
 
     def os_cascade(self):
         import platform
@@ -363,40 +412,6 @@ class App(Base):
                     return p1
         return None
 
-    # -L needs to contain more paths than libs linked with -l would require
-    def get_ldpathflags(self, names, exclude = []):
-        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
-                scope = 2, add_self=True, names_only=True)
-        r = ''
-        for m in deps:
-            if m in exclude:
-                continue
-            libname = self.get_libname([m])
-            if len(libname):
-                r = r + ' -L' + self.proj_dir(m) + '/lib'
-        print(r[1:])
-
-    def get_ldflags(self, names, exclude = [], add_self_ = False):
-        #print(names)
-        deps = self.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
-                scope = 1, add_self=add_self_, names_only=True)
-        self.debug("deps = " + ' '.join(deps))
-        #print(deps)
-        r = ''
-        for m in reversed(deps):
-            if m in exclude:
-                continue
-            libname = self.get_libname([m])
-            if len(libname):
-                #r = r + ' -L' + self.proj_dir(m) + '/lib -l' + libname
-                r = r + ' -l' + libname
-        if len(r):
-            ldpathflags = self.get_ldpathflags(names, exclude)
-            if ldpathflags:
-                r = ldpathflags + ' ' + r
-            return r[1::]
-        return ''
-
     def contains(self, small, big):
         for i in xrange(len(big)-len(small)+1):
             for j in xrange(len(small)):

src/python/jw/pkg/cmds/projects/CmdBuild.py

@@ -84,7 +84,7 @@ class CmdBuild(Cmd): # export
         def run_make(module, target, cur_project, num_projects):
             #make_cmd = "make " + target + " 2>&1"
             make_cmd = [ "make", target ]
-            path = self.app.proj_dir(module)
+            path = self.app.find_dir(module, pretty=False)
             delim_len = 120
             delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
             delim = delim + '-' * (delim_len - len(delim))

src/python/jw/pkg/cmds/projects/CmdCflags.py

@@ -16,13 +16,10 @@ class CmdCflags(Cmd): # export
     def _run(self, args: Namespace) -> None:
         deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build',
                 scope = 2, add_self=True, names_only=True)
-        r = ''
+        out = []
         for m in reversed(deps):
-            try:
+            path = self.app.find_dir(m, ['/include'])
-                pd = self.app.proj_dir(m)
+            if path is not None:
-                if pd is None:
+                out.append('-I' + path)
-                    continue
+        if out:
-                r = r + ' -I' + pd + '/include'
+            print(' '.join(out))
-            except Exception as e:
-                self.app.warn(f'No include path for module "{m}", ignoring: {e}')
-        print(r[1:])

src/python/jw/pkg/cmds/projects/CmdExepath.py

@@ -16,11 +16,9 @@ class CmdExepath(Cmd): # export
     def _run(self, args: Namespace) -> None:
         deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
                 scope = 2, add_self=True, names_only=True)
-        self.app.debug('deps = ', deps)
+        out = []
-        r = ''
         for m in deps:
-            pd = self.app.proj_dir(m)
+            path = self.app.find_dir(m, ['/bin'])
-            if pd is None:
+            if path is not None:
-                continue
+                out.append(path)
-            r = r + ':' + pd + '/bin'
+        print(':'.join(out))
-        print(r[1:])

src/python/jw/pkg/cmds/projects/CmdGetAuthInfo.py

@@ -23,7 +23,7 @@ class CmdGetAuthInfo(Cmd): # export
         keys = ['username', 'password']
 
         # --- Milk jw-pkg repo
-        jw_pkg_dir = self.app.proj_dir('jw-pkg')
+        jw_pkg_dir = self.app.find_dir('jw-pkg', pretty=False)
         if not os.path.isdir(jw_pkg_dir + '/.git'):
             self.app.debug(f'jw-pkg directory is not a Git repo: {jw_pkg_dir}')
             return

src/python/jw/pkg/cmds/projects/CmdLdflags.py

@@ -16,5 +16,38 @@ class CmdLdflags(Cmd): # export
         parser.add_argument('-s', '--add-self', action='store_true',
             default=False, help='Include libflags of specified modules, too, not only their dependencies')
 
+    # -L needs to contain more paths than libs linked with -l would require
+    def __get_ldpathflags(self, names: list[str], exclude: list[str] = []) -> str:
+        deps = self.app.get_modules_from_project_txt(names, ['pkg.requires.jw'], 'build',
+                scope = 2, add_self=True, names_only=True)
+        ret = []
+        for m in deps:
+            if m in exclude:
+                continue
+            libname = self.app.get_libname([m])
+            if not len(libname):
+                continue
+            path = self.app.find_dir(m, ['/lib'])
+            if not path:
+                continue
+            ret.append('-L' + path)
+        if not ret:
+            return None
+        return(' '.join(ret))
+
     def _run(self, args: Namespace) -> None:
-        print(self.app.get_ldflags(args.module, args.exclude, args.add_self))
+        deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'build',
+                scope = 1, add_self=args.add_self, names_only=True)
+        out = []
+        for m in reversed(deps):
+            if m in args.exclude:
+                continue
+            libname = self.app.get_libname([m])
+            if len(libname):
+                out.append('-l' + libname)
+        if not out:
+            return
+        ldpathflags = self.__get_ldpathflags(args.module, args.exclude)
+        if ldpathflags is not None:
+            out.insert(0, ldpathflags)
+        print(' '.join(out))

src/python/jw/pkg/cmds/projects/CmdLdlibpath.py

@@ -16,10 +16,9 @@ class CmdLdlibpath(Cmd): # export
     def _run(self, args: Namespace) -> None:
         deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ],
                 scope = 2, add_self=True, names_only=True)
-        r = ''
+        out = []
         for m in deps:
-            pd = self.app.proj_dir(m)
+            path = self.app.find_dir(m, ['/lib'])
-            if pd is None:
+            if path is not None:
-                continue
+                out.append(path)
-            r = r + ':' + pd + '/lib'
+        print(':'.join(out))
-        print(r[1:])

src/python/jw/pkg/cmds/projects/CmdPath.py

@@ -16,10 +16,9 @@ class CmdPath(Cmd): # export
     def _run(self, args: Namespace) -> None:
         deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], 'run',
                 scope = 2, add_self=True, names_only=True)
-        r = ''
+        out = []
         for m in deps:
-            pd = self.app.proj_dir(m)
+            path = self.app.find_dir(m, '/bin')
-            if pd is None:
+            if path is not None:
-                continue
+                out.append(path)
-            r = r + ':' + pd + '/bin'
+        print(':'.join(out))
-        print(r[1:])

src/python/jw/pkg/cmds/projects/CmdProjDir.py

@@ -14,14 +14,12 @@ class CmdProjDir(Cmd): # export
         parser.add_argument('module', nargs='*', help='Modules')
 
     def _run(self, args: Namespace) -> None:
-        r = []
+        out = []
         for m in args.module:
-            try:
+            path = self.app.find_dir(m)
-                pd = self.app.proj_dir(m)
+            if path is None:
-                if pd is None:
-                    continue
-                r.append(pd)
-            except Exception as e:
                 self.app.warn(f'No project directory for module "{m}: {e}')
                 continue
-        print(' '.join(r))
+            out.append(path)
+        if out:
+            print(' '.join(out))

src/python/jw/pkg/cmds/projects/CmdPythonpath.py

@@ -14,16 +14,12 @@ class CmdPythonpath(Cmd): # export
         p.add_argument('module', help='Modules', nargs='*')
 
     def _run(self, args: Namespace) -> None:
-        import os
         deps = self.app.get_modules_from_project_txt(args.module, ['pkg.requires.jw'], [ 'run', 'build' ],
                 scope = 2, add_self=True, names_only=True)
-        r = ''
+        out = []
         for m in deps:
-            pd = self.app.proj_dir(m)
+            path = self.app.find_dir(m, ['src/python', 'tools/python'])
-            if pd is None:
+            if path is not None:
-                continue
+                out.append(path)
-            for subdir in [ 'src/python', 'tools/python' ]:
+        if out:
-                cand = pd + "/" + subdir
+            print(':'.join(out))
-                if os.path.isdir(cand):
-                    r = r + ':' + cand
-        print(r[1:])

src/python/jw/pkg/cmds/projects/CmdPythonpathOrig.py

@@ -18,7 +18,7 @@ class CmdPythonpathOrig(Cmd): # export
                 scope = 2, add_self=True, names_only=True)
         r = ''
         for m in deps:
-            pd = self.app.proj_dir(m)
+            pd = self.app.find_dir(m, pretty=False)
             if pd is None:
                 continue
             for subdir in [ 'src/python', 'tools/python' ]: