make/defs.mk

@@ -282,7 +282,6 @@ ifeq ($(VCS),cvs)
 endif
 
 REV_SUBDIRS                  = $(call reverse $(SUBDIRS))
-FRESH_CVSDIR                 = $(HOME)/local/src/cvs.fresh
 PCKG_DEFS_DIR                = $(JWBDIR)/make/defs.d
 HDRDIR_SCOPE_SUFFIX         ?= $(PROJECT)
 
@@ -405,7 +404,8 @@ INSTALLED_INIT              += $(addprefix $(INSTALL_INITDIR)/,$(INIT_SCRIPTS))
 
 # -- MAKE
 INSTALLATION_FILE_TYPES     += MAKE
-BUILD_MAKEDIR                = $(TOPDIR)/make
+#BUILD_MAKEDIR is currently not used anywhere and costs performance. Disabled for the time being.
+#BUILD_MAKEDIR                = $(TOPDIR)/make
 MKFILES                     += $(filter-out pckg-defs.mk pckg-deps.mk local.mk,$(filter-out $(DONT_INSTALL),$(wildcard *.mk)))
 INSTALL_MAKEDIR             ?= $(PREFIX)/make
 INSTALLED_MAKE              += $(addprefix $(INSTALL_MAKEDIR)/,$(MKFILES))

make/install-files.mk

@@ -25,6 +25,7 @@ all:
 install: install.done
 clean: done.clean
 distclean:
+test:
 
 done.clean:
 	$(RM) -f *.done

make/ldlibpath.mk

@@ -16,5 +16,3 @@ ifeq ($(origin JW_PKG_EXE_PATH),undefined)
   JW_PKG_EXE_PATH           := $(call proj_query, exepath --delimiter ' ' $(PROJECT) $(PREREQ_RUN))
 endif
 export PATH                 := $(subst $(space),:,$(JW_PKG_EXE_PATH)):$(EXE_SEARCH_PATH_ENV)
-
-include $(JWBDIR)/make/py-path.mk

make/projects-dir.mk

@@ -155,7 +155,8 @@ PROJECTS_WITH_PROJECT_CONF   = $(patsubst %/make/project.conf,%,$(wildcard $(add
 
 # --- mandatory targets
 
-all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
+all:
+all test check check-pre check-post: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
 	$(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS)
 clean: clean-dirs
 distclean: clean-all-dirs done.clean

make/py-check.mk

@@ -0,0 +1,93 @@
+.PHONY: \
+	all \
+	check \
+	format \
+	check-syntax \
+	check-format \
+	py-check \
+	py-check-syntax \
+	py-check-format \
+	py-format \
+	py-format-assignments \
+	py-check-annotation-imports \
+	py-format-annotation-imports \
+	clean \
+	clean.py-check
+
+ifndef PY_CHECK_ROOTS
+  PY_CHECK_ROOTS             = .
+endif
+
+ifndef PY_CHECK_MYPY
+  PY_CHECK_MYPY             := mypy
+endif
+
+ifndef PY_CHECK_RUFF
+  PY_CHECK_RUFF             := $(shell which ruff 2>/dev/null)
+  ifneq ($(PY_CHECK_RUFF),)
+    PY_CHECK_RUFF           += --config $(TOPDIR)/pyproject.toml
+  endif
+endif
+
+ifndef PY_CHECK_YAPF
+  PY_CHECK_YAPF             := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3))
+endif
+
+ifndef PY_CHECK_PYRIGHT
+  PY_CHECK_PYRIGHT          := $(shell which pyright 2>/dev/null)
+  TD_GENERATE_FILES         += pyrightconfig.json
+endif
+
+all:
+check: py-check
+format: py-format
+check-syntax: py-check-syntax
+check-format: py-check-format
+
+py-check: py-check-syntax py-check-format py-check-bad-patterns
+
+py-check-syntax:
+ifneq ($(PY_CHECK_RUFF),)
+	$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
+endif
+	$(PY_CHECK_MYPY) $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
+ifneq ($(PY_CHECK_PYRIGHT),)
+	$(PY_CHECK_PYRIGHT) $(PY_CHECK_ROOTS)
+endif
+
+py-check-bad-patterns:
+	if find $(PY_CHECK_ROOTS) -type f -name '*.py' -print0 | xargs -0 grep breakpoint; then exit 1; fi
+	if find $(PY_CHECK_ROOTS) -type f -name '*.py' -print0 | xargs -0 grep "^\s*).*#\s*export"; then exit 1; fi
+
+py-check-format:
+ifneq ($(PY_CHECK_YAPF),)
+	$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
+endif
+
+py-format:
+	find . -type f -name '*.py' -print0 | \
+		xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
+ifneq ($(PY_CHECK_YAPF),)
+	$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
+endif
+
+py-format-assignments:
+	find $(PY_CHECK_ROOTS) \
+	  -path './.git' -prune -o \
+	  -type f -name '*.py' \
+	  -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
+	git diff --exit-code
+
+py-check-annotation-imports:
+ifneq ($(PY_CHECK_RUFF),)
+	$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
+endif
+
+py-format-annotation-imports:
+ifneq ($(PY_CHECK_RUFF),)
+	$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
+endif
+
+clean: clean.py-check
+clean.py-check:
+	rm -rf .mypy_cache .ruff_cache .pytest_cache

make/py-defs.mk

@@ -19,8 +19,6 @@ else
   PY_SITE_PACKAGES_PATH     := $(shell $(PYTHON) -c "import site; print([d for d in site.getsitepackages() if d.find('/local/') == -1][0])")
 endif
 
-PY_MYPY                     ?= mypy --ignore-missing-imports --no-namespace-packages
-
 PY_SRC_PY                   ?= $(wildcard *.py)
 PY_ALL_PY                    = $(PY_SRC_PY)
 
@@ -37,6 +35,8 @@ endif
 
 # deduce PY_INSTALL_DIR_PY from working directory below .. python/
 ifeq ($(PY_INSTALL_DIR_PY),)
+  ECHO                      ?= echo
+  SED                       ?= sed
   PY_INSTALL_PKG_MOD        ?= $(shell $(ECHO) $(CWD) | $(SED) 's%.*/python/%%; s%/.*%%')
   PY_INSTALL_SUB_MOD        ?= $(shell $(ECHO) $(CWD) | $(SED) "s%.*/$(PY_INSTALL_PKG_MOD)\(/\|$$\)%%")
   ifneq ($(PY_INSTALL_SUB_MOD),)
@@ -93,4 +93,4 @@ endif
 
 PY_DEFS_MK_INCLUDED         := true
 
-include $(JWBDIR)/make/ldlibpath.mk
+include $(JWBDIR)/make/py-path.mk

make/py-path.mk

@@ -11,3 +11,7 @@ all:
 clean: py-tools.clean
 py-tools.clean:
 	$(RM) -rf $(wildcard *.pyc) __pycache__ .mypy_cache .ruff_cache .pytest_cache
+
+py-path:
+	@echo "PYTHONPATH=$(PYTHONPATH)"
+	@#echo "MYPYPATH=$(MYPYPATH)"

make/py-rules.mk

@@ -29,8 +29,7 @@ $(PY_INSTALL_DIR_PY)/%.pyc: %.pyc
 endif
 	$(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@
 
-check:
+include $(JWBDIR)/make/py-check.mk
-	$(PY_MYPY) $(shell /bin/bash $(JWB_SCRIPT_DIR)/scm.sh ls-files | grep '\.py$$')
 
 $(PY_INSTALL_DIR_PY)/py.typed: py.typed
 	$(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@

make/py-run.mk

@@ -1,7 +1,5 @@
 include $(JWBDIR)/make/defs.mk
 include $(JWBDIR)/make/py-defs.mk
-#include $(JWBDIR)/make/scripts-targets.mk
-#include $(JWBDIR)/make/rules.mk
 
 EXE                         ?= $(firstword $(wildcard main.py runme.py test.py *.py))
 EXE_ARGS                    ?=

make/py-topdir.mk

@@ -11,71 +11,9 @@ ifndef PY_CHECK_ROOTS
   PY_CHECK_ROOTS             = $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
 endif
 
-ifndef PY_CHECK_RUFF
-  PY_CHECK_RUFF             := $(shell which ruff 2>/dev/null)
-  ifneq ($(PY_CHECK_RUFF),)
-    PY_CHECK_RUFF           += --config pyproject.toml
-  endif
-endif
-
-ifndef PY_CHECK_YAPF
-  PY_CHECK_YAPF             := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3))
-endif
-
-ifndef PY_CHECK_PYRIGHT
-  PY_CHECK_PYRIGHT          := $(shell which pyright 2>/dev/null)
-  TD_GENERATE_FILES         += pyrightconfig.json
-endif
-
 all:
-
-format: py-format
-check-syntax: py-check-syntax
-check-format: py-check-format
-
-py-check: py-check-syntax py-check-format
-py-check-syntax:
-ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
-endif
-	mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
-ifneq ($(PY_CHECK_PYRIGHT),)
-	pyright $(PY_CHECK_ROOTS)
-endif
-
-py-check-format:
-ifneq ($(PY_CHECK_YAPF),)
-	$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
-endif
-
-py-format:
-	find . -type f -name '*.py' -print0 | \
-		xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
-ifneq ($(PY_CHECK_YAPF),)
-	$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
-endif
-
-py-format-assignments:
-	find $(PY_CHECK_ROOTS) \
-	  -path './.git' -prune -o \
-	  -type f -name '*.py' \
-	  -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
-	git diff --exit-code
-
-py-check-annotation-imports:
-ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
-endif
-
-py-format-annotation-imports:
-ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
-endif
-
 clean: clean.topdir
 clean.topdir: clean.py-check
-clean.py-check:
-	rm -rf .mypy_cache .ruff_cache .pytest_cache
 
 pyproject.toml:
 	$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \
@@ -86,3 +24,5 @@ pyrightconfig.json:
 	$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \
 		--field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp
 	mv $@.tmp $@
+
+include $(JWBDIR)/make/py-check.mk

make/rules.mk

@@ -390,12 +390,14 @@ $$(TOPDIR)/dir_install_$(1).done:
 	mkdir -p $$(INSTALL_$(1)DIR)
 	touch $$@
 
+ifneq ($$(BUILD_$(1)DIR),)
 $$(BUILD_$(1)DIR)/%: % | $$(TOPDIR)/dir_build_$(1).done
 	$(Q)if [ ! $$< -ef $$@ -a "`echo $$< | $(SED) 's/\..*//'`" != local ]; then \
 		echo $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\
 		$(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\
 		$(RM) -f $$(TOPDIR)/dirs-*.done ;\
 	fi
+endif
 
 $$(INSTALL_$(1)DIR):
 ifeq ($(PACKAGE_INSTALL_DIR),true)

make/test-jw-pkg.mk

@@ -1,6 +1,7 @@
 EXE                         ?= $(TOPDIR)/scripts/jw-pkg.py
 
 LOG_LEVEL                   ?= info
+TEST_CMD_GLOBAL_OPTS        ?= -t $(TOPDIR)
 ifneq ($(LOG_LEVEL),)
   TEST_OPTS_LOG_LEVEL       := --log-level $(LOG_LEVEL)
 endif

src/python/jw/pkg/App.py

@@ -5,7 +5,6 @@
 from __future__ import annotations
 
 import os
-import pwd
 import re
 import sys
 
@@ -115,6 +114,8 @@ class App(Base):
                 return self.__pretty_topdir
             return self.__topdir
         for d in [self.__projs_root, self.___opt_root]:
+            if d is None:
+                continue
             ret = d + '/' + name
             if os.path.exists(ret):
                 return ret
@@ -353,7 +354,7 @@ class App(Base):
 
         # -- Members with default values
         self.__topdir_fmt = 'absolute'
-        self.__projs_root = pwd.getpwuid(os.getuid()).pw_dir + '/local/src/jw.dev/proj'
+        self.__projs_root: str | None = None
         self.___opt_root = '/opt'
         self.__pretty_projs_root = None
 
@@ -485,7 +486,9 @@ class App(Base):
         return self.__top_name
 
     @property
-    def projs_root(self):
+    def projs_root(self) -> str:
+        if self.__projs_root is None:
+            raise Exception('Tried to get unknown projects root directory')
         return self.__projs_root
 
     @property

src/python/jw/pkg/cmds/projects/CmdBuild.py

@@ -53,6 +53,14 @@ class CmdBuild(Cmd):  # export
                 'on the command line'
             ),
         )
+        parser.add_argument(
+            '--dep-flavours',
+            default = 'auto',
+            help = (
+                'Dependency flavours to take into consideration for build, '
+                'comma or space separated'
+            )
+        )
         parser.add_argument(
             '--env-reinit',
             action = 'store_true',
@@ -247,9 +255,12 @@ class CmdBuild(Cmd):  # export
             # -- build
             order: list[str] = []
 
+            if args.dep_flavours != 'auto':
+                dep_flavours = re.split(r'[\s,]', args.dep_flavours)
+            else:
                 dep_flavours = ['build']
                 if re.match('pkg-.*', target) is not None:
-                dep_flavours = ['build', 'run', 'release', 'devel']
+                    dep_flavours.extend(['run', 'release', 'devel'])
 
             if target != 'order' and not args.build_order:
                 log(NOTICE, 'Using prerequisite flavours ' + ' '.join(dep_flavours))

src/python/jw/pkg/lib/App.py

@@ -292,8 +292,8 @@ class App:  # export
                 self.__async_runner = None
         return ret
 
-def run_sub_commands(
+def run_sub_commands(  # export
     description = '', name_filter = '^Cmd.*', modules = None, argv = None
-):  # export
+):
     app = App(description, name_filter, modules)
     return app.run(argv = argv)

src/python/jw/pkg/lib/ec/SSHClient.py

@@ -109,9 +109,9 @@ class SSHClient(ExecContext):
     def password(self) -> str | None:
         return self.uri.password
 
-def ssh_client(
+def ssh_client(  # export
     *args, type: str | list[str] | None = None, **kwargs
-) -> SSHClient:  # export
+) -> SSHClient:
     from importlib import import_module
 
     errors: list[str] = []

src/python/jw/pkg/lib/ec/ssh/Paramiko.py

@@ -2,9 +2,11 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-import paramiko  # type: ignore[import-untyped] # error: Library stubs not installed for "paramiko"
+# Tolerate missing paramiko imports. jw-pkg is designed to work with what it
-import paramiko.agent  # type: ignore[import-untyped]
+# finds.
-import paramiko.SCPClient  # type: ignore[import-untyped]
+import paramiko  # type: ignore[import-untyped,import-not-found] # error: Library stubs not installed for "paramiko"
+import paramiko.agent  # type: ignore[import-untyped,import-not-found]
+import paramiko.SCPClient  # type: ignore[import-untyped,import-not-found]
 
 from ...base import Result
 from ...log import ERR, log

src/python/jw/pkg/lib/pm/dpkg.py

@@ -33,20 +33,20 @@ async def _run(
         if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive)
     ).stdout_str
 
-async def run_dpkg(
+async def run_dpkg(  # export
     args: list[str],
     sudo: bool = False,
     ec: ExecContext | None = None
-) -> str:  # export
+) -> str:
     cmd = ['/usr/bin/dpkg']
     cmd.extend(args)
     return await _run(cmd, sudo, ec)
 
-async def run_dpkg_query(
+async def run_dpkg_query(  # export
     args: list[str],
     sudo: bool = False,
     ec: ExecContext | None = None
-) -> str:  # export
+) -> str:
     cmd = ['/usr/bin/dpkg-query']
     cmd.extend(args)
     return await _run(cmd, sudo, ec)

src/python/jw/pkg/lib/pm/rpm.py

@@ -25,13 +25,13 @@ def meta_map():
         )
     return _meta_map
 
-async def run_rpm(
+async def run_rpm(  # export
     args: list[str],
     sudo: bool = False,
     ec: ExecContext | None = None,
     mode: InputMode = InputMode.OptInteractive,
     **kwargs,
-) -> str:  # export
+) -> str:
     cmd = ['/usr/bin/rpm']
     cmd.extend(args)
     result = (
@@ -40,10 +40,10 @@ async def run_rpm(
     )
     return result.stdout_str
 
-async def query_packages(
+async def query_packages(  # export
     names: Iterable[str] = [],
     ec: ExecContext | None = None,
-) -> Iterable[Package]:  # export
+) -> Iterable[Package]:
     fmt_str = (
         '|'.join([(f'%{{{tag}}}' if tag else '')
                   for tag in meta_map().values()]) + r'\n'

src/python/jw/pkg/lib/util.py

@@ -209,12 +209,12 @@ async def copy(
             return e
     assert False, 'Unreachable code'
 
-async def get_username(
+async def get_username(  # export
     args: Namespace | None = None,
     url: str | None = None,
     askpass_env: list[str] = [],
     ec: ExecContext | None = None,
-) -> str | None:  # export
+) -> str | None:
     url_user = None if url is None else Uri(url).username
     if args is not None:
         if args.username is not None:
@@ -228,12 +228,12 @@ async def get_username(
         return url_user
     return await run_askpass(askpass_env, AskpassKey.Username, ec = ec)
 
-async def get_password(
+async def get_password(  # export
     args: Namespace | None = None,
     url: str | None = None,
     askpass_env: list[str] = [],
     ec: ExecContext | None = None,
-) -> str | None:  # export
+) -> str | None:
     if args is None and url is None and not askpass_env:
         raise Exception(
             'Neither URL nor command-line arguments nor askpass environment variable '
@@ -251,11 +251,11 @@ async def get_password(
             return ret
     return await run_askpass(askpass_env, AskpassKey.Password, ec = ec)
 
-async def get_profile_env(
+async def get_profile_env(  # export
     throw: bool = True,
     keep: Iterable[str] | bool = False,
     ec: ExecContext | None = None,
-) -> dict[str, str]:  # export
+) -> dict[str, str]:
     """
     Get a fresh environment from /etc/profile