janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

make: Make targets tests & check more universally useful #51

Closed
Jan Lindemann wants to merge 9 commits from jan/feature/20260722-make-make-targets-tests-check-more-universally-useful into master AGit
pull from: jan/feature/20260722-make-make-targets-tests-check-more-universally-useful
merge into: janware:master
Conversation 0 Commits 9 Files changed 12 +141 -91
12 changed files with 141 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

3
make/defs.mk
View file

@ -405,7 +405,8 @@ INSTALLED_INIT += $(addprefix $(INSTALL_INITDIR)/,$(INIT_SCRIPTS))
# -- MAKE
INSTALLATION_FILE_TYPES += MAKE
BUILD_MAKEDIR = $(TOPDIR)/make
#BUILD_MAKEDIR is currently not used anywhere and costs performance. Disabled for the time being.
#BUILD_MAKEDIR = $(TOPDIR)/make
MKFILES += $(filter-out pckg-defs.mk pckg-deps.mk local.mk,$(filter-out $(DONT_INSTALL),$(wildcard *.mk)))
INSTALL_MAKEDIR ?= $(PREFIX)/make
INSTALLED_MAKE += $(addprefix $(INSTALL_MAKEDIR)/,$(MKFILES))

1
make/install-files.mk
View file

@ -25,6 +25,7 @@ all:
install: install.done
clean: done.clean
distclean:
test:
done.clean:
$(RM) -f *.done

2
make/ldlibpath.mk
View file

@ -16,5 +16,3 @@ ifeq ($(origin JW_PKG_EXE_PATH),undefined)
JW_PKG_EXE_PATH := $(call proj_query, exepath --delimiter ' ' $(PROJECT) $(PREREQ_RUN))
endif
export PATH := $(subst $(space),:,$(JW_PKG_EXE_PATH)):$(EXE_SEARCH_PATH_ENV)
include $(JWBDIR)/make/py-path.mk

3
make/projects-dir.mk
View file

@ -155,7 +155,8 @@ PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(add
# --- mandatory targets
all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
all:
all test check check-pre check-post: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
$(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS)
clean: clean-dirs
distclean: clean-all-dirs done.clean

89
make/py-check.mk Normal file
View file

@ -0,0 +1,89 @@
.PHONY: \
all \
check \
format \
check-syntax \
check-format \
py-check \
py-check-syntax \
py-check-format \
py-format \
py-format-assignments \
py-check-annotation-imports \
py-format-annotation-imports \
clean \
clean.py-check
ifndef PY_CHECK_ROOTS
PY_CHECK_ROOTS = .
endif
ifndef PY_CHECK_MYPY
PY_CHECK_MYPY := mypy
endif
ifndef PY_CHECK_RUFF
PY_CHECK_RUFF := $(shell which ruff 2>/dev/null)
ifneq ($(PY_CHECK_RUFF),)
PY_CHECK_RUFF += --config $(TOPDIR)/pyproject.toml
endif
endif
ifndef PY_CHECK_YAPF
PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3))
endif
ifndef PY_CHECK_PYRIGHT
PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null)
TD_GENERATE_FILES += pyrightconfig.json
endif
all:
check: py-check
format: py-format
check-syntax: py-check-syntax
check-format: py-check-format
py-check: py-check-syntax py-check-format
py-check-syntax:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
endif
$(PY_CHECK_MYPY) $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
ifneq ($(PY_CHECK_PYRIGHT),)
$(PY_CHECK_PYRIGHT) $(PY_CHECK_ROOTS)
endif
py-check-format:
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
endif
py-format:
find . -type f -name '*.py' -print0 | \
xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
endif
py-format-assignments:
find $(PY_CHECK_ROOTS) \
-path './.git' -prune -o \
-type f -name '*.py' \
-execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
git diff --exit-code
py-check-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
endif
py-format-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
endif
clean: clean.py-check
clean.py-check:
rm -rf .mypy_cache .ruff_cache .pytest_cache

4
make/py-defs.mk
View file

@ -19,8 +19,6 @@ else
PY_SITE_PACKAGES_PATH := $(shell $(PYTHON) -c "import site; print([d for d in site.getsitepackages() if d.find('/local/') == -1][0])")
endif
PY_MYPY ?= mypy --ignore-missing-imports --no-namespace-packages
PY_SRC_PY ?= $(wildcard *.py)
PY_ALL_PY = $(PY_SRC_PY)
@ -37,6 +35,8 @@ endif
# deduce PY_INSTALL_DIR_PY from working directory below .. python/
ifeq ($(PY_INSTALL_DIR_PY),)
ECHO ?= echo
SED ?= sed
PY_INSTALL_PKG_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) 's%.*/python/%%; s%/.*%%')
PY_INSTALL_SUB_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) "s%.*/$(PY_INSTALL_PKG_MOD)\(/\|$$\)%%")
ifneq ($(PY_INSTALL_SUB_MOD),)

4
make/py-path.mk
View file

@ -11,3 +11,7 @@ all:
clean: py-tools.clean
py-tools.clean:
$(RM) -rf $(wildcard *.pyc) __pycache__ .mypy_cache .ruff_cache .pytest_cache
py-path:
@echo "PYTHONPATH=$(PYTHONPATH)"
@#echo "MYPYPATH=$(MYPYPATH)"

3
make/py-rules.mk
View file

@ -29,8 +29,7 @@ $(PY_INSTALL_DIR_PY)/%.pyc: %.pyc
endif
$(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@
check:
$(PY_MYPY) $(shell /bin/bash $(JWB_SCRIPT_DIR)/scm.sh ls-files | grep '\.py$$')
include $(JWBDIR)/make/py-check.mk
$(PY_INSTALL_DIR_PY)/py.typed: py.typed
$(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@

64
make/py-topdir.mk
View file

@ -11,71 +11,9 @@ ifndef PY_CHECK_ROOTS
PY_CHECK_ROOTS = $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
endif
ifndef PY_CHECK_RUFF
PY_CHECK_RUFF := $(shell which ruff 2>/dev/null)
ifneq ($(PY_CHECK_RUFF),)
PY_CHECK_RUFF += --config pyproject.toml
endif
endif
ifndef PY_CHECK_YAPF
PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3))
endif
ifndef PY_CHECK_PYRIGHT
PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null)
TD_GENERATE_FILES += pyrightconfig.json
endif
all:
format: py-format
check-syntax: py-check-syntax
check-format: py-check-format
py-check: py-check-syntax py-check-format
py-check-syntax:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
endif
mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
ifneq ($(PY_CHECK_PYRIGHT),)
pyright $(PY_CHECK_ROOTS)
endif
py-check-format:
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
endif
py-format:
find . -type f -name '*.py' -print0 | \
xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
endif
py-format-assignments:
find $(PY_CHECK_ROOTS) \
-path './.git' -prune -o \
-type f -name '*.py' \
-execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
git diff --exit-code
py-check-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
endif
py-format-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
endif
clean: clean.topdir
clean.topdir: clean.py-check
clean.py-check:
rm -rf .mypy_cache .ruff_cache .pytest_cache
pyproject.toml:
$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \
@ -86,3 +24,5 @@ pyrightconfig.json:
$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \
--field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp
mv $@.tmp $@
include $(JWBDIR)/make/py-check.mk

2
make/rules.mk
View file

@ -390,12 +390,14 @@ $$(TOPDIR)/dir_install_$(1).done:
mkdir -p $$(INSTALL_$(1)DIR)
touch $$@
ifneq ($$(BUILD_$(1)DIR),)
$$(BUILD_$(1)DIR)/%: % | $$(TOPDIR)/dir_build_$(1).done
$(Q)if [ ! $$< -ef $$@ -a "`echo $$< | $(SED) 's/\..*//'`" != local ]; then \
echo $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\
$(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\
$(RM) -f $$(TOPDIR)/dirs-*.done ;\
fi
endif
$$(INSTALL_$(1)DIR):
ifeq ($(PACKAGE_INSTALL_DIR),true)

17
src/python/jw/pkg/cmds/projects/CmdBuild.py
View file

@ -53,6 +53,14 @@ class CmdBuild(Cmd): # export
'on the command line'
),
)
parser.add_argument(
'--dep-flavours',
default = 'auto',
help = (
'Dependency flavours to take into consideration for build, '
'comma or space separated'
)
)
parser.add_argument(
'--env-reinit',
action = 'store_true',
@ -247,9 +255,12 @@ class CmdBuild(Cmd): # export
# -- build
order: list[str] = []
dep_flavours = ['build']
if re.match('pkg-.*', target) is not None:
dep_flavours = ['build', 'run', 'release', 'devel']
if args.dep_flavours != 'auto':
dep_flavours = re.split(r'[\s,]', args.dep_flavours)
else:
dep_flavours = ['build']
if re.match('pkg-.*', target) is not None:
dep_flavours.extend(['run', 'release', 'devel'])
if target != 'order' and not args.build_order:
log(NOTICE, 'Using prerequisite flavours ' + ' '.join(dep_flavours))

40
test/integration/jw-pkg/help/test-expected.txt
View file

@ -254,31 +254,35 @@ Available subcommands of projects:
tmpl-dir Print directory containing templates of a given module
============= Running: jw-pkg.py --log-level info projects build --help
usage: jw-pkg.py projects build [-h] [--exclude EXCLUDE] [-n] [-O] [-I]
[--env-reinit] [--env-keep ENV_KEEP]
[--dep-flavours DEP_FLAVOURS] [--env-reinit]
[--env-keep ENV_KEEP]
target modules [modules ...]
janware software project build tool
positional arguments:
target Build target
modules Modules to be built
target Build target
modules Modules to be built
options:
-h, --help show this help message and exit
--exclude EXCLUDE Space seperated ist of modules to be excluded from
build (default: )
-n, --dry-run Don't build anything, just print what would be done.
(default: False)
-O, --build-order Don't build anything, just print the build order.
(default: False)
-I, --ignore-deps Don't build dependencies, i.e. build only modules
specified on the command line (default: False)
--env-reinit Source /etc/profile before each build step. Discard
environment unless --env-keep is specified (default:
False)
--env-keep ENV_KEEP Comma seperated list of environment variables to keep,
"all" or "none", only meaningful if --env-reinit is
specified (default: none)
-h, --help show this help message and exit
--exclude EXCLUDE Space seperated ist of modules to be excluded from
build (default: )
-n, --dry-run Don't build anything, just print what would be done.
(default: False)
-O, --build-order Don't build anything, just print the build order.
(default: False)
-I, --ignore-deps Don't build dependencies, i.e. build only modules
specified on the command line (default: False)
--dep-flavours DEP_FLAVOURS
Dependency flavours to take into consideration for
build, comma or space separated (default: auto)
--env-reinit Source /etc/profile before each build step. Discard
environment unless --env-keep is specified (default:
False)
--env-keep ENV_KEEP Comma seperated list of environment variables to keep,
"all" or "none", only meaningful if --env-reinit is
specified (default: none)
============= Running: jw-pkg.py --log-level info projects canonicalize-remotes --help
usage: jw-pkg.py projects canonicalize-remotes [-h] [-n]