From 80248dec8877e0dfb4370392d0a70f51e31dc217 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 12 Jun 2026 06:57:23 +0200 Subject: [PATCH 1/2] python-tools.sh, py-mod.mk: Use --symbol-filter Letting python-tools.sh rewrite symbols is more robust than rewriting an entire __init__.py with PY_INIT_FILTER in the including Makefile. The latter can break in non-obvious ways if python-tools.sh changes __init__.py's format. Make python-tools.sh support --symbol-filter to remedy that. The option takes an sed script which should expect a string of two non-whitespace tokens: The module from which the symbol is imported, and the name of the symbol in that module. It's output will then be used as the symbol to be exported from __init__.py. Also, support the PY_SYMBOL_FILTER variable in py-mod.mk. If it's defined, it is used for --symbol-filter. Signed-off-by: Jan Lindemann --- make/py-mod.mk | 6 +++++- scripts/python-tools.sh | 40 +++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/make/py-mod.mk b/make/py-mod.mk index c7819114..882db095 100644 --- a/make/py-mod.mk +++ b/make/py-mod.mk @@ -6,6 +6,10 @@ include $(JWBDIR)/make/dev-utils.mk PY_INIT_TMPL = $(wildcard __init__.py.tmpl) PY_SED_EXTRACT_EXPORT ?= /\(\(class\|def\)\s\+[a-zA-Z_].*\|^ *\S\+\s*=.*\)\# *export/ !d; /^\s*\#/ d; s/\(async\)* *\(class\|def\) *//; s/[(:=].*// PY_INIT_FILTER ?= cat +PY_GENERATE_INIT_PY ?= /bin/bash $(JWB_SCRIPT_DIR)/python-tools.sh create-init -m . -e "$(PY_SED_EXTRACT_EXPORT)" +ifneq ($(PY_SYMBOL_FILTER),) + PY_GENERATE_INIT_PY += --symbol-filter "$(PY_SYMBOL_FILTER)" +endif #leftparen := ( #PY_EXPORT ?= $(shell sed '/\(class\|def\) ..*\# *export/ !d; s/\(class\|def\) *//; s/[$(leftparen):].*//' $(PY_SRC_PY)) @@ -20,7 +24,7 @@ include $(JWBDIR)/make/py-rules.mk ifeq ($(PY_UPDATE_INIT_PY),true) __init__.py: $(PY_INIT_TMPL) $(filter-out __init__.py,$(PY_SRC_PY)) if [ "$(PY_INIT_TMPL)" ]; then cat "$(PY_INIT_TMPL)" > $@.tmp; else > $@.tmp; fi - set -e -o pipefail; /bin/bash $(JWB_SCRIPT_DIR)/python-tools.sh create-init -m . -e "$(PY_SED_EXTRACT_EXPORT)" \ + set -e -o pipefail; $(PY_GENERATE_INIT_PY) \ $(filter-out __init__.py,$(PY_ALL_PY)) $(SUBDIRS_TO_ITERATE) | $(PY_INIT_FILTER) | tee -a $@.tmp mv $@.tmp $@ endif diff --git a/scripts/python-tools.sh b/scripts/python-tools.sh index 4d2baa0e..975ed9ae 100644 --- a/scripts/python-tools.sh +++ b/scripts/python-tools.sh @@ -34,6 +34,12 @@ module_path() cmd_create_init() { + __add_seen() { + local type="$1" + [[ -n "${seen[$type]+x}" ]] && fatal "Duplicate symbol: $type" + seen["$type"]=1 + } + local import_submodules=0 local files="$*" local del="-------------------------- generated by $myname" @@ -42,26 +48,38 @@ cmd_create_init() echo "from pkgutil import extend_path" echo "" echo "__path__ = extend_path(__path__, __name__)" + echo local f local -A seen=() + local dst_type for f in $files; do test -d $f && continue local base=${f##*/} base=${base%.py} - if [ "$sed_extract_command" ]; then - local type types=`sed "$sed_extract_command" $f` - for type in $types; do - echo "from `module_path $base` import $type as $type" - [[ -n "${seen[$type]+x}" ]] && fatal "Duplicate symbol: $type" - seen["$type"]=1 + if [ "$sed_extract_cmd" ]; then + local src_type types=$(sed "$sed_extract_cmd" $f) + for src_type in $types; do + [ "$sed_symbol_filter_cmd" ] && dst_type=$(echo $base $src_type | sed "$sed_symbol_filter_cmd") + echo "from `module_path $base` import $src_type as $dst_type" + __add_seen $dst_type done fi done if [ "$import_submodules" = 1 ]; then for f in $files; do - [ -f $f/__init__.py ] && echo "import `module_path $f` as $f" + [ -f $f/__init__.py ] || continue + echo "import `module_path $f` as $f" + __add_seen $f done fi + + echo + echo "__all__ = [" + for dst_type in ${!seen[@]}; do + echo " \"$dst_type\"," + done + echo "]" + echo "# << $del <<" } @@ -69,11 +87,15 @@ cmd_create_init() myname=`basename $0` -eval set -- `getopt -o 'he:m:' "$@"` +eval set -- `getopt -l 'symbol-filter:' -o 'he:m:' "$@"` while [ "$1" != -- ]; do case $1 in -e) - sed_extract_command="$2" + sed_extract_cmd="$2" + shift + ;; + --symbol-filter) + sed_symbol_filter_cmd="$2" shift ;; -m) From 9039dc7e4044d5cf7849e7b0434e9d1561d06b01 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 12 Jun 2026 06:57:23 +0200 Subject: [PATCH 2/2] python-tools.sh, py-mod.mk: Use --symbol-filter Letting python-tools.sh rewrite symbols is more robust than rewriting an entire __init__.py with PY_INIT_FILTER in the including Makefile. The latter can break in non-obvious ways if python-tools.sh changes __init__.py's format. Make python-tools.sh support --symbol-filter to remedy that. The option takes an sed script which should expect a string of two non-whitespace tokens: The module from which the symbol is imported, and the name of the symbol in that module. It's output will then be used as the symbol to be exported from __init__.py. Also, support the PY_SYMBOL_FILTER variable in py-mod.mk. If it's defined, it is used for --symbol-filter. Signed-off-by: Jan Lindemann --- make/py-mod.mk | 6 +++++- scripts/python-tools.sh | 45 ++++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/make/py-mod.mk b/make/py-mod.mk index c7819114..882db095 100644 --- a/make/py-mod.mk +++ b/make/py-mod.mk @@ -6,6 +6,10 @@ include $(JWBDIR)/make/dev-utils.mk PY_INIT_TMPL = $(wildcard __init__.py.tmpl) PY_SED_EXTRACT_EXPORT ?= /\(\(class\|def\)\s\+[a-zA-Z_].*\|^ *\S\+\s*=.*\)\# *export/ !d; /^\s*\#/ d; s/\(async\)* *\(class\|def\) *//; s/[(:=].*// PY_INIT_FILTER ?= cat +PY_GENERATE_INIT_PY ?= /bin/bash $(JWB_SCRIPT_DIR)/python-tools.sh create-init -m . -e "$(PY_SED_EXTRACT_EXPORT)" +ifneq ($(PY_SYMBOL_FILTER),) + PY_GENERATE_INIT_PY += --symbol-filter "$(PY_SYMBOL_FILTER)" +endif #leftparen := ( #PY_EXPORT ?= $(shell sed '/\(class\|def\) ..*\# *export/ !d; s/\(class\|def\) *//; s/[$(leftparen):].*//' $(PY_SRC_PY)) @@ -20,7 +24,7 @@ include $(JWBDIR)/make/py-rules.mk ifeq ($(PY_UPDATE_INIT_PY),true) __init__.py: $(PY_INIT_TMPL) $(filter-out __init__.py,$(PY_SRC_PY)) if [ "$(PY_INIT_TMPL)" ]; then cat "$(PY_INIT_TMPL)" > $@.tmp; else > $@.tmp; fi - set -e -o pipefail; /bin/bash $(JWB_SCRIPT_DIR)/python-tools.sh create-init -m . -e "$(PY_SED_EXTRACT_EXPORT)" \ + set -e -o pipefail; $(PY_GENERATE_INIT_PY) \ $(filter-out __init__.py,$(PY_ALL_PY)) $(SUBDIRS_TO_ITERATE) | $(PY_INIT_FILTER) | tee -a $@.tmp mv $@.tmp $@ endif diff --git a/scripts/python-tools.sh b/scripts/python-tools.sh index 4d2baa0e..dc4c19c7 100644 --- a/scripts/python-tools.sh +++ b/scripts/python-tools.sh @@ -34,6 +34,12 @@ module_path() cmd_create_init() { + __add_seen() { + local type="$1" + [[ -n "${seen[$type]+x}" ]] && fatal "Duplicate symbol: $type" + seen["$type"]=1 + } + local import_submodules=0 local files="$*" local del="-------------------------- generated by $myname" @@ -42,26 +48,43 @@ cmd_create_init() echo "from pkgutil import extend_path" echo "" echo "__path__ = extend_path(__path__, __name__)" + echo local f local -A seen=() + local dst_type for f in $files; do test -d $f && continue local base=${f##*/} base=${base%.py} - if [ "$sed_extract_command" ]; then - local type types=`sed "$sed_extract_command" $f` - for type in $types; do - echo "from `module_path $base` import $type as $type" - [[ -n "${seen[$type]+x}" ]] && fatal "Duplicate symbol: $type" - seen["$type"]=1 + if [ "$sed_extract_cmd" ]; then + local src_type types=$(sed "$sed_extract_cmd" $f) + for src_type in $types; do + if [ -z "$sed_symbol_filter_cmd" ]; then + dst_type="$src_type" + else + dst_type=$(echo $base $src_type | sed "$sed_symbol_filter_cmd") + fi + [ "$sed_symbol_filter_cmd" ] && dst_type=$(echo $base $src_type | sed "$sed_symbol_filter_cmd") + echo "from `module_path $base` import $src_type as $dst_type" + __add_seen $dst_type done fi done if [ "$import_submodules" = 1 ]; then for f in $files; do - [ -f $f/__init__.py ] && echo "import `module_path $f` as $f" + [ -f $f/__init__.py ] || continue + echo "import `module_path $f` as $f" + __add_seen $f done fi + + echo + echo "__all__ = [" + for dst_type in ${!seen[@]}; do + echo " \"$dst_type\"," + done + echo "]" + echo "# << $del <<" } @@ -69,11 +92,15 @@ cmd_create_init() myname=`basename $0` -eval set -- `getopt -o 'he:m:' "$@"` +eval set -- `getopt -l 'symbol-filter:' -o 'he:m:' "$@"` while [ "$1" != -- ]; do case $1 in -e) - sed_extract_command="$2" + sed_extract_cmd="$2" + shift + ;; + --symbol-filter) + sed_symbol_filter_cmd="$2" shift ;; -m)