From b697b7b754319d85d4177f1ae91120043790ce0a Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 29 Sep 2026 14:36:02 +0200 Subject: [PATCH] util.copy(): Add recursive directory copy copy() copies a single file: it fetches the source root as one content and puts it at the destination, so addressing a directory source fails, and there is no way to mirror a tree between file contexts. Add a recursive mode: - a directory source is mirrored below the destination root, into it if the root is an existing directory, as it if it does not exist yet - entries are enumerated with readdir(), destination directories are created level by level, and every leaf is put with the requested owner, group, and mode - a file source is unaffected, and the destination parent must exist, as before posix copy exposes the mode as -r/--recursive; the help test reference picks up the new option. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.87.1 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/cmds/posix/CmdCopy.py | 7 ++ src/python/jw/pkg/lib/util.py | 46 ++++++- .../integration/jw-pkg/help/test-expected.txt | 4 +- test/unit/python/jw/pkg/lib/util/Makefile | 7 ++ test/unit/python/jw/pkg/lib/util/test.py | 115 ++++++++++++++++++ 5 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 test/unit/python/jw/pkg/lib/util/Makefile create mode 100644 test/unit/python/jw/pkg/lib/util/test.py diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index 2dc600ec..cd9b7dab 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -27,6 +27,12 @@ class CmdCopy(Cmd): # export parser.add_argument( '-m', '--mode', default = None, help = 'Destination file mode' ) + parser.add_argument( + '-r', + '--recursive', + action = 'store_true', + help = 'Copy directories recursively', + ) parser.add_argument( '-F', '--fixed-strings', @@ -53,4 +59,5 @@ class CmdCopy(Cmd): # export owner = args.owner, group = args.group, mode = int(args.mode, 0), + recursive = args.recursive, ) diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 8033902c..4e97cff4 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -174,11 +174,12 @@ async def copy( group: str | None = None, mode: int | None = None, throw: bool = True, + recursive: bool = False, ) -> Exception | str | list[str]: if not isinstance(src_uri, str): ret: list[str] = [] for uri in src_uri: # TODO: Group identical netlocs into one CopyContext - rr = await copy(uri, dst, owner, group, mode, throw) + rr = await copy(uri, dst, owner, group, mode, throw, recursive) if isinstance(rr, Exception): return rr if isinstance(rr, list): @@ -191,7 +192,50 @@ async def copy( from .CopyContext import CopyContext async with CopyContext(src_uri, dst) as ctx: + + async def __mkdir_p(path: str) -> None: + # -- Create each level of the destination path that does not + # exist yet; put() does not create parent directories + cur = '' + for part in path.strip('/').split('/'): + cur = f'{cur}/{part}' + if not await ctx.dst.is_dir(cur): + await ctx.dst.mkdir(cur) + + async def __copy_tree(src_path: str, dst_path: str) -> list[str]: + ret: list[str] = [] + for entry in await ctx.src.readdir(src_path): + s_path = f'{src_path}/{entry}' + d_path = f'{dst_path}/{entry}' + if await ctx.src.is_dir(s_path): + await __mkdir_p(d_path) + ret.extend(await __copy_tree(s_path, d_path)) + else: + result = await ctx.src.get(s_path, throw = True) + parent = d_path.rpartition('/')[0] + if parent: + await __mkdir_p(parent) + await ctx.dst.put( + path = d_path, + content = result.stdout, + owner = owner, + group = group, + mode = mode, + throw = True, + ) + ret.append(d_path) + return ret + try: + if recursive and await ctx.src.is_dir(ctx.src.root): + # -- A directory source is mirrored below the destination + # root: into it, if the root is an existing directory, + # as it, if it does not exist yet + dst_root = ctx.dst.root + if await ctx.dst.is_dir(dst_root): + dst_root += '/' + os.path.basename(src_uri) + await __mkdir_p(dst_root) + return await __copy_tree(ctx.src.root, dst_root) result = await ctx.src.get(ctx.src.root, throw = True) dst_path = ctx.dst.root if await ctx.dst.is_dir(ctx.dst.root): diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 2ab2fac0..9dd91555 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -176,7 +176,8 @@ Available subcommands of posix: copy Copy files tar Handle tar archives ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help -usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F] + src dst Copy files @@ -189,6 +190,7 @@ options: -o, --owner OWNER Destination file owner (default: None) -g, --group GROUP Destination file group (default: None) -m, --mode MODE Destination file mode (default: None) + -r, --recursive Copy directories recursively (default: False) -F, --fixed-strings Don't expand macros in and (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help usage: jw-pkg.py posix tar [-h] ... diff --git a/test/unit/python/jw/pkg/lib/util/Makefile b/test/unit/python/jw/pkg/lib/util/Makefile new file mode 100644 index 00000000..de693650 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/util/test.py b/test/unit/python/jw/pkg/lib/util/test.py new file mode 100644 index 00000000..001b9987 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/test.py @@ -0,0 +1,115 @@ +import asyncio +import os +import stat +import tempfile + +from jw.pkg.lib.util import copy + +def uri(path: str) -> str: + return 'local://localhost' + path + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + # -- Source tree + # + # src/a/f1 + # src/a/sub/f2 + # src/a/sub/subsub/f3 + # src/a/empty + # src/single + for rel, content in [ + ('src/a/f1', 'one\n'), + ('src/a/sub/f2', 'two\n'), + ('src/a/sub/subsub/f3', 'three\n'), + ('src/single', 'solo\n'), + ]: + path = os.path.join(tmp, rel) + os.makedirs(os.path.dirname(path), exist_ok = True) + with open(path, 'w') as f: + f.write(content) + os.makedirs(os.path.join(tmp, 'src/a/empty')) + + # -- A single file to a nonexistent destination path; like cp, + # the destination's parent must exist + os.makedirs(os.path.join(tmp, 'dst')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/single')) + ) + assert ret == os.path.join(tmp, 'dst/single'), ret + with open(os.path.join(tmp, 'dst/single')) as f: + assert f.read() == 'solo\n' + + # -- A single file into an existing directory gets the basename + os.makedirs(os.path.join(tmp, 'dst/dir')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/dir')) + ) + assert ret == os.path.join(tmp, 'dst/dir/single'), ret + with open(os.path.join(tmp, 'dst/dir/single')) as f: + assert f.read() == 'solo\n' + + # -- A file with recursive=True behaves like a plain file copy + ret = await copy( + uri(os.path.join(tmp, 'src/single')), + uri(os.path.join(tmp, 'dst/recursive-file')), + recursive = True, + ) + assert ret == os.path.join(tmp, 'dst/recursive-file'), ret + + # -- A tree to a nonexistent destination: the destination becomes + # the tree, parent directories are created, empty directories + # survive, and the returned paths are the copied leaves + dst_tree = os.path.join(tmp, 'dst', 'tree') + ret = await copy( + uri(os.path.join(tmp, 'src/a')), uri(dst_tree), recursive = True + ) + assert isinstance(ret, list), ret + assert sorted(ret) == sorted([ + os.path.join(dst_tree, 'f1'), + os.path.join(dst_tree, 'sub/f2'), + os.path.join(dst_tree, 'sub/subsub/f3'), + ]), ret + for rel, content in [ + ('f1', 'one\n'), + ('sub/f2', 'two\n'), + ('sub/subsub/f3', 'three\n'), + ]: + with open(os.path.join(dst_tree, rel)) as f: + assert f.read() == content, rel + assert os.path.isdir(os.path.join(dst_tree, 'empty')) + + # -- A tree into an existing directory is copied into it under + # the source basename + os.makedirs(os.path.join(tmp, 'dst', 'into')) + ret = await copy( + uri(os.path.join(tmp, 'src/a')), + uri(os.path.join(tmp, 'dst/into')), + recursive = True, + ) + assert os.path.isfile(os.path.join(tmp, 'dst/into/a/sub/f2')), ret + + # -- A mode applies to every copied leaf + dst_mode = os.path.join(tmp, 'dst', 'mode') + await copy( + uri(os.path.join(tmp, 'src/a')), + uri(dst_mode), + mode = 0o640, + recursive = True + ) + st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3')) + assert stat.S_IMODE(st.st_mode) == 0o640, st + + # -- A missing source fails loudly + try: + await copy( + uri(os.path.join(tmp, 'src/missing')), uri(os.path.join(tmp, 'dst/x')) + ) + raise AssertionError('expected an exception') + except Exception: + pass + +def main() -> None: + asyncio.run(run_tests()) + print('All util copy tests passed') + +main()