util.copy(): Add recursive directory copy

copy() copies a single file: it fetches the source root as one content and
puts it at the destination, so addressing a directory source fails, and
there is no way to mirror a tree between file contexts.

Add a recursive mode:

- a directory source is mirrored below the destination root, into it if the
  root is an existing directory, as it if it does not exist yet
- entries are enumerated with readdir(), destination directories are
  created level by level, and every leaf is put with the requested owner,
  group, and mode
- a file source is unaffected, and the destination parent must exist, as
  before

posix copy exposes the mode as -r/--recursive; the help test reference
picks up the new option.

Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.87.1
Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/jw/pkg/cmds/posix/CmdCopy.py

@@ -27,6 +27,12 @@ class CmdCopy(Cmd):  # export
         parser.add_argument(
             '-m', '--mode', default = None, help = 'Destination file mode'
         )
+        parser.add_argument(
+            '-r',
+            '--recursive',
+            action = 'store_true',
+            help = 'Copy directories recursively',
+        )
         parser.add_argument(
             '-F',
             '--fixed-strings',
@@ -53,4 +59,5 @@ class CmdCopy(Cmd):  # export
             owner = args.owner,
             group = args.group,
             mode = int(args.mode, 0),
+            recursive = args.recursive,
         )

src/python/jw/pkg/lib/util.py

@@ -174,11 +174,12 @@ async def copy(
     group: str | None = None,
     mode: int | None = None,
     throw: bool = True,
+    recursive: bool = False,
 ) -> Exception | str | list[str]:
     if not isinstance(src_uri, str):
         ret: list[str] = []
         for uri in src_uri:  # TODO: Group identical netlocs into one CopyContext
-            rr = await copy(uri, dst, owner, group, mode, throw)
+            rr = await copy(uri, dst, owner, group, mode, throw, recursive)
             if isinstance(rr, Exception):
                 return rr
             if isinstance(rr, list):
@@ -191,7 +192,50 @@ async def copy(
     from .CopyContext import CopyContext
 
     async with CopyContext(src_uri, dst) as ctx:
+
+        async def __mkdir_p(path: str) -> None:
+            # -- Create each level of the destination path that does not
+            #    exist yet; put() does not create parent directories
+            cur = ''
+            for part in path.strip('/').split('/'):
+                cur = f'{cur}/{part}'
+                if not await ctx.dst.is_dir(cur):
+                    await ctx.dst.mkdir(cur)
+
+        async def __copy_tree(src_path: str, dst_path: str) -> list[str]:
+            ret: list[str] = []
+            for entry in await ctx.src.readdir(src_path):
+                s_path = f'{src_path}/{entry}'
+                d_path = f'{dst_path}/{entry}'
+                if await ctx.src.is_dir(s_path):
+                    await __mkdir_p(d_path)
+                    ret.extend(await __copy_tree(s_path, d_path))
+                else:
+                    result = await ctx.src.get(s_path, throw = True)
+                    parent = d_path.rpartition('/')[0]
+                    if parent:
+                        await __mkdir_p(parent)
+                    await ctx.dst.put(
+                        path = d_path,
+                        content = result.stdout,
+                        owner = owner,
+                        group = group,
+                        mode = mode,
+                        throw = True,
+                    )
+                    ret.append(d_path)
+            return ret
+
         try:
+            if recursive and await ctx.src.is_dir(ctx.src.root):
+                # -- A directory source is mirrored below the destination
+                #    root: into it, if the root is an existing directory,
+                #    as it, if it does not exist yet
+                dst_root = ctx.dst.root
+                if await ctx.dst.is_dir(dst_root):
+                    dst_root += '/' + os.path.basename(src_uri)
+                await __mkdir_p(dst_root)
+                return await __copy_tree(ctx.src.root, dst_root)
             result = await ctx.src.get(ctx.src.root, throw = True)
             dst_path = ctx.dst.root
             if await ctx.dst.is_dir(ctx.dst.root):

test/integration/jw-pkg/help/test-expected.txt

@@ -176,7 +176,8 @@ Available subcommands of posix:
     copy      Copy files
     tar       Handle tar archives
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help
-usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst
+usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F]
+                            src dst
 
 Copy files
 
@@ -189,6 +190,7 @@ options:
   -o, --owner OWNER    Destination file owner (default: None)
   -g, --group GROUP    Destination file group (default: None)
   -m, --mode MODE      Destination file mode (default: None)
+  -r, --recursive      Copy directories recursively (default: False)
   -F, --fixed-strings  Don't expand macros in <src> and <dst> (default: False)
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help
 usage: jw-pkg.py posix tar [-h]  ...

test/unit/python/jw/pkg/lib/util/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/util/test.py

@@ -0,0 +1,115 @@
+import asyncio
+import os
+import stat
+import tempfile
+
+from jw.pkg.lib.util import copy
+
+def uri(path: str) -> str:
+    return 'local://localhost' + path
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        # -- Source tree
+        #
+        #   src/a/f1
+        #   src/a/sub/f2
+        #   src/a/sub/subsub/f3
+        #   src/a/empty
+        #   src/single
+        for rel, content in [
+            ('src/a/f1', 'one\n'),
+            ('src/a/sub/f2', 'two\n'),
+            ('src/a/sub/subsub/f3', 'three\n'),
+            ('src/single', 'solo\n'),
+        ]:
+            path = os.path.join(tmp, rel)
+            os.makedirs(os.path.dirname(path), exist_ok = True)
+            with open(path, 'w') as f:
+                f.write(content)
+        os.makedirs(os.path.join(tmp, 'src/a/empty'))
+
+        # -- A single file to a nonexistent destination path; like cp,
+        #    the destination's parent must exist
+        os.makedirs(os.path.join(tmp, 'dst'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/single'))
+        )
+        assert ret == os.path.join(tmp, 'dst/single'), ret
+        with open(os.path.join(tmp, 'dst/single')) as f:
+            assert f.read() == 'solo\n'
+
+        # -- A single file into an existing directory gets the basename
+        os.makedirs(os.path.join(tmp, 'dst/dir'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/dir'))
+        )
+        assert ret == os.path.join(tmp, 'dst/dir/single'), ret
+        with open(os.path.join(tmp, 'dst/dir/single')) as f:
+            assert f.read() == 'solo\n'
+
+        # -- A file with recursive=True behaves like a plain file copy
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')),
+            uri(os.path.join(tmp, 'dst/recursive-file')),
+            recursive = True,
+        )
+        assert ret == os.path.join(tmp, 'dst/recursive-file'), ret
+
+        # -- A tree to a nonexistent destination: the destination becomes
+        #    the tree, parent directories are created, empty directories
+        #    survive, and the returned paths are the copied leaves
+        dst_tree = os.path.join(tmp, 'dst', 'tree')
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/a')), uri(dst_tree), recursive = True
+        )
+        assert isinstance(ret, list), ret
+        assert sorted(ret) == sorted([
+            os.path.join(dst_tree, 'f1'),
+            os.path.join(dst_tree, 'sub/f2'),
+            os.path.join(dst_tree, 'sub/subsub/f3'),
+        ]), ret
+        for rel, content in [
+            ('f1', 'one\n'),
+            ('sub/f2', 'two\n'),
+            ('sub/subsub/f3', 'three\n'),
+        ]:
+            with open(os.path.join(dst_tree, rel)) as f:
+                assert f.read() == content, rel
+        assert os.path.isdir(os.path.join(dst_tree, 'empty'))
+
+        # -- A tree into an existing directory is copied into it under
+        #    the source basename
+        os.makedirs(os.path.join(tmp, 'dst', 'into'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/a')),
+            uri(os.path.join(tmp, 'dst/into')),
+            recursive = True,
+        )
+        assert os.path.isfile(os.path.join(tmp, 'dst/into/a/sub/f2')), ret
+
+        # -- A mode applies to every copied leaf
+        dst_mode = os.path.join(tmp, 'dst', 'mode')
+        await copy(
+            uri(os.path.join(tmp, 'src/a')),
+            uri(dst_mode),
+            mode = 0o640,
+            recursive = True
+        )
+        st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3'))
+        assert stat.S_IMODE(st.st_mode) == 0o640, st
+
+        # -- A missing source fails loudly
+        try:
+            await copy(
+                uri(os.path.join(tmp, 'src/missing')), uri(os.path.join(tmp, 'dst/x'))
+            )
+            raise AssertionError('expected an exception')
+        except Exception:
+            pass
+
+def main() -> None:
+    asyncio.run(run_tests())
+    print('All util copy tests passed')
+
+main()