src/python/jw/pkg/cmds/posix/CmdCopy.py

@@ -27,6 +27,12 @@ class CmdCopy(Cmd):  # export
         parser.add_argument(
             '-m', '--mode', default = None, help = 'Destination file mode'
         )
+        parser.add_argument(
+            '-r',
+            '--recursive',
+            action = 'store_true',
+            help = 'Copy directories recursively',
+        )
         parser.add_argument(
             '-F',
             '--fixed-strings',
@@ -53,4 +59,5 @@ class CmdCopy(Cmd):  # export
             owner = args.owner,
             group = args.group,
             mode = int(args.mode, 0),
+            recursive = args.recursive,
         )

src/python/jw/pkg/lib/ExecContext.py

@@ -5,7 +5,7 @@ import errno
 import sys
 
 from decimal import ROUND_FLOOR, Decimal
-from typing import TYPE_CHECKING, Any, NamedTuple, override
+from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, override
 
 if TYPE_CHECKING:
     from typing import Type
@@ -688,3 +688,10 @@ class ExecContext(Base):
             ['chmod', self.__mode_str(mode), path],
             cmd_input = InputMode.NonInteractive
         )
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        result = await self.run(
+            ['ls', '-1A', path], cmd_input = InputMode.NonInteractive
+        )
+        return result.stdout_str.splitlines()

src/python/jw/pkg/lib/FileContext.py

@@ -4,7 +4,7 @@ import abc
 
 from enum import Enum, auto
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, override
+from typing import TYPE_CHECKING, Any, Iterable, override
 
 from .log import DEBUG, ERR, log
 from .ProcFilter import ProcPipeline
@@ -301,6 +301,14 @@ class FileContext(abc.ABC):
     async def is_dir(self, path: str, follow_symlinks: bool = True) -> bool:
         return await self._is_dir(self._chroot(path), follow_symlinks = follow_symlinks)
 
+    async def _readdir(self, path: str) -> Iterable[str]:
+        raise NotImplementedError(
+            f'{self.log_name}: readdir("{path}") is not implemented'
+        )
+
+    async def readdir(self, path: str) -> Iterable[str]:
+        return await self._readdir(self._chroot(path))
+
     @classmethod
     def create(cls, uri: str | Uri, *args: Any, **kwargs: Any) -> 'FileContext':
         uri = Uri.pimp(uri)

src/python/jw/pkg/lib/ec/Curl.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, override
+import errno
+
+from typing import TYPE_CHECKING, Any, Iterable, override
 
 from ..FileContext import FileContext as Base
 
@@ -48,3 +50,8 @@ class Curl(Base):
             path = '/' + path
         cmd.append(self.uri.to_string + self._chroot(path))
         return await self.__ec.run(cmd, throw = throw, verbose = verbose)
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        msg = f'{self.log_name}: readdir("{path}") is not implemented'
+        raise OSError(errno.ENOSYS, msg)

src/python/jw/pkg/lib/ec/Local.py

@@ -7,7 +7,7 @@ import pwd
 import sys
 
 from functools import cache
-from typing import TYPE_CHECKING, override
+from typing import TYPE_CHECKING, Iterable, override
 
 from ..base import Result, StatResult
 from ..ExecContext import ExecContext as Base
@@ -206,3 +206,7 @@ class Local(Base):
         if (not follow_symlinks) and os.path.islink(path):
             return False
         return os.path.isdir(path)
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        return os.listdir(path)

src/python/jw/pkg/lib/util.py

@@ -174,11 +174,12 @@ async def copy(
     group: str | None = None,
     mode: int | None = None,
     throw: bool = True,
+    recursive: bool = False,
 ) -> Exception | str | list[str]:
     if not isinstance(src_uri, str):
         ret: list[str] = []
         for uri in src_uri:  # TODO: Group identical netlocs into one CopyContext
-            rr = await copy(uri, dst, owner, group, mode, throw)
+            rr = await copy(uri, dst, owner, group, mode, throw, recursive)
             if isinstance(rr, Exception):
                 return rr
             if isinstance(rr, list):
@@ -191,7 +192,50 @@ async def copy(
     from .CopyContext import CopyContext
 
     async with CopyContext(src_uri, dst) as ctx:
+
+        async def __mkdir_p(path: str) -> None:
+            # -- Create each level of the destination path that does not
+            #    exist yet; put() does not create parent directories
+            cur = ''
+            for part in path.strip('/').split('/'):
+                cur = f'{cur}/{part}'
+                if not await ctx.dst.is_dir(cur):
+                    await ctx.dst.mkdir(cur)
+
+        async def __copy_tree(src_path: str, dst_path: str) -> list[str]:
+            ret: list[str] = []
+            for entry in await ctx.src.readdir(src_path):
+                s_path = f'{src_path}/{entry}'
+                d_path = f'{dst_path}/{entry}'
+                if await ctx.src.is_dir(s_path):
+                    await __mkdir_p(d_path)
+                    ret.extend(await __copy_tree(s_path, d_path))
+                else:
+                    result = await ctx.src.get(s_path, throw = True)
+                    parent = d_path.rpartition('/')[0]
+                    if parent:
+                        await __mkdir_p(parent)
+                    await ctx.dst.put(
+                        path = d_path,
+                        content = result.stdout,
+                        owner = owner,
+                        group = group,
+                        mode = mode,
+                        throw = True,
+                    )
+                    ret.append(d_path)
+            return ret
+
         try:
+            if recursive and await ctx.src.is_dir(ctx.src.root):
+                # -- A directory source is mirrored below the destination
+                #    root: into it, if the root is an existing directory,
+                #    as it, if it does not exist yet
+                dst_root = ctx.dst.root
+                if await ctx.dst.is_dir(dst_root):
+                    dst_root += '/' + os.path.basename(src_uri)
+                await __mkdir_p(dst_root)
+                return await __copy_tree(ctx.src.root, dst_root)
             result = await ctx.src.get(ctx.src.root, throw = True)
             dst_path = ctx.dst.root
             if await ctx.dst.is_dir(ctx.dst.root):

test/integration/jw-pkg/help/test-expected.txt

@@ -176,7 +176,8 @@ Available subcommands of posix:
     copy      Copy files
     tar       Handle tar archives
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help
-usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst
+usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F]
+                            src dst
 
 Copy files
 
@@ -189,6 +190,7 @@ options:
   -o, --owner OWNER    Destination file owner (default: None)
   -g, --group GROUP    Destination file group (default: None)
   -m, --mode MODE      Destination file mode (default: None)
+  -r, --recursive      Copy directories recursively (default: False)
   -F, --fixed-strings  Don't expand macros in <src> and <dst> (default: False)
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help
 usage: jw-pkg.py posix tar [-h]  ...

test/unit/python/jw/pkg/lib/ec/Curl/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/Curl/test.py

@@ -0,0 +1,15 @@
+import asyncio
+import errno
+
+from jw.pkg.lib.ec.Curl import Curl
+
+async def run_tests() -> None:
+    curl = Curl('https://example.com/')
+    try:
+        await curl.readdir('some/path')
+        raise AssertionError('expected OSError with errno ENOSYS')
+    except OSError as e:
+        assert e.errno == errno.ENOSYS, e.errno
+
+asyncio.run(run_tests())
+print('All Curl readdir tests passed')

test/unit/python/jw/pkg/lib/ec/Local/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/Local/test.py

@@ -0,0 +1,33 @@
+import asyncio
+import os
+import tempfile
+
+from jw.pkg.lib.ec.Local import Local
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        for name in ['a', 'b', '.hidden']:
+            with open(os.path.join(tmp, name), 'w') as f:
+                f.write(name)
+        os.mkdir(os.path.join(tmp, 'subdir'))
+
+        local = Local()
+        entries = sorted(await local.readdir(tmp))
+        assert entries == ['.hidden', 'a', 'b', 'subdir'], entries
+
+        # -- readdir() of a missing directory raises FileNotFoundError
+        try:
+            await local.readdir(os.path.join(tmp, 'missing'))
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+
+        # -- mkdir() in a chrooted context creates below the context root
+        root = os.path.join(tmp, 'root')
+        os.mkdir(root)
+        chrooted = Local('local://localhost' + root, chroot = True)
+        await chrooted.mkdir('sub')
+        assert os.path.isdir(os.path.join(root, 'sub'))
+
+asyncio.run(run_tests())
+print('All Local readdir tests passed')

test/unit/python/jw/pkg/lib/util/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/util/test.py

@@ -0,0 +1,115 @@
+import asyncio
+import os
+import stat
+import tempfile
+
+from jw.pkg.lib.util import copy
+
+def uri(path: str) -> str:
+    return 'local://localhost' + path
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        # -- Source tree
+        #
+        #   src/a/f1
+        #   src/a/sub/f2
+        #   src/a/sub/subsub/f3
+        #   src/a/empty
+        #   src/single
+        for rel, content in [
+            ('src/a/f1', 'one\n'),
+            ('src/a/sub/f2', 'two\n'),
+            ('src/a/sub/subsub/f3', 'three\n'),
+            ('src/single', 'solo\n'),
+        ]:
+            path = os.path.join(tmp, rel)
+            os.makedirs(os.path.dirname(path), exist_ok = True)
+            with open(path, 'w') as f:
+                f.write(content)
+        os.makedirs(os.path.join(tmp, 'src/a/empty'))
+
+        # -- A single file to a nonexistent destination path; like cp,
+        #    the destination's parent must exist
+        os.makedirs(os.path.join(tmp, 'dst'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/single'))
+        )
+        assert ret == os.path.join(tmp, 'dst/single'), ret
+        with open(os.path.join(tmp, 'dst/single')) as f:
+            assert f.read() == 'solo\n'
+
+        # -- A single file into an existing directory gets the basename
+        os.makedirs(os.path.join(tmp, 'dst/dir'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/dir'))
+        )
+        assert ret == os.path.join(tmp, 'dst/dir/single'), ret
+        with open(os.path.join(tmp, 'dst/dir/single')) as f:
+            assert f.read() == 'solo\n'
+
+        # -- A file with recursive=True behaves like a plain file copy
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/single')),
+            uri(os.path.join(tmp, 'dst/recursive-file')),
+            recursive = True,
+        )
+        assert ret == os.path.join(tmp, 'dst/recursive-file'), ret
+
+        # -- A tree to a nonexistent destination: the destination becomes
+        #    the tree, parent directories are created, empty directories
+        #    survive, and the returned paths are the copied leaves
+        dst_tree = os.path.join(tmp, 'dst', 'tree')
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/a')), uri(dst_tree), recursive = True
+        )
+        assert isinstance(ret, list), ret
+        assert sorted(ret) == sorted([
+            os.path.join(dst_tree, 'f1'),
+            os.path.join(dst_tree, 'sub/f2'),
+            os.path.join(dst_tree, 'sub/subsub/f3'),
+        ]), ret
+        for rel, content in [
+            ('f1', 'one\n'),
+            ('sub/f2', 'two\n'),
+            ('sub/subsub/f3', 'three\n'),
+        ]:
+            with open(os.path.join(dst_tree, rel)) as f:
+                assert f.read() == content, rel
+        assert os.path.isdir(os.path.join(dst_tree, 'empty'))
+
+        # -- A tree into an existing directory is copied into it under
+        #    the source basename
+        os.makedirs(os.path.join(tmp, 'dst', 'into'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/a')),
+            uri(os.path.join(tmp, 'dst/into')),
+            recursive = True,
+        )
+        assert os.path.isfile(os.path.join(tmp, 'dst/into/a/sub/f2')), ret
+
+        # -- A mode applies to every copied leaf
+        dst_mode = os.path.join(tmp, 'dst', 'mode')
+        await copy(
+            uri(os.path.join(tmp, 'src/a')),
+            uri(dst_mode),
+            mode = 0o640,
+            recursive = True
+        )
+        st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3'))
+        assert stat.S_IMODE(st.st_mode) == 0o640, st
+
+        # -- A missing source fails loudly
+        try:
+            await copy(
+                uri(os.path.join(tmp, 'src/missing')), uri(os.path.join(tmp, 'dst/x'))
+            )
+            raise AssertionError('expected an exception')
+        except Exception:
+            pass
+
+def main() -> None:
+    asyncio.run(run_tests())
+    print('All util copy tests passed')
+
+main()