src/python/jw/pkg/cmds/posix/CmdCopy.py

@@ -33,6 +33,11 @@ class CmdCopy(Cmd):  # export
             action = 'store_true',
             help = 'Copy directories recursively',
         )
+        parser.add_argument(
+            '--parents',
+            action = 'store_true',
+            help = 'Land a single object at its source path below the destination root',
+        )
         parser.add_argument(
             '-F',
             '--fixed-strings',
@@ -61,4 +66,5 @@ class CmdCopy(Cmd):  # export
             # -- A mode is given as a string, if at all
             mode = int(args.mode, 0) if args.mode is not None else None,
             recursive = args.recursive,
+            parents = args.parents,
         )

src/python/jw/pkg/lib/FileContext.py

@@ -112,6 +112,10 @@ class FileContext(abc.ABC):
     def root(self) -> str:
         return self.__uri.path
 
+    @property
+    def chroot(self) -> bool:
+        return self.__chroot
+
     @property
     def username(self) -> str | None:
         return self.__uri.username
@@ -308,6 +312,30 @@ class FileContext(abc.ABC):
     async def is_dir(self, path: str, follow_symlinks: bool = True) -> bool:
         return await self._is_dir(self._chroot(path), follow_symlinks = follow_symlinks)
 
+    async def _is_gettable(self, path: str, follow_symlinks: bool) -> bool:
+        try:
+            return (await self._stat(path, follow_symlinks)).is_gettable
+        except NotImplementedError:
+            log(
+                DEBUG,
+                (
+                    f"{self.log_name} doesn't implement stat(), judging by "
+                    f'trailing slash if {path} is a file'
+                ),
+            )
+            return path[-1] != '/'
+        except FileNotFoundError as e:
+            log(DEBUG, f'{self.log_name}: Failed to stat({path}) ({str(e)})')
+            return False
+        except Exception as e:
+            log(ERR, f'{self.log_name}: Failed to stat({path}) ({str(e)})')
+            raise
+
+    async def is_gettable(self, path: str, follow_symlinks: bool = True) -> bool:
+        return await self._is_gettable(
+            self._chroot(path), follow_symlinks = follow_symlinks
+        )
+
     async def _readdir(self, path: str) -> Iterable[str]:
         raise NotImplementedError(
             f'{self.log_name}: readdir("{path}") is not implemented'

src/python/jw/pkg/lib/util.py

@@ -8,6 +8,7 @@ from enum import Enum, auto
 from typing import TYPE_CHECKING, Any, Iterable, TypeVar, cast
 
 from .base import Input, InputMode, Result
+from .FileContext import FileContext
 from .log import DEBUG, ERR, log
 from .Uri import Uri
 
@@ -15,7 +16,6 @@ if TYPE_CHECKING:
     from argparse import Namespace
 
     from .ExecContext import ExecContext
-    from .FileContext import FileContext
     from .ProcFilter import ProcFilter, ProcPipeline
 
 T = TypeVar('T')
@@ -169,18 +169,22 @@ async def get(
     return await run_pipeline(await ctx.get(uri.path, *args, **kwargs), content_filter)
 
 async def copy(
-    src_uri: str | Iterable[str],
+    src: str | FileContext | Iterable[str],
     dst: str | FileContext,
     owner: str | None = None,
     group: str | None = None,
     mode: int | None = None,
     throw: bool = True,
     recursive: bool = False,
+    parents: bool = False,
 ) -> Exception | str | list[str]:
-    if not isinstance(src_uri, str):
+
-        ret: list[str] = []
+    # Destination address / addresses
-        for uri in src_uri:  # TODO: Group identical netlocs into one CopyContext
+    ret: str | list[str]
-            rr = await copy(uri, dst, owner, group, mode, throw, recursive)
+    if not isinstance(src, (str, FileContext)):
+        ret = []
+        for uri in src:  # TODO: Group identical netlocs into one CopyContext
+            rr = await copy(uri, dst, owner, group, mode, throw, recursive, parents)
             if isinstance(rr, Exception):
                 return rr
             if isinstance(rr, list):
@@ -190,72 +194,91 @@ async def copy(
             else:
                 raise Exception(f'copy() returned unexpected type {type(rr)}')
         return ret
-    from .CopyContext import CopyContext
-
-    async with CopyContext(src_uri, dst) as ctx:
-
-        async def __mkdir_p(path: str) -> None:
-            # -- Create each level of the destination path that does not
-            #    exist yet; put() does not create parent directories
-            cur = ''
-            for part in path.strip('/').split('/'):
-                cur = f'{cur}/{part}'
-                if not await ctx.dst.is_dir(cur):
-                    await ctx.dst.mkdir(cur)
 
     async def __copy_tree(src_path: str, dst_path: str) -> list[str]:
         ret: list[str] = []
-            for entry in await ctx.src.readdir(src_path):
+        s_st = await ctx.src.stat(src_path)
-                s_path = f'{src_path}/{entry}'
+        if dst_path and not await ctx.dst.is_dir(dst_path):
-                d_path = f'{dst_path}/{entry}'
+            await ctx.dst.mkdir(dst_path, ftype = s_st.ftype)
-                if await ctx.src.is_dir(s_path):
+        for dirent in await ctx.src.readdir(src_path):
-                    await __mkdir_p(d_path)
+            s_path = f'{src_path}/{dirent}' if src_path else dirent
+            d_path = f'{dst_path}/{dirent}' if dst_path else dirent
+            s_st = await ctx.src.stat(s_path)
+            # -- Descend into directories that are not fetchable
+            #    whole; an entry, which is both, lands as a file
+            if s_st.is_dir and not s_st.is_gettable:
                 ret.extend(await __copy_tree(s_path, d_path))
-                else:
+                continue
             result = await ctx.src.get(s_path, throw = True)
-                    parent = d_path.rpartition('/')[0]
-                    if parent:
-                        await __mkdir_p(parent)
             await ctx.dst.put(
                 path = d_path,
                 content = result.stdout,
                 owner = owner,
                 group = group,
                 mode = mode,
+                ftype = s_st.ftype,
                 throw = True,
             )
             ret.append(d_path)
         return ret
 
     try:
-            if recursive and await ctx.src.is_dir(ctx.src.root):
+
-                # -- A directory source is mirrored below the destination
+        from .CopyContext import CopyContext
-                #    root: into it, if the root is an existing directory,
+
-                #    as it, if it does not exist yet
+        async with CopyContext(src, dst) as ctx:
-                dst_root = ctx.dst.root
+
-                if await ctx.dst.is_dir(dst_root):
+            src_uri = src.uri if isinstance(src, FileContext) else Uri(src)
-                    dst_root += '/' + os.path.basename(src_uri)
+            ret = '' if ctx.dst.chroot else ctx.dst.root
-                await __mkdir_p(dst_root)
+            if parents:
-                return await __copy_tree(ctx.src.root, dst_root)
+                # -- cp --parents: the source lands at its source path
-            result = await ctx.src.get(ctx.src.root, throw = True)
+                #    below the destination root, each parent level
-            dst_path = ctx.dst.root
+                #    typed by the source-side node it mirrors
-            if await ctx.dst.is_dir(ctx.dst.root):
+                parts = [p for p in src_uri.path.split('/') if p]
-                dst_path += '/' + os.path.basename(src_uri)
+                if parts:
+                    src_path = '/'.join(parts)
+                    base = ret
+                    ret = f'{base}/{src_path}' if base else src_path
+                    s_cur = ''
+                    for part in parts[:-1]:
+                        s_cur = f'{s_cur}/{part}' if s_cur else part
+                        d_cur = f'{base}/{s_cur}' if base else s_cur
+                        if await ctx.dst.is_dir(d_cur):
+                            continue
+                        level_ctx = FileContext.create(
+                            f'{ctx.src.uri.scheme_plus_authority}/{s_cur}',
+                            chroot = True,
+                        )
+                        async with level_ctx:
+                            ftype = (await level_ctx.stat('')).ftype
+                        await ctx.dst.mkdir(d_cur, ftype = ftype)
+            elif ret:
+                # Land the source's basename within existing destination directories
+                basename = os.path.basename(src_uri.path.rstrip('/'))
+                if basename and await ctx.dst.is_dir(ret):
+                    ret += '/' + basename
+            src_root = '' if ctx.src.chroot else ctx.src.root
+            s_st = await ctx.src.stat(src_root)
+            if recursive and s_st.is_dir and not s_st.is_gettable:
+                return await __copy_tree(src_root, ret)
+
+            result = await ctx.src.get(src_root, throw = True)
             await ctx.dst.put(
-                path = dst_path,
+                path = ret,
                 content = result.stdout,
                 owner = owner,
                 group = group,
                 mode = mode,
+                ftype = s_st.ftype,
                 throw = True,
             )
-            return dst_path
+            return ret
+
     except Exception as e:
         if throw:
             raise
-            log(ERR, f'Failed to copy {src_uri} -> {dst} ({str(e)})')
+        log(ERR, f'Failed to copy {src} -> {dst} ({str(e)})')
         return e
-    assert False, 'Unreachable code'
 
 async def get_username(  # export
     args: Namespace | None = None,

test/integration/jw_pkg/help/test-expected.txt

@@ -186,7 +186,8 @@ Available subcommands of posix:
     copy      Copy files
     tar       Handle tar archives
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help
-usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F]
+usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r]
+                            [--parents] [-F]
                             src dst
 
 Copy files
@@ -201,6 +202,8 @@ options:
   -g, --group GROUP    Destination file group (default: None)
   -m, --mode MODE      Destination file mode (default: None)
   -r, --recursive      Copy directories recursively (default: False)
+  --parents            Land a single object at its source path below the
+                       destination root (default: False)
   -F, --fixed-strings  Don't expand macros in <src> and <dst> (default: False)
 ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help
 usage: jw-pkg.py posix tar [-h]  ...

test/unit/python/jw/pkg/lib/util/test.py

@@ -3,6 +3,7 @@ import os
 import stat
 import tempfile
 
+from jw.pkg.lib.ec.Local import Local
 from jw.pkg.lib.util import copy
 
 def uri(path: str) -> str:
@@ -99,6 +100,71 @@ async def run_tests() -> None:
         st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3'))
         assert stat.S_IMODE(st.st_mode) == 0o640, st
 
+        # -- A chrooted context source: the empty address is the
+        #    context's root, and landing into an existing directory
+        #    takes the root's basename
+        os.makedirs(os.path.join(tmp, 'dst/chroot'))
+        src_fc = Local(uri(os.path.join(tmp, 'src/a')), chroot = True)
+        ret = await copy(src_fc, uri(os.path.join(tmp, 'dst/chroot')), recursive = True)
+        assert os.path.isfile(os.path.join(tmp, 'dst/chroot/a/f1')), ret
+        assert os.path.isfile(os.path.join(tmp, 'dst/chroot/a/sub/f2')), ret
+
+        # -- A trailing slash does not change the source's name
+        os.makedirs(os.path.join(tmp, 'dst/slash'))
+        ret = await copy(
+            uri(os.path.join(tmp, 'src/a/')),
+            uri(os.path.join(tmp, 'dst/slash')),
+            recursive = True,
+        )
+        assert isinstance(ret, list), ret
+        assert os.path.join(tmp, 'dst/slash/a/f1') in ret, ret
+        src_fc = Local(uri(os.path.join(tmp, 'src/a/')), chroot = True)
+        ret = await copy(src_fc, uri(os.path.join(tmp, 'dst/slash')), recursive = True)
+        assert isinstance(ret, list), ret
+        assert os.path.join(tmp, 'dst/slash/a/f1') in ret, ret
+
+        # -- parents lands the object at its source path below the
+        #    destination root, like cp --parents
+        dst_parents = os.path.join(tmp, 'dst/parents')
+        os.makedirs(dst_parents)
+        src_single = os.path.join(tmp, 'src/single')
+        ret = await copy(uri(src_single), uri(dst_parents), parents = True)
+        assert ret == dst_parents + src_single, ret
+        with open(dst_parents + src_single) as f:
+            assert f.read() == 'solo\n'
+
+        # -- parents on a tree lands the tree at its source path
+        dst_ptree = os.path.join(tmp, 'dst/ptree')
+        os.makedirs(dst_ptree)
+        src_tree = os.path.join(tmp, 'src/a')
+        ret = await copy(
+            uri(src_tree), uri(dst_ptree), recursive = True, parents = True
+        )
+        assert isinstance(ret, list), ret
+        expected = dst_ptree + src_tree
+        assert sorted(ret) == sorted([
+            os.path.join(expected, 'f1'),
+            os.path.join(expected, 'sub/f2'),
+            os.path.join(expected, 'sub/subsub/f3'),
+        ]), ret
+        assert os.path.isdir(os.path.join(expected, 'empty')), ret
+
+        # -- a chrooted context source lands at its URI path
+        dst_pchroot = os.path.join(tmp, 'dst/pchroot')
+        os.makedirs(dst_pchroot)
+        src_fc = Local(uri(src_tree), chroot = True)
+        ret = await copy(src_fc, uri(dst_pchroot), recursive = True, parents = True)
+        assert isinstance(ret, list), ret
+        assert os.path.isfile(os.path.join(dst_pchroot + src_tree, 'f1')), ret
+
+        # -- re-copying with parents replaces content, not layout
+        with open(dst_parents + src_single, 'w') as f:
+            f.write('replaced\n')
+        ret = await copy(uri(src_single), uri(dst_parents), parents = True)
+        assert ret == dst_parents + src_single, ret
+        with open(dst_parents + src_single) as f:
+            assert f.read() == 'solo\n'
+
         # -- A missing source fails loudly
         try:
             await copy(