diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index aa578b64..96014c79 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -33,6 +33,11 @@ class CmdCopy(Cmd): # export action = 'store_true', help = 'Copy directories recursively', ) + parser.add_argument( + '--parents', + action = 'store_true', + help = 'Land a single object at its source path below the destination root', + ) parser.add_argument( '-F', '--fixed-strings', @@ -61,4 +66,5 @@ class CmdCopy(Cmd): # export # -- A mode is given as a string, if at all mode = int(args.mode, 0) if args.mode is not None else None, recursive = args.recursive, + parents = args.parents, ) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index e452d871..736caa97 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -112,6 +112,10 @@ class FileContext(abc.ABC): def root(self) -> str: return self.__uri.path + @property + def chroot(self) -> bool: + return self.__chroot + @property def username(self) -> str | None: return self.__uri.username @@ -308,6 +312,30 @@ class FileContext(abc.ABC): async def is_dir(self, path: str, follow_symlinks: bool = True) -> bool: return await self._is_dir(self._chroot(path), follow_symlinks = follow_symlinks) + async def _is_gettable(self, path: str, follow_symlinks: bool) -> bool: + try: + return (await self._stat(path, follow_symlinks)).is_gettable + except NotImplementedError: + log( + DEBUG, + ( + f"{self.log_name} doesn't implement stat(), judging by " + f'trailing slash if {path} is a file' + ), + ) + return path[-1] != '/' + except FileNotFoundError as e: + log(DEBUG, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + return False + except Exception as e: + log(ERR, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + raise + + async def is_gettable(self, path: str, follow_symlinks: bool = True) -> bool: + return await self._is_gettable( + self._chroot(path), follow_symlinks = follow_symlinks + ) + async def _readdir(self, path: str) -> Iterable[str]: raise NotImplementedError( f'{self.log_name}: readdir("{path}") is not implemented' diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 1295e4c7..af6b6b2a 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -8,6 +8,7 @@ from enum import Enum, auto from typing import TYPE_CHECKING, Any, Iterable, TypeVar, cast from .base import Input, InputMode, Result +from .FileContext import FileContext from .log import DEBUG, ERR, log from .Uri import Uri @@ -15,7 +16,6 @@ if TYPE_CHECKING: from argparse import Namespace from .ExecContext import ExecContext - from .FileContext import FileContext from .ProcFilter import ProcFilter, ProcPipeline T = TypeVar('T') @@ -169,18 +169,22 @@ async def get( return await run_pipeline(await ctx.get(uri.path, *args, **kwargs), content_filter) async def copy( - src_uri: str | Iterable[str], + src: str | FileContext | Iterable[str], dst: str | FileContext, owner: str | None = None, group: str | None = None, mode: int | None = None, throw: bool = True, recursive: bool = False, + parents: bool = False, ) -> Exception | str | list[str]: - if not isinstance(src_uri, str): - ret: list[str] = [] - for uri in src_uri: # TODO: Group identical netlocs into one CopyContext - rr = await copy(uri, dst, owner, group, mode, throw, recursive) + + # Destination address / addresses + ret: str | list[str] + if not isinstance(src, (str, FileContext)): + ret = [] + for uri in src: # TODO: Group identical netlocs into one CopyContext + rr = await copy(uri, dst, owner, group, mode, throw, recursive, parents) if isinstance(rr, Exception): return rr if isinstance(rr, list): @@ -190,72 +194,91 @@ async def copy( else: raise Exception(f'copy() returned unexpected type {type(rr)}') return ret - from .CopyContext import CopyContext - async with CopyContext(src_uri, dst) as ctx: - - async def __mkdir_p(path: str) -> None: - # -- Create each level of the destination path that does not - # exist yet; put() does not create parent directories - cur = '' - for part in path.strip('/').split('/'): - cur = f'{cur}/{part}' - if not await ctx.dst.is_dir(cur): - await ctx.dst.mkdir(cur) - - async def __copy_tree(src_path: str, dst_path: str) -> list[str]: - ret: list[str] = [] - for entry in await ctx.src.readdir(src_path): - s_path = f'{src_path}/{entry}' - d_path = f'{dst_path}/{entry}' - if await ctx.src.is_dir(s_path): - await __mkdir_p(d_path) - ret.extend(await __copy_tree(s_path, d_path)) - else: - result = await ctx.src.get(s_path, throw = True) - parent = d_path.rpartition('/')[0] - if parent: - await __mkdir_p(parent) - await ctx.dst.put( - path = d_path, - content = result.stdout, - owner = owner, - group = group, - mode = mode, - throw = True, - ) - ret.append(d_path) - return ret - - try: - if recursive and await ctx.src.is_dir(ctx.src.root): - # -- A directory source is mirrored below the destination - # root: into it, if the root is an existing directory, - # as it, if it does not exist yet - dst_root = ctx.dst.root - if await ctx.dst.is_dir(dst_root): - dst_root += '/' + os.path.basename(src_uri) - await __mkdir_p(dst_root) - return await __copy_tree(ctx.src.root, dst_root) - result = await ctx.src.get(ctx.src.root, throw = True) - dst_path = ctx.dst.root - if await ctx.dst.is_dir(ctx.dst.root): - dst_path += '/' + os.path.basename(src_uri) + async def __copy_tree(src_path: str, dst_path: str) -> list[str]: + ret: list[str] = [] + s_st = await ctx.src.stat(src_path) + if dst_path and not await ctx.dst.is_dir(dst_path): + await ctx.dst.mkdir(dst_path, ftype = s_st.ftype) + for dirent in await ctx.src.readdir(src_path): + s_path = f'{src_path}/{dirent}' if src_path else dirent + d_path = f'{dst_path}/{dirent}' if dst_path else dirent + s_st = await ctx.src.stat(s_path) + # -- Descend into directories that are not fetchable + # whole; an entry, which is both, lands as a file + if s_st.is_dir and not s_st.is_gettable: + ret.extend(await __copy_tree(s_path, d_path)) + continue + result = await ctx.src.get(s_path, throw = True) await ctx.dst.put( - path = dst_path, + path = d_path, content = result.stdout, owner = owner, group = group, mode = mode, + ftype = s_st.ftype, throw = True, ) - return dst_path - except Exception as e: - if throw: - raise - log(ERR, f'Failed to copy {src_uri} -> {dst} ({str(e)})') - return e - assert False, 'Unreachable code' + ret.append(d_path) + return ret + + try: + + from .CopyContext import CopyContext + + async with CopyContext(src, dst) as ctx: + + src_uri = src.uri if isinstance(src, FileContext) else Uri(src) + ret = '' if ctx.dst.chroot else ctx.dst.root + if parents: + # -- cp --parents: the source lands at its source path + # below the destination root, each parent level + # typed by the source-side node it mirrors + parts = [p for p in src_uri.path.split('/') if p] + if parts: + src_path = '/'.join(parts) + base = ret + ret = f'{base}/{src_path}' if base else src_path + s_cur = '' + for part in parts[:-1]: + s_cur = f'{s_cur}/{part}' if s_cur else part + d_cur = f'{base}/{s_cur}' if base else s_cur + if await ctx.dst.is_dir(d_cur): + continue + level_ctx = FileContext.create( + f'{ctx.src.uri.scheme_plus_authority}/{s_cur}', + chroot = True, + ) + async with level_ctx: + ftype = (await level_ctx.stat('')).ftype + await ctx.dst.mkdir(d_cur, ftype = ftype) + elif ret: + # Land the source's basename within existing destination directories + basename = os.path.basename(src_uri.path.rstrip('/')) + if basename and await ctx.dst.is_dir(ret): + ret += '/' + basename + src_root = '' if ctx.src.chroot else ctx.src.root + s_st = await ctx.src.stat(src_root) + if recursive and s_st.is_dir and not s_st.is_gettable: + return await __copy_tree(src_root, ret) + + result = await ctx.src.get(src_root, throw = True) + await ctx.dst.put( + path = ret, + content = result.stdout, + owner = owner, + group = group, + mode = mode, + ftype = s_st.ftype, + throw = True, + ) + return ret + + except Exception as e: + if throw: + raise + log(ERR, f'Failed to copy {src} -> {dst} ({str(e)})') + return e async def get_username( # export args: Namespace | None = None, diff --git a/test/integration/jw_pkg/help/test-expected.txt b/test/integration/jw_pkg/help/test-expected.txt index 0b0c0b39..9c2e5f12 100644 --- a/test/integration/jw_pkg/help/test-expected.txt +++ b/test/integration/jw_pkg/help/test-expected.txt @@ -186,7 +186,8 @@ Available subcommands of posix: copy Copy files tar Handle tar archives ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help -usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F] +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] + [--parents] [-F] src dst Copy files @@ -201,6 +202,8 @@ options: -g, --group GROUP Destination file group (default: None) -m, --mode MODE Destination file mode (default: None) -r, --recursive Copy directories recursively (default: False) + --parents Land a single object at its source path below the + destination root (default: False) -F, --fixed-strings Don't expand macros in and (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help usage: jw-pkg.py posix tar [-h] ... diff --git a/test/unit/python/jw/pkg/lib/util/test.py b/test/unit/python/jw/pkg/lib/util/test.py index 001b9987..0520ee3b 100644 --- a/test/unit/python/jw/pkg/lib/util/test.py +++ b/test/unit/python/jw/pkg/lib/util/test.py @@ -3,6 +3,7 @@ import os import stat import tempfile +from jw.pkg.lib.ec.Local import Local from jw.pkg.lib.util import copy def uri(path: str) -> str: @@ -99,6 +100,71 @@ async def run_tests() -> None: st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3')) assert stat.S_IMODE(st.st_mode) == 0o640, st + # -- A chrooted context source: the empty address is the + # context's root, and landing into an existing directory + # takes the root's basename + os.makedirs(os.path.join(tmp, 'dst/chroot')) + src_fc = Local(uri(os.path.join(tmp, 'src/a')), chroot = True) + ret = await copy(src_fc, uri(os.path.join(tmp, 'dst/chroot')), recursive = True) + assert os.path.isfile(os.path.join(tmp, 'dst/chroot/a/f1')), ret + assert os.path.isfile(os.path.join(tmp, 'dst/chroot/a/sub/f2')), ret + + # -- A trailing slash does not change the source's name + os.makedirs(os.path.join(tmp, 'dst/slash')) + ret = await copy( + uri(os.path.join(tmp, 'src/a/')), + uri(os.path.join(tmp, 'dst/slash')), + recursive = True, + ) + assert isinstance(ret, list), ret + assert os.path.join(tmp, 'dst/slash/a/f1') in ret, ret + src_fc = Local(uri(os.path.join(tmp, 'src/a/')), chroot = True) + ret = await copy(src_fc, uri(os.path.join(tmp, 'dst/slash')), recursive = True) + assert isinstance(ret, list), ret + assert os.path.join(tmp, 'dst/slash/a/f1') in ret, ret + + # -- parents lands the object at its source path below the + # destination root, like cp --parents + dst_parents = os.path.join(tmp, 'dst/parents') + os.makedirs(dst_parents) + src_single = os.path.join(tmp, 'src/single') + ret = await copy(uri(src_single), uri(dst_parents), parents = True) + assert ret == dst_parents + src_single, ret + with open(dst_parents + src_single) as f: + assert f.read() == 'solo\n' + + # -- parents on a tree lands the tree at its source path + dst_ptree = os.path.join(tmp, 'dst/ptree') + os.makedirs(dst_ptree) + src_tree = os.path.join(tmp, 'src/a') + ret = await copy( + uri(src_tree), uri(dst_ptree), recursive = True, parents = True + ) + assert isinstance(ret, list), ret + expected = dst_ptree + src_tree + assert sorted(ret) == sorted([ + os.path.join(expected, 'f1'), + os.path.join(expected, 'sub/f2'), + os.path.join(expected, 'sub/subsub/f3'), + ]), ret + assert os.path.isdir(os.path.join(expected, 'empty')), ret + + # -- a chrooted context source lands at its URI path + dst_pchroot = os.path.join(tmp, 'dst/pchroot') + os.makedirs(dst_pchroot) + src_fc = Local(uri(src_tree), chroot = True) + ret = await copy(src_fc, uri(dst_pchroot), recursive = True, parents = True) + assert isinstance(ret, list), ret + assert os.path.isfile(os.path.join(dst_pchroot + src_tree, 'f1')), ret + + # -- re-copying with parents replaces content, not layout + with open(dst_parents + src_single, 'w') as f: + f.write('replaced\n') + ret = await copy(uri(src_single), uri(dst_parents), parents = True) + assert ret == dst_parents + src_single, ret + with open(dst_parents + src_single) as f: + assert f.read() == 'solo\n' + # -- A missing source fails loudly try: await copy(