diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index 2dc600ec..cd9b7dab 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -27,6 +27,12 @@ class CmdCopy(Cmd): # export parser.add_argument( '-m', '--mode', default = None, help = 'Destination file mode' ) + parser.add_argument( + '-r', + '--recursive', + action = 'store_true', + help = 'Copy directories recursively', + ) parser.add_argument( '-F', '--fixed-strings', @@ -53,4 +59,5 @@ class CmdCopy(Cmd): # export owner = args.owner, group = args.group, mode = int(args.mode, 0), + recursive = args.recursive, ) diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 8033902c..4e97cff4 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -174,11 +174,12 @@ async def copy( group: str | None = None, mode: int | None = None, throw: bool = True, + recursive: bool = False, ) -> Exception | str | list[str]: if not isinstance(src_uri, str): ret: list[str] = [] for uri in src_uri: # TODO: Group identical netlocs into one CopyContext - rr = await copy(uri, dst, owner, group, mode, throw) + rr = await copy(uri, dst, owner, group, mode, throw, recursive) if isinstance(rr, Exception): return rr if isinstance(rr, list): @@ -191,7 +192,50 @@ async def copy( from .CopyContext import CopyContext async with CopyContext(src_uri, dst) as ctx: + + async def __mkdir_p(path: str) -> None: + # -- Create each level of the destination path that does not + # exist yet; put() does not create parent directories + cur = '' + for part in path.strip('/').split('/'): + cur = f'{cur}/{part}' + if not await ctx.dst.is_dir(cur): + await ctx.dst.mkdir(cur) + + async def __copy_tree(src_path: str, dst_path: str) -> list[str]: + ret: list[str] = [] + for entry in await ctx.src.readdir(src_path): + s_path = f'{src_path}/{entry}' + d_path = f'{dst_path}/{entry}' + if await ctx.src.is_dir(s_path): + await __mkdir_p(d_path) + ret.extend(await __copy_tree(s_path, d_path)) + else: + result = await ctx.src.get(s_path, throw = True) + parent = d_path.rpartition('/')[0] + if parent: + await __mkdir_p(parent) + await ctx.dst.put( + path = d_path, + content = result.stdout, + owner = owner, + group = group, + mode = mode, + throw = True, + ) + ret.append(d_path) + return ret + try: + if recursive and await ctx.src.is_dir(ctx.src.root): + # -- A directory source is mirrored below the destination + # root: into it, if the root is an existing directory, + # as it, if it does not exist yet + dst_root = ctx.dst.root + if await ctx.dst.is_dir(dst_root): + dst_root += '/' + os.path.basename(src_uri) + await __mkdir_p(dst_root) + return await __copy_tree(ctx.src.root, dst_root) result = await ctx.src.get(ctx.src.root, throw = True) dst_path = ctx.dst.root if await ctx.dst.is_dir(ctx.dst.root): diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 2ab2fac0..9dd91555 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -176,7 +176,8 @@ Available subcommands of posix: copy Copy files tar Handle tar archives ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help -usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F] + src dst Copy files @@ -189,6 +190,7 @@ options: -o, --owner OWNER Destination file owner (default: None) -g, --group GROUP Destination file group (default: None) -m, --mode MODE Destination file mode (default: None) + -r, --recursive Copy directories recursively (default: False) -F, --fixed-strings Don't expand macros in and (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help usage: jw-pkg.py posix tar [-h] ... diff --git a/test/unit/python/jw/pkg/lib/util/Makefile b/test/unit/python/jw/pkg/lib/util/Makefile new file mode 100644 index 00000000..de693650 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/util/test.py b/test/unit/python/jw/pkg/lib/util/test.py new file mode 100644 index 00000000..001b9987 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/test.py @@ -0,0 +1,115 @@ +import asyncio +import os +import stat +import tempfile + +from jw.pkg.lib.util import copy + +def uri(path: str) -> str: + return 'local://localhost' + path + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + # -- Source tree + # + # src/a/f1 + # src/a/sub/f2 + # src/a/sub/subsub/f3 + # src/a/empty + # src/single + for rel, content in [ + ('src/a/f1', 'one\n'), + ('src/a/sub/f2', 'two\n'), + ('src/a/sub/subsub/f3', 'three\n'), + ('src/single', 'solo\n'), + ]: + path = os.path.join(tmp, rel) + os.makedirs(os.path.dirname(path), exist_ok = True) + with open(path, 'w') as f: + f.write(content) + os.makedirs(os.path.join(tmp, 'src/a/empty')) + + # -- A single file to a nonexistent destination path; like cp, + # the destination's parent must exist + os.makedirs(os.path.join(tmp, 'dst')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/single')) + ) + assert ret == os.path.join(tmp, 'dst/single'), ret + with open(os.path.join(tmp, 'dst/single')) as f: + assert f.read() == 'solo\n' + + # -- A single file into an existing directory gets the basename + os.makedirs(os.path.join(tmp, 'dst/dir')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/dir')) + ) + assert ret == os.path.join(tmp, 'dst/dir/single'), ret + with open(os.path.join(tmp, 'dst/dir/single')) as f: + assert f.read() == 'solo\n' + + # -- A file with recursive=True behaves like a plain file copy + ret = await copy( + uri(os.path.join(tmp, 'src/single')), + uri(os.path.join(tmp, 'dst/recursive-file')), + recursive = True, + ) + assert ret == os.path.join(tmp, 'dst/recursive-file'), ret + + # -- A tree to a nonexistent destination: the destination becomes + # the tree, parent directories are created, empty directories + # survive, and the returned paths are the copied leaves + dst_tree = os.path.join(tmp, 'dst', 'tree') + ret = await copy( + uri(os.path.join(tmp, 'src/a')), uri(dst_tree), recursive = True + ) + assert isinstance(ret, list), ret + assert sorted(ret) == sorted([ + os.path.join(dst_tree, 'f1'), + os.path.join(dst_tree, 'sub/f2'), + os.path.join(dst_tree, 'sub/subsub/f3'), + ]), ret + for rel, content in [ + ('f1', 'one\n'), + ('sub/f2', 'two\n'), + ('sub/subsub/f3', 'three\n'), + ]: + with open(os.path.join(dst_tree, rel)) as f: + assert f.read() == content, rel + assert os.path.isdir(os.path.join(dst_tree, 'empty')) + + # -- A tree into an existing directory is copied into it under + # the source basename + os.makedirs(os.path.join(tmp, 'dst', 'into')) + ret = await copy( + uri(os.path.join(tmp, 'src/a')), + uri(os.path.join(tmp, 'dst/into')), + recursive = True, + ) + assert os.path.isfile(os.path.join(tmp, 'dst/into/a/sub/f2')), ret + + # -- A mode applies to every copied leaf + dst_mode = os.path.join(tmp, 'dst', 'mode') + await copy( + uri(os.path.join(tmp, 'src/a')), + uri(dst_mode), + mode = 0o640, + recursive = True + ) + st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3')) + assert stat.S_IMODE(st.st_mode) == 0o640, st + + # -- A missing source fails loudly + try: + await copy( + uri(os.path.join(tmp, 'src/missing')), uri(os.path.join(tmp, 'dst/x')) + ) + raise AssertionError('expected an exception') + except Exception: + pass + +def main() -> None: + asyncio.run(run_tests()) + print('All util copy tests passed') + +main()