diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index 2dc600ec..cd9b7dab 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -27,6 +27,12 @@ class CmdCopy(Cmd): # export parser.add_argument( '-m', '--mode', default = None, help = 'Destination file mode' ) + parser.add_argument( + '-r', + '--recursive', + action = 'store_true', + help = 'Copy directories recursively', + ) parser.add_argument( '-F', '--fixed-strings', @@ -53,4 +59,5 @@ class CmdCopy(Cmd): # export owner = args.owner, group = args.group, mode = int(args.mode, 0), + recursive = args.recursive, ) diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 55faf2c7..1e5d6f30 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -5,7 +5,7 @@ import errno import sys from decimal import ROUND_FLOOR, Decimal -from typing import TYPE_CHECKING, Any, NamedTuple, override +from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, override if TYPE_CHECKING: from typing import Type @@ -688,3 +688,10 @@ class ExecContext(Base): ['chmod', self.__mode_str(mode), path], cmd_input = InputMode.NonInteractive ) + + @override + async def _readdir(self, path: str) -> Iterable[str]: + result = await self.run( + ['ls', '-1A', path], cmd_input = InputMode.NonInteractive + ) + return result.stdout_str.splitlines() diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 08d2e89f..c16a5b69 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -4,7 +4,7 @@ import abc from enum import Enum, auto from functools import cached_property -from typing import TYPE_CHECKING, Any, override +from typing import TYPE_CHECKING, Any, Iterable, override from .log import DEBUG, ERR, log from .ProcFilter import ProcPipeline @@ -301,6 +301,14 @@ class FileContext(abc.ABC): async def is_dir(self, path: str, follow_symlinks: bool = True) -> bool: return await self._is_dir(self._chroot(path), follow_symlinks = follow_symlinks) + async def _readdir(self, path: str) -> Iterable[str]: + raise NotImplementedError( + f'{self.log_name}: readdir("{path}") is not implemented' + ) + + async def readdir(self, path: str) -> Iterable[str]: + return await self._readdir(self._chroot(path)) + @classmethod def create(cls, uri: str | Uri, *args: Any, **kwargs: Any) -> 'FileContext': uri = Uri.pimp(uri) diff --git a/src/python/jw/pkg/lib/ec/Curl.py b/src/python/jw/pkg/lib/ec/Curl.py index 2637cf15..aae38ae1 100644 --- a/src/python/jw/pkg/lib/ec/Curl.py +++ b/src/python/jw/pkg/lib/ec/Curl.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, override +import errno + +from typing import TYPE_CHECKING, Any, Iterable, override from ..FileContext import FileContext as Base @@ -48,3 +50,8 @@ class Curl(Base): path = '/' + path cmd.append(self.uri.to_string + self._chroot(path)) return await self.__ec.run(cmd, throw = throw, verbose = verbose) + + @override + async def _readdir(self, path: str) -> Iterable[str]: + msg = f'{self.log_name}: readdir("{path}") is not implemented' + raise OSError(errno.ENOSYS, msg) diff --git a/src/python/jw/pkg/lib/ec/Local.py b/src/python/jw/pkg/lib/ec/Local.py index 9eca6817..b4c530ff 100644 --- a/src/python/jw/pkg/lib/ec/Local.py +++ b/src/python/jw/pkg/lib/ec/Local.py @@ -7,7 +7,7 @@ import pwd import sys from functools import cache -from typing import TYPE_CHECKING, override +from typing import TYPE_CHECKING, Iterable, override from ..base import Result, StatResult from ..ExecContext import ExecContext as Base @@ -206,3 +206,7 @@ class Local(Base): if (not follow_symlinks) and os.path.islink(path): return False return os.path.isdir(path) + + @override + async def _readdir(self, path: str) -> Iterable[str]: + return os.listdir(path) diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 8033902c..4e97cff4 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -174,11 +174,12 @@ async def copy( group: str | None = None, mode: int | None = None, throw: bool = True, + recursive: bool = False, ) -> Exception | str | list[str]: if not isinstance(src_uri, str): ret: list[str] = [] for uri in src_uri: # TODO: Group identical netlocs into one CopyContext - rr = await copy(uri, dst, owner, group, mode, throw) + rr = await copy(uri, dst, owner, group, mode, throw, recursive) if isinstance(rr, Exception): return rr if isinstance(rr, list): @@ -191,7 +192,50 @@ async def copy( from .CopyContext import CopyContext async with CopyContext(src_uri, dst) as ctx: + + async def __mkdir_p(path: str) -> None: + # -- Create each level of the destination path that does not + # exist yet; put() does not create parent directories + cur = '' + for part in path.strip('/').split('/'): + cur = f'{cur}/{part}' + if not await ctx.dst.is_dir(cur): + await ctx.dst.mkdir(cur) + + async def __copy_tree(src_path: str, dst_path: str) -> list[str]: + ret: list[str] = [] + for entry in await ctx.src.readdir(src_path): + s_path = f'{src_path}/{entry}' + d_path = f'{dst_path}/{entry}' + if await ctx.src.is_dir(s_path): + await __mkdir_p(d_path) + ret.extend(await __copy_tree(s_path, d_path)) + else: + result = await ctx.src.get(s_path, throw = True) + parent = d_path.rpartition('/')[0] + if parent: + await __mkdir_p(parent) + await ctx.dst.put( + path = d_path, + content = result.stdout, + owner = owner, + group = group, + mode = mode, + throw = True, + ) + ret.append(d_path) + return ret + try: + if recursive and await ctx.src.is_dir(ctx.src.root): + # -- A directory source is mirrored below the destination + # root: into it, if the root is an existing directory, + # as it, if it does not exist yet + dst_root = ctx.dst.root + if await ctx.dst.is_dir(dst_root): + dst_root += '/' + os.path.basename(src_uri) + await __mkdir_p(dst_root) + return await __copy_tree(ctx.src.root, dst_root) result = await ctx.src.get(ctx.src.root, throw = True) dst_path = ctx.dst.root if await ctx.dst.is_dir(ctx.dst.root): diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 2ab2fac0..9dd91555 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -176,7 +176,8 @@ Available subcommands of posix: copy Copy files tar Handle tar archives ============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help -usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-r] [-F] + src dst Copy files @@ -189,6 +190,7 @@ options: -o, --owner OWNER Destination file owner (default: None) -g, --group GROUP Destination file group (default: None) -m, --mode MODE Destination file mode (default: None) + -r, --recursive Copy directories recursively (default: False) -F, --fixed-strings Don't expand macros in and (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help usage: jw-pkg.py posix tar [-h] ... diff --git a/test/unit/python/jw/pkg/lib/ec/Curl/Makefile b/test/unit/python/jw/pkg/lib/ec/Curl/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/Curl/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/Curl/test.py b/test/unit/python/jw/pkg/lib/ec/Curl/test.py new file mode 100644 index 00000000..b220a4b7 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/Curl/test.py @@ -0,0 +1,15 @@ +import asyncio +import errno + +from jw.pkg.lib.ec.Curl import Curl + +async def run_tests() -> None: + curl = Curl('https://example.com/') + try: + await curl.readdir('some/path') + raise AssertionError('expected OSError with errno ENOSYS') + except OSError as e: + assert e.errno == errno.ENOSYS, e.errno + +asyncio.run(run_tests()) +print('All Curl readdir tests passed') diff --git a/test/unit/python/jw/pkg/lib/ec/Local/Makefile b/test/unit/python/jw/pkg/lib/ec/Local/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/Local/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/Local/test.py b/test/unit/python/jw/pkg/lib/ec/Local/test.py new file mode 100644 index 00000000..5ddf0af6 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/Local/test.py @@ -0,0 +1,33 @@ +import asyncio +import os +import tempfile + +from jw.pkg.lib.ec.Local import Local + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + for name in ['a', 'b', '.hidden']: + with open(os.path.join(tmp, name), 'w') as f: + f.write(name) + os.mkdir(os.path.join(tmp, 'subdir')) + + local = Local() + entries = sorted(await local.readdir(tmp)) + assert entries == ['.hidden', 'a', 'b', 'subdir'], entries + + # -- readdir() of a missing directory raises FileNotFoundError + try: + await local.readdir(os.path.join(tmp, 'missing')) + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + + # -- mkdir() in a chrooted context creates below the context root + root = os.path.join(tmp, 'root') + os.mkdir(root) + chrooted = Local('local://localhost' + root, chroot = True) + await chrooted.mkdir('sub') + assert os.path.isdir(os.path.join(root, 'sub')) + +asyncio.run(run_tests()) +print('All Local readdir tests passed') diff --git a/test/unit/python/jw/pkg/lib/util/Makefile b/test/unit/python/jw/pkg/lib/util/Makefile new file mode 100644 index 00000000..de693650 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/util/test.py b/test/unit/python/jw/pkg/lib/util/test.py new file mode 100644 index 00000000..001b9987 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/util/test.py @@ -0,0 +1,115 @@ +import asyncio +import os +import stat +import tempfile + +from jw.pkg.lib.util import copy + +def uri(path: str) -> str: + return 'local://localhost' + path + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + # -- Source tree + # + # src/a/f1 + # src/a/sub/f2 + # src/a/sub/subsub/f3 + # src/a/empty + # src/single + for rel, content in [ + ('src/a/f1', 'one\n'), + ('src/a/sub/f2', 'two\n'), + ('src/a/sub/subsub/f3', 'three\n'), + ('src/single', 'solo\n'), + ]: + path = os.path.join(tmp, rel) + os.makedirs(os.path.dirname(path), exist_ok = True) + with open(path, 'w') as f: + f.write(content) + os.makedirs(os.path.join(tmp, 'src/a/empty')) + + # -- A single file to a nonexistent destination path; like cp, + # the destination's parent must exist + os.makedirs(os.path.join(tmp, 'dst')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/single')) + ) + assert ret == os.path.join(tmp, 'dst/single'), ret + with open(os.path.join(tmp, 'dst/single')) as f: + assert f.read() == 'solo\n' + + # -- A single file into an existing directory gets the basename + os.makedirs(os.path.join(tmp, 'dst/dir')) + ret = await copy( + uri(os.path.join(tmp, 'src/single')), uri(os.path.join(tmp, 'dst/dir')) + ) + assert ret == os.path.join(tmp, 'dst/dir/single'), ret + with open(os.path.join(tmp, 'dst/dir/single')) as f: + assert f.read() == 'solo\n' + + # -- A file with recursive=True behaves like a plain file copy + ret = await copy( + uri(os.path.join(tmp, 'src/single')), + uri(os.path.join(tmp, 'dst/recursive-file')), + recursive = True, + ) + assert ret == os.path.join(tmp, 'dst/recursive-file'), ret + + # -- A tree to a nonexistent destination: the destination becomes + # the tree, parent directories are created, empty directories + # survive, and the returned paths are the copied leaves + dst_tree = os.path.join(tmp, 'dst', 'tree') + ret = await copy( + uri(os.path.join(tmp, 'src/a')), uri(dst_tree), recursive = True + ) + assert isinstance(ret, list), ret + assert sorted(ret) == sorted([ + os.path.join(dst_tree, 'f1'), + os.path.join(dst_tree, 'sub/f2'), + os.path.join(dst_tree, 'sub/subsub/f3'), + ]), ret + for rel, content in [ + ('f1', 'one\n'), + ('sub/f2', 'two\n'), + ('sub/subsub/f3', 'three\n'), + ]: + with open(os.path.join(dst_tree, rel)) as f: + assert f.read() == content, rel + assert os.path.isdir(os.path.join(dst_tree, 'empty')) + + # -- A tree into an existing directory is copied into it under + # the source basename + os.makedirs(os.path.join(tmp, 'dst', 'into')) + ret = await copy( + uri(os.path.join(tmp, 'src/a')), + uri(os.path.join(tmp, 'dst/into')), + recursive = True, + ) + assert os.path.isfile(os.path.join(tmp, 'dst/into/a/sub/f2')), ret + + # -- A mode applies to every copied leaf + dst_mode = os.path.join(tmp, 'dst', 'mode') + await copy( + uri(os.path.join(tmp, 'src/a')), + uri(dst_mode), + mode = 0o640, + recursive = True + ) + st = os.stat(os.path.join(dst_mode, 'sub/subsub/f3')) + assert stat.S_IMODE(st.st_mode) == 0o640, st + + # -- A missing source fails loudly + try: + await copy( + uri(os.path.join(tmp, 'src/missing')), uri(os.path.join(tmp, 'dst/x')) + ) + raise AssertionError('expected an exception') + except Exception: + pass + +def main() -> None: + asyncio.run(run_tests()) + print('All util copy tests passed') + +main()