diff --git a/docs/sandbox.md b/docs/sandbox.md index 76c87b22..b9c0aa75 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -1780,6 +1780,10 @@ def join_local(root: str, rel: str) -> str A local path for a relative entry path, on any platform. +Paths come from the remote side on pull, so they get the archive +member check, and a symlink among the parents under root is refused: +writing through it would land outside root. + #### prune\_descendants @@ -1822,6 +1826,29 @@ Write the changed entries as a gzip'd tar spooled to dest. Members mirror the executor's archive: paths relative and /-separated, modes and whole-second mtimes kept, symlinks as links. + + +#### clean\_path + +```python +def clean_path(name: str) -> str +``` + +A relative path safe to write under the destination, or ValueError. + +The same refusal the executor applies to archives it extracts: +absolute paths and .. escapes never touch the filesystem. + + + +#### open\_nofollow + +```python +def open_nofollow(path: str) -> BinaryIO +``` + +Open path for writing, refusing to follow a symlink at path. + #### extract\_archive @@ -4247,7 +4274,9 @@ and only the changed set is extracted. Files larger than one batch go through upload_file/download_file instead (multipart on executors v0.0.24+, resumable downloads), and a pull's archive leaves them out so their bytes travel once. -Symlinks travel as links, modes are kept, owners are not. +Symlinks travel as links, modes are kept, owners are not. A pull +never writes outside local: a path that leaves it, or that goes +through a symlink, fails the sync. A no-op sync sends nothing: nothing is transferred or deleted; beyond the listing only ``checksum=True`` asks more — one sha256 @@ -4704,7 +4733,9 @@ and only the changed set is extracted. Files larger than one batch go through upload_file/download_file instead (multipart on executors v0.0.24+, resumable downloads), and a pull's archive leaves them out so their bytes travel once. -Symlinks travel as links, modes are kept, owners are not. +Symlinks travel as links, modes are kept, owners are not. A pull +never writes outside local: a path that leaves it, or that goes +through a symlink, fails the sync. A no-op sync sends nothing: nothing is transferred or deleted; beyond the listing only ``checksum=True`` asks more — one sha256 diff --git a/koyeb/sandbox/filesystem.py b/koyeb/sandbox/filesystem.py index 1d0b0833..0bb132f3 100644 --- a/koyeb/sandbox/filesystem.py +++ b/koyeb/sandbox/filesystem.py @@ -41,10 +41,12 @@ SYNC_BATCH_BYTES, SyncDiff, build_archive, + clean_path, diff, extract_archive, hash_candidates, join_local, + open_nofollow, path_excluded, plan_batches, prune_descendants, @@ -1080,7 +1082,7 @@ def download_file( done = 0 renamed = False try: - with open(part_path, "wb") as out: + with open_nofollow(part_path) as out: for chunk in client.files_read(remote_path, timeout=timeout): out.write(chunk) done += len(chunk) @@ -1179,7 +1181,9 @@ def sync( batch go through upload_file/download_file instead (multipart on executors v0.0.24+, resumable downloads), and a pull's archive leaves them out so their bytes travel once. - Symlinks travel as links, modes are kept, owners are not. + Symlinks travel as links, modes are kept, owners are not. A pull + never writes outside local: a path that leaves it, or that goes + through a symlink, fails the sync. A no-op sync sends nothing: nothing is transferred or deleted; beyond the listing only ``checksum=True`` asks more — one sha256 @@ -1361,6 +1365,10 @@ def _sync_pull( remote_entries = [ e for e in remote_entries if not path_excluded(e["path"], exclude) ] + # The listing comes from the sandbox: refuse a path that would land + # outside local before anything is written. + for entry in remote_entries: + clean_path(entry["path"]) hashes = self._sync_hashes( client, local, remote, checksum, True, remote_entries, local_entries ) @@ -1420,7 +1428,13 @@ def _sync_local_deletes(self, local: str, result: SyncDiff, delete: bool) -> int by_path = {e["path"]: e for e in result.deletes} count = 0 for path in prune_descendants(list(by_path)): - target = join_local(local, path) + try: + target = join_local(local, path) + except ValueError: + # A type flip's extract replaced a parent with a symlink: + # the path is gone from the tree, never delete through it. + count += 1 + continue try: if os.path.isdir(target) and not os.path.islink(target): shutil.rmtree(target) @@ -2123,7 +2137,7 @@ async def download_file( done = 0 renamed = False try: - with open(part_path, "wb") as out: + with open_nofollow(part_path) as out: async for chunk in client.files_read( remote_path, timeout=timeout ): @@ -2224,7 +2238,9 @@ async def sync( batch go through upload_file/download_file instead (multipart on executors v0.0.24+, resumable downloads), and a pull's archive leaves them out so their bytes travel once. - Symlinks travel as links, modes are kept, owners are not. + Symlinks travel as links, modes are kept, owners are not. A pull + never writes outside local: a path that leaves it, or that goes + through a symlink, fails the sync. A no-op sync sends nothing: nothing is transferred or deleted; beyond the listing only ``checksum=True`` asks more — one sha256 @@ -2350,6 +2366,10 @@ async def _sync_pull( remote_entries = [ e for e in remote_entries if not path_excluded(e["path"], exclude) ] + # The listing comes from the sandbox: refuse a path that would land + # outside local before anything is written. + for entry in remote_entries: + clean_path(entry["path"]) hashes = await self._sync_hashes( client, local, remote, checksum, True, remote_entries, local_entries ) @@ -2482,7 +2502,13 @@ def _sync_local_deletes(self, local: str, result: SyncDiff, delete: bool) -> int by_path = {e["path"]: e for e in result.deletes} count = 0 for path in prune_descendants(list(by_path)): - target = join_local(local, path) + try: + target = join_local(local, path) + except ValueError: + # A type flip's extract replaced a parent with a symlink: + # the path is gone from the tree, never delete through it. + count += 1 + continue try: if os.path.isdir(target) and not os.path.islink(target): shutil.rmtree(target) diff --git a/koyeb/sandbox/sync.py b/koyeb/sandbox/sync.py index ec53c24a..a73bc470 100644 --- a/koyeb/sandbox/sync.py +++ b/koyeb/sandbox/sync.py @@ -23,12 +23,24 @@ import tarfile from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple +from typing import ( + Any, + BinaryIO, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, +) __all__ = [ "SYNC_BATCH_BYTES", "SyncDiff", "build_archive", + "clean_path", "diff", "excluded", "extract_archive", @@ -36,6 +48,7 @@ "hash_candidates", "join_local", "mtime_seconds", + "open_nofollow", "path_excluded", "path_match", "plan_batches", @@ -550,8 +563,27 @@ def remote_join(remote: str, rel: str) -> str: def join_local(root: str, rel: str) -> str: - """A local path for a relative entry path, on any platform.""" - return os.path.join(root, *rel.split("/")) + """A local path for a relative entry path, on any platform. + + Paths come from the remote side on pull, so they get the archive + member check, and a symlink among the parents under root is refused: + writing through it would land outside root. + """ + parts = clean_path(rel).split("/") + full = os.path.join(root, *parts) + # Windows separators and drives inside a name escape a POSIX check. + base = os.path.abspath(root) + if os.path.commonpath([base, os.path.abspath(full)]) != base: + raise ValueError(f"refusing path outside destination: {rel!r}") + path = root + for part in parts[:-1]: + path = os.path.join(path, part) + try: + if stat_module.S_ISLNK(os.lstat(path).st_mode): + raise ValueError(f"refusing path through a symlink: {rel!r}") + except (FileNotFoundError, NotADirectoryError): + break + return full def prune_descendants(paths: Sequence[str]) -> List[str]: @@ -630,8 +662,8 @@ def build_archive(root: str, entries: Sequence[Dict[str, Any]], dest: str) -> No tar.addfile(info, f) -def _clean_member(name: str) -> str: - """A member path safe to write under the destination, or ValueError. +def clean_path(name: str) -> str: + """A relative path safe to write under the destination, or ValueError. The same refusal the executor applies to archives it extracts: absolute paths and .. escapes never touch the filesystem. @@ -642,6 +674,13 @@ def _clean_member(name: str) -> str: return normalized +def open_nofollow(path: str) -> BinaryIO: + """Open path for writing, refusing to follow a symlink at path.""" + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(path, flags | getattr(os, "O_NOFOLLOW", 0), 0o666) + return os.fdopen(fd, "wb") + + def _clear_member(path: str) -> None: """Make room for a member: replace whatever sits at its path.""" if os.path.isdir(path) and not os.path.islink(path): @@ -657,7 +696,7 @@ def _extract_file(tar: "tarfile.TarFile", member: "tarfile.TarInfo", path: str) source = tar.extractfile(member) if source is None: return - with open(path, "wb") as out: + with open_nofollow(path) as out: shutil.copyfileobj(source, out, 1 << 20) os.chmod(path, member.mode) os.utime(path, (int(member.mtime), int(member.mtime))) @@ -703,18 +742,25 @@ def extract_archive( dirs: List[Tuple[str, int, int]] = [] with tarfile.open(archive_path, "r:gz") as tar: for member in tar: - name = _clean_member(member.name) + name = clean_path(member.name) if keep is not None and name not in keep: continue - path = os.path.join(dest, *name.split("/")) - kind = _extract_member(tar, member, path) + kind = _extract_member(tar, member, join_local(dest, name)) if kind == "skip": continue if kind == "dir": - dirs.append((path, member.mode, int(member.mtime))) + dirs.append((name, member.mode, int(member.mtime))) extracted.append(name) # Directory modes and mtimes last, deepest first, like the executor. - for path, mode, mtime in sorted(dirs, key=lambda d: d[0], reverse=True): + for name, mode, mtime in sorted(dirs, key=lambda d: d[0], reverse=True): + # A later member may have replaced the directory or a parent with + # a symlink: chmod and utime would follow it. + path = join_local(dest, name) + try: + if not stat_module.S_ISDIR(os.lstat(path).st_mode): + continue + except (FileNotFoundError, NotADirectoryError): + continue os.chmod(path, mode) os.utime(path, (mtime, mtime)) return extracted diff --git a/koyeb/sandbox/test_filesystem_sync.py b/koyeb/sandbox/test_filesystem_sync.py index 1be6ca0a..6748723b 100644 --- a/koyeb/sandbox/test_filesystem_sync.py +++ b/koyeb/sandbox/test_filesystem_sync.py @@ -311,6 +311,60 @@ async def files_upload_abort(self, upload_id, timeout=None): return _FakeExecutorTree.files_upload_abort(self, upload_id) +class _ScriptedExecutorTree(_FakeExecutorTree): + """An executor whose listing and archive are given verbatim: a pull + must stay inside its destination whatever the sandbox sends.""" + + def __init__(self, root, listing, members): + super().__init__(root) + self.listing = listing + self.members = members + + def files_list(self, path, recursive=False): + self.calls.append(("files_list", path, recursive)) + return iter(self.listing) + + def files_archive_get(self, path, exclude=()): + import io + import tarfile + + self.calls.append(("files_archive_get", path, tuple(exclude))) + out = io.BytesIO() + with tarfile.open(fileobj=out, mode="w:gz") as tar: + for name, linkname, payload in self.members: + info = tarfile.TarInfo(name) + if linkname is not None: + info.type = tarfile.SYMTYPE + info.linkname = linkname + tar.addfile(info) + else: + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + yield out.getvalue() + + def files_stat(self, path, hash=None): + return { + "type": "file", + "size": 2, + "mode": "0644", + "mtime": "2026-10-27T10:00:00Z", + } + + def files_read(self, path, chunk_size=65536, timeout=None): + self.calls.append(("files_read", path)) + yield b"hi" + + +def _listed(path, type_, size=0): + return { + "path": path, + "type": type_, + "size": size, + "mode": "0644", + "mtime": "2026-10-27T10:00:00Z", + } + + class _FakeLegacyExecutor: """An executor without the streamed /files routes.""" @@ -922,6 +976,94 @@ def test_pull_missing_remote_raises(self): with self.assertRaises(SandboxFileNotFoundError): fs.sync(local, "missing", direction="pull") + def _outside(self): + outside = os.path.join(self.base, "outside") + _write(os.path.join(outside, "child.txt"), b"keep") + return outside + + def test_pull_refuses_listing_path_outside_dest(self): + outside = self._outside() + fake = _ScriptedExecutorTree( + self.remote_root, + [_listed("../outside/evil.txt", "file", 2)], + [("../outside/evil.txt", None, b"hi")], + ) + dest = os.path.join(self.base, "dest") + + with _fs(fake) as fs: + with self.assertRaises(SandboxFilesystemError): + fs.sync(dest, "dst", direction="pull") + + self.assertEqual(os.listdir(outside), ["child.txt"]) + self.assertEqual([c for c in fake.calls if c[0] == "files_archive_get"], []) + + def test_pull_refuses_member_through_symlink(self): + outside = self._outside() + fake = _ScriptedExecutorTree( + self.remote_root, + [_listed("a", "symlink"), _listed("a/evil.txt", "file", 2)], + [("a", outside, None), ("a/evil.txt", None, b"hi")], + ) + dest = os.path.join(self.base, "dest") + + with _fs(fake) as fs: + with self.assertRaises(SandboxFilesystemError): + fs.sync(dest, "dst", direction="pull") + + self.assertEqual(os.listdir(outside), ["child.txt"]) + + def test_pull_refuses_oversized_download_through_symlink(self): + outside = self._outside() + fake = _ScriptedExecutorTree( + self.remote_root, + [ + _listed("a", "symlink"), + _listed("a/big.bin", "file", SYNC_BATCH_BYTES + 1), + ], + [("a", outside, None)], + ) + dest = os.path.join(self.base, "dest") + + with _fs(fake) as fs: + with self.assertRaises(SandboxFilesystemError): + fs.sync(dest, "dst", direction="pull") + + self.assertEqual(os.listdir(outside), ["child.txt"]) + self.assertEqual([c for c in fake.calls if c[0] == "files_read"], []) + + def test_pull_oversized_part_symlink_not_followed(self): + # The archive lands before the oversized downloads: a symlink at + # .part must not carry the download's bytes elsewhere. + victim = os.path.join(self._outside(), "child.txt") + remote = os.path.join(self.remote_root, "dst") + big = os.path.join(remote, "big.bin") + _write(big, b"") + with open(big, "wb") as f: + f.truncate(SYNC_BATCH_BYTES + 1) + os.symlink(victim, os.path.join(remote, "big.bin.part")) + dest = os.path.join(self.base, "dest") + + with _fs(self.fake) as fs: + with self.assertRaises(SandboxFilesystemError): + fs.sync(dest, "dst", direction="pull") + + with open(victim, "rb") as f: + self.assertEqual(f.read(), b"keep") + + def test_pull_type_flip_to_symlink_deletes_nothing_through_it(self): + outside = self._outside() + os.makedirs(os.path.join(self.remote_root, "dst")) + os.symlink(outside, os.path.join(self.remote_root, "dst", "flip")) + dest = os.path.join(self.base, "dest") + _write(os.path.join(dest, "flip", "child.txt")) + + with _fs(self.fake) as fs: + result = fs.sync(dest, "dst", direction="pull", delete=True) + + self.assertTrue(os.path.islink(os.path.join(dest, "flip"))) + self.assertEqual(os.listdir(outside), ["child.txt"]) + self.assertEqual(result["deleted"], 1) + class TestAsyncSyncPush(unittest.IsolatedAsyncioTestCase): def setUp(self): @@ -1289,6 +1431,39 @@ async def test_checksum_detects_content_change(self): # Both file entries passed the size+mtime test, so both hashed. self.assertEqual(len([c for c in self.fake.calls if c[0] == "files_stat"]), 2) + async def test_pull_oversized_part_symlink_not_followed(self): + victim = os.path.join(self.base, "outside", "child.txt") + _write(victim, b"keep") + remote = os.path.join(self.remote_root, "dst") + big = os.path.join(remote, "big.bin") + _write(big, b"") + with open(big, "wb") as f: + f.truncate(SYNC_BATCH_BYTES + 1) + os.symlink(victim, os.path.join(remote, "big.bin.part")) + dest = os.path.join(self.base, "dest") + + async with _async_ctx(self.fake) as fs: + with self.assertRaises(SandboxFilesystemError): + await fs.sync(dest, "dst", direction="pull") + + with open(victim, "rb") as f: + self.assertEqual(f.read(), b"keep") + + async def test_pull_type_flip_to_symlink_deletes_nothing_through_it(self): + outside = os.path.join(self.base, "outside") + _write(os.path.join(outside, "child.txt"), b"keep") + os.makedirs(os.path.join(self.remote_root, "dst")) + os.symlink(outside, os.path.join(self.remote_root, "dst", "flip")) + dest = os.path.join(self.base, "dest") + _write(os.path.join(dest, "flip", "child.txt")) + + async with _async_ctx(self.fake) as fs: + result = await fs.sync(dest, "dst", direction="pull", delete=True) + + self.assertTrue(os.path.islink(os.path.join(dest, "flip"))) + self.assertEqual(os.listdir(outside), ["child.txt"]) + self.assertEqual(result["deleted"], 1) + async def test_pull_delete_removes_local_extras(self): source = _local_tree(self.base) async with _async_ctx(self.fake) as fs: @@ -1330,6 +1505,15 @@ def test_join_local(self): join_local("/base", "a/b.txt"), os.path.join("/base", "a", "b.txt") ) + def test_join_local_refuses_paths_outside_root(self): + from koyeb.sandbox.sync import join_local + + with tempfile.TemporaryDirectory() as root: + os.symlink(tempfile.gettempdir(), os.path.join(root, "link")) + for rel in ("../x", "/etc/x", "a/../../x", "link/x"): + with self.subTest(rel=rel), self.assertRaises(ValueError): + join_local(root, rel) + if __name__ == "__main__": unittest.main() diff --git a/koyeb/sandbox/test_sync.py b/koyeb/sandbox/test_sync.py index be4e8770..be723fd1 100644 --- a/koyeb/sandbox/test_sync.py +++ b/koyeb/sandbox/test_sync.py @@ -689,6 +689,79 @@ def test_absolute_members_refused(self): with self.assertRaises(ValueError): extract_archive(self.archive, self.dst) + def _raw_archive(self, *members): + """Write an archive of (name, kind, payload) members as sent.""" + import tarfile + + with tarfile.open(self.archive, "w:gz") as tar: + for name, kind, payload in members: + info = tarfile.TarInfo(name) + if kind == "dir": + info.type = tarfile.DIRTYPE + info.mode = payload + tar.addfile(info) + elif kind == "sym": + info.type = tarfile.SYMTYPE + info.linkname = payload + tar.addfile(info) + else: + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + + def _outside(self): + outside = os.path.join(self.root, "outside") + os.makedirs(outside) + os.chmod(outside, 0o755) + return outside + + def test_member_through_symlink_member_refused(self): + outside = self._outside() + self._raw_archive(("a", "sym", outside), ("a/evil.txt", "file", b"hi")) + + with self.assertRaises(ValueError): + extract_archive(self.archive, self.dst) + self.assertEqual(os.listdir(outside), []) + + def test_member_through_existing_symlink_refused(self): + outside = self._outside() + os.symlink(outside, os.path.join(self.dst, "a")) + self._raw_archive(("a/sub", "dir", 0o755), ("a/sub/evil.txt", "file", b"hi")) + + with self.assertRaises(ValueError): + extract_archive(self.archive, self.dst) + self.assertEqual(os.listdir(outside), []) + + def test_file_member_replaces_symlink_without_following(self): + outside = self._outside() + victim = os.path.join(outside, "victim.txt") + with open(victim, "wb") as f: + f.write(b"keep") + os.symlink(victim, os.path.join(self.dst, "a.txt")) + self._raw_archive(("a.txt", "file", b"new")) + + extract_archive(self.archive, self.dst) + + with open(victim, "rb") as f: + self.assertEqual(f.read(), b"keep") + self.assertFalse(os.path.islink(os.path.join(self.dst, "a.txt"))) + + def test_dir_mode_not_applied_through_later_symlink(self): + outside = self._outside() + self._raw_archive(("d", "dir", 0o700), ("d", "sym", outside)) + + extract_archive(self.archive, self.dst) + + self.assertEqual(os.stat(outside).st_mode & 0o777, 0o755) + + def test_symlink_members_kept_as_sent(self): + # Links are data, like rsync -l: kept, never written through. + self._raw_archive(("abs", "sym", "/usr/bin/python3"), ("rel", "sym", "../up")) + + extract_archive(self.archive, self.dst) + + self.assertEqual(os.readlink(os.path.join(self.dst, "abs")), "/usr/bin/python3") + self.assertEqual(os.readlink(os.path.join(self.dst, "rel")), "../up") + if __name__ == "__main__": unittest.main()