Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions docs/sandbox.md
Original file line number Diff line number Diff line change
Expand Up @@ -1780,6 +1780,10 @@ def join_local(root: str, rel: str) -> str

A local path for a relative entry path, on any platform.

Paths come from the remote side on pull, so they get the archive
member check, and a symlink among the parents under root is refused:
writing through it would land outside root.

<a id="koyeb.sandbox.sync.prune_descendants"></a>

#### prune\_descendants
Expand Down Expand Up @@ -1822,6 +1826,29 @@ Write the changed entries as a gzip'd tar spooled to dest.
Members mirror the executor's archive: paths relative and
/-separated, modes and whole-second mtimes kept, symlinks as links.

<a id="koyeb.sandbox.sync.clean_path"></a>

#### clean\_path

```python
def clean_path(name: str) -> str
```

A relative path safe to write under the destination, or ValueError.

The same refusal the executor applies to archives it extracts:
absolute paths and .. escapes never touch the filesystem.

<a id="koyeb.sandbox.sync.open_nofollow"></a>

#### open\_nofollow

```python
def open_nofollow(path: str) -> BinaryIO
```

Open path for writing, refusing to follow a symlink at path.

<a id="koyeb.sandbox.sync.extract_archive"></a>

#### extract\_archive
Expand Down Expand Up @@ -4247,7 +4274,9 @@ and only the changed set is extracted. Files larger than one
batch go through upload_file/download_file instead (multipart
on executors v0.0.24+, resumable downloads), and a pull's
archive leaves them out so their bytes travel once.
Symlinks travel as links, modes are kept, owners are not.
Symlinks travel as links, modes are kept, owners are not. A pull
never writes outside local: a path that leaves it, or that goes
through a symlink, fails the sync.

A no-op sync sends nothing: nothing is transferred or deleted;
beyond the listing only ``checksum=True`` asks more — one sha256
Expand Down Expand Up @@ -4704,7 +4733,9 @@ and only the changed set is extracted. Files larger than one
batch go through upload_file/download_file instead (multipart
on executors v0.0.24+, resumable downloads), and a pull's
archive leaves them out so their bytes travel once.
Symlinks travel as links, modes are kept, owners are not.
Symlinks travel as links, modes are kept, owners are not. A pull
never writes outside local: a path that leaves it, or that goes
through a symlink, fails the sync.

A no-op sync sends nothing: nothing is transferred or deleted;
beyond the listing only ``checksum=True`` asks more — one sha256
Expand Down
38 changes: 32 additions & 6 deletions koyeb/sandbox/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,12 @@
SYNC_BATCH_BYTES,
SyncDiff,
build_archive,
clean_path,
diff,
extract_archive,
hash_candidates,
join_local,
open_nofollow,
path_excluded,
plan_batches,
prune_descendants,
Expand Down Expand Up @@ -1080,7 +1082,7 @@ def download_file(
done = 0
renamed = False
try:
with open(part_path, "wb") as out:
with open_nofollow(part_path) as out:
for chunk in client.files_read(remote_path, timeout=timeout):
out.write(chunk)
done += len(chunk)
Expand Down Expand Up @@ -1179,7 +1181,9 @@ def sync(
batch go through upload_file/download_file instead (multipart
on executors v0.0.24+, resumable downloads), and a pull's
archive leaves them out so their bytes travel once.
Symlinks travel as links, modes are kept, owners are not.
Symlinks travel as links, modes are kept, owners are not. A pull
never writes outside local: a path that leaves it, or that goes
through a symlink, fails the sync.

A no-op sync sends nothing: nothing is transferred or deleted;
beyond the listing only ``checksum=True`` asks more — one sha256
Expand Down Expand Up @@ -1361,6 +1365,10 @@ def _sync_pull(
remote_entries = [
e for e in remote_entries if not path_excluded(e["path"], exclude)
]
# The listing comes from the sandbox: refuse a path that would land
# outside local before anything is written.
for entry in remote_entries:
clean_path(entry["path"])
hashes = self._sync_hashes(
client, local, remote, checksum, True, remote_entries, local_entries
)
Expand Down Expand Up @@ -1420,7 +1428,13 @@ def _sync_local_deletes(self, local: str, result: SyncDiff, delete: bool) -> int
by_path = {e["path"]: e for e in result.deletes}
count = 0
for path in prune_descendants(list(by_path)):
target = join_local(local, path)
try:
target = join_local(local, path)
except ValueError:
# A type flip's extract replaced a parent with a symlink:
# the path is gone from the tree, never delete through it.
count += 1
continue
try:
if os.path.isdir(target) and not os.path.islink(target):
shutil.rmtree(target)
Expand Down Expand Up @@ -2123,7 +2137,7 @@ async def download_file(
done = 0
renamed = False
try:
with open(part_path, "wb") as out:
with open_nofollow(part_path) as out:
async for chunk in client.files_read(
remote_path, timeout=timeout
):
Expand Down Expand Up @@ -2224,7 +2238,9 @@ async def sync(
batch go through upload_file/download_file instead (multipart
on executors v0.0.24+, resumable downloads), and a pull's
archive leaves them out so their bytes travel once.
Symlinks travel as links, modes are kept, owners are not.
Symlinks travel as links, modes are kept, owners are not. A pull
never writes outside local: a path that leaves it, or that goes
through a symlink, fails the sync.

A no-op sync sends nothing: nothing is transferred or deleted;
beyond the listing only ``checksum=True`` asks more — one sha256
Expand Down Expand Up @@ -2350,6 +2366,10 @@ async def _sync_pull(
remote_entries = [
e for e in remote_entries if not path_excluded(e["path"], exclude)
]
# The listing comes from the sandbox: refuse a path that would land
# outside local before anything is written.
for entry in remote_entries:
clean_path(entry["path"])
hashes = await self._sync_hashes(
client, local, remote, checksum, True, remote_entries, local_entries
)
Expand Down Expand Up @@ -2482,7 +2502,13 @@ def _sync_local_deletes(self, local: str, result: SyncDiff, delete: bool) -> int
by_path = {e["path"]: e for e in result.deletes}
count = 0
for path in prune_descendants(list(by_path)):
target = join_local(local, path)
try:
target = join_local(local, path)
except ValueError:
# A type flip's extract replaced a parent with a symlink:
# the path is gone from the tree, never delete through it.
count += 1
continue
try:
if os.path.isdir(target) and not os.path.islink(target):
shutil.rmtree(target)
Expand Down
68 changes: 57 additions & 11 deletions koyeb/sandbox/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,32 @@
import tarfile
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from typing import (
Any,
BinaryIO,
Callable,
Dict,
Iterable,
List,
Optional,
Sequence,
Set,
Tuple,
)

__all__ = [
"SYNC_BATCH_BYTES",
"SyncDiff",
"build_archive",
"clean_path",
"diff",
"excluded",
"extract_archive",
"format_mtime",
"hash_candidates",
"join_local",
"mtime_seconds",
"open_nofollow",
"path_excluded",
"path_match",
"plan_batches",
Expand Down Expand Up @@ -550,8 +563,27 @@ def remote_join(remote: str, rel: str) -> str:


def join_local(root: str, rel: str) -> str:
"""A local path for a relative entry path, on any platform."""
return os.path.join(root, *rel.split("/"))
"""A local path for a relative entry path, on any platform.

Paths come from the remote side on pull, so they get the archive
member check, and a symlink among the parents under root is refused:
writing through it would land outside root.
"""
parts = clean_path(rel).split("/")
full = os.path.join(root, *parts)
# Windows separators and drives inside a name escape a POSIX check.
base = os.path.abspath(root)
if os.path.commonpath([base, os.path.abspath(full)]) != base:
raise ValueError(f"refusing path outside destination: {rel!r}")
path = root
for part in parts[:-1]:
path = os.path.join(path, part)
try:
if stat_module.S_ISLNK(os.lstat(path).st_mode):
raise ValueError(f"refusing path through a symlink: {rel!r}")
except (FileNotFoundError, NotADirectoryError):
break
return full


def prune_descendants(paths: Sequence[str]) -> List[str]:
Expand Down Expand Up @@ -630,8 +662,8 @@ def build_archive(root: str, entries: Sequence[Dict[str, Any]], dest: str) -> No
tar.addfile(info, f)


def _clean_member(name: str) -> str:
"""A member path safe to write under the destination, or ValueError.
def clean_path(name: str) -> str:
"""A relative path safe to write under the destination, or ValueError.

The same refusal the executor applies to archives it extracts:
absolute paths and .. escapes never touch the filesystem.
Expand All @@ -642,6 +674,13 @@ def _clean_member(name: str) -> str:
return normalized


def open_nofollow(path: str) -> BinaryIO:
"""Open path for writing, refusing to follow a symlink at path."""
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0)
fd = os.open(path, flags | getattr(os, "O_NOFOLLOW", 0), 0o666)
return os.fdopen(fd, "wb")


def _clear_member(path: str) -> None:
"""Make room for a member: replace whatever sits at its path."""
if os.path.isdir(path) and not os.path.islink(path):
Expand All @@ -657,7 +696,7 @@ def _extract_file(tar: "tarfile.TarFile", member: "tarfile.TarInfo", path: str)
source = tar.extractfile(member)
if source is None:
return
with open(path, "wb") as out:
with open_nofollow(path) as out:
shutil.copyfileobj(source, out, 1 << 20)
os.chmod(path, member.mode)
os.utime(path, (int(member.mtime), int(member.mtime)))
Expand Down Expand Up @@ -703,18 +742,25 @@ def extract_archive(
dirs: List[Tuple[str, int, int]] = []
with tarfile.open(archive_path, "r:gz") as tar:
for member in tar:
name = _clean_member(member.name)
name = clean_path(member.name)
if keep is not None and name not in keep:
continue
path = os.path.join(dest, *name.split("/"))
kind = _extract_member(tar, member, path)
kind = _extract_member(tar, member, join_local(dest, name))
if kind == "skip":
continue
if kind == "dir":
dirs.append((path, member.mode, int(member.mtime)))
dirs.append((name, member.mode, int(member.mtime)))
extracted.append(name)
# Directory modes and mtimes last, deepest first, like the executor.
for path, mode, mtime in sorted(dirs, key=lambda d: d[0], reverse=True):
for name, mode, mtime in sorted(dirs, key=lambda d: d[0], reverse=True):
# A later member may have replaced the directory or a parent with
# a symlink: chmod and utime would follow it.
path = join_local(dest, name)
try:
if not stat_module.S_ISDIR(os.lstat(path).st_mode):
continue
except (FileNotFoundError, NotADirectoryError):
continue
os.chmod(path, mode)
os.utime(path, (mtime, mtime))
return extracted
Loading
Loading