Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions packages/syft-job/src/syft_job/job_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,27 @@ def get_job_timeout_seconds() -> int:
return timeout


def format_timeout_duration(seconds: int) -> str:
"""Format a job timeout duration for console messages.

Sub-minute timeouts are reported in seconds so that values like a
30 second limit do not read as "0 minutes". Durations of a minute or
longer keep the original minutes phrasing.

Args:
seconds: Timeout duration in seconds.

Returns:
Human-readable duration, e.g. "30 seconds" or "10 minutes".
"""
if seconds < 60:
unit = "second" if seconds == 1 else "seconds"
return f"{seconds} {unit}"
minutes = seconds // 60
unit = "minute" if minutes == 1 else "minutes"
return f"{minutes} {unit}"


IS_IN_JOB_ENV_VAR = "SYFT_IS_IN_JOB"

# A job runs from a fresh copy of its approved submission in the system temp
Expand Down Expand Up @@ -314,7 +335,9 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int, run_dir: Path) -> in
_kill_process_tree(process.pid)
process.wait()
timed_out = True
print(f" Job {job_name} timed out after {timeout // 60} minutes")
print(
f" Job {job_name} timed out after {format_timeout_duration(timeout)}"
)
stdout_f.write("\n--- PROCESS TIMED OUT ---\n")
stderr_f.write("\n--- PROCESS TIMED OUT ---\n")
break
Expand Down Expand Up @@ -385,7 +408,7 @@ def _execute_job_captured(self, ref: JobRef, timeout: int, run_dir: Path) -> int
returncode = -1
stdout = (stdout or "") + "\n--- PROCESS TIMED OUT ---\n"
stderr = (stderr or "") + "\n--- PROCESS TIMED OUT ---\n"
print(f" Job {job_name} timed out after {timeout // 60} minutes")
print(f" Job {job_name} timed out after {format_timeout_duration(timeout)}")

staging_dir = self.manager.staging_dir(ref)
staging_dir.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -521,7 +544,7 @@ def _run_approved_job(
self._report_result(ref, returncode)

except subprocess.TimeoutExpired:
print(f" Job {job_name} timed out after {timeout // 60} minutes")
print(f" Job {job_name} timed out after {format_timeout_duration(timeout)}")
self._finalize(ref, -1)
except Exception as e:
print(f" Error executing job {job_name}: {e}")
Expand Down
73 changes: 73 additions & 0 deletions packages/syft-job/tests/test_timeout_duration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Tests for human-readable timeout durations in job runner messages."""

import sys
from pathlib import Path

from syft_job.config import SyftJobConfig
from syft_job.job_ref import JobRef
from syft_job.job_runner import SyftJobRunner, format_timeout_duration
from syft_job.migrations.registry import JOB_PROTOCOL_VERSION

DO_EMAIL = "do@test.org"
DS_EMAIL = "ds@test.org"

SLEEP_PY = """\
import time

print("working", flush=True)
time.sleep(30)
"""


def test_sub_minute_timeout_is_reported_in_seconds() -> None:
assert format_timeout_duration(30) == "30 seconds"


def test_one_second_timeout_uses_singular() -> None:
assert format_timeout_duration(1) == "1 second"


def test_boundary_below_one_minute_stays_in_seconds() -> None:
assert format_timeout_duration(59) == "59 seconds"


def test_one_minute_timeout_uses_singular() -> None:
assert format_timeout_duration(60) == "1 minute"


def test_sub_minute_remainder_truncates_like_before() -> None:
# 61s previously printed "1 minutes" via 61 // 60; keep the truncation,
# but with the correct plural.
assert format_timeout_duration(61) == "1 minute"


def test_default_timeout_keeps_minutes_phrasing() -> None:
assert format_timeout_duration(600) == "10 minutes"


def test_two_minute_timeout_is_plural() -> None:
assert format_timeout_duration(120) == "2 minutes"


def test_captured_timeout_message_reports_seconds(tmp_path: Path, capsys) -> None:
"""A sub-minute timeout prints the duration in seconds, not "0 minutes"."""
config = SyftJobConfig(syftbox_folder=tmp_path, current_user_email=DO_EMAIL)
runner = SyftJobRunner(config=config)
ref = JobRef(DO_EMAIL, DS_EMAIL, "short-timeout.job", JOB_PROTOCOL_VERSION)

# Run main.py with the test's own Python, skipping the venv setup of a real job.
run_dir = tmp_path / "run"
staging_dir = runner.manager.staging_dir(ref)
run_dir.mkdir()
(run_dir / "main.py").write_text(SLEEP_PY)
(run_dir / "run.sh").write_text(f'#!/bin/bash\n"{sys.executable}" main.py\n')

returncode = runner._execute_job_captured(ref, timeout=1, run_dir=run_dir)

assert returncode == -1
stdout = (staging_dir / "stdout.txt").read_text()
assert "--- PROCESS TIMED OUT ---" in stdout

captured = capsys.readouterr().out
assert "timed out after 1 second" in captured
assert "0 minutes" not in captured
Loading