Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions problemtools/checks/testdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
TestDataGroup,
)
from ..run import Program
from .validators import check_testcase_input
from .validators import InputValidationCache


def check_testdata(
Expand All @@ -47,14 +47,17 @@ def check_testdata(
has_custom_grader = graders.grader is not None
has_default_grader = DEFAULT_GRADER is not None

input_validation = InputValidationCache(input_validators, work_dir)
input_validation.precompute(testdata, context)

_check_group(
testdata,
context,
metadata,
probdir,
has_custom_grader,
has_default_grader,
input_validators,
input_validation,
output_validator,
work_dir,
diag,
Expand All @@ -68,7 +71,7 @@ def _check_group(
probdir: Path,
has_custom_grader: bool,
has_default_grader: bool,
input_validators: InputValidators,
input_validation: InputValidationCache,
output_validator: Program,
work_dir: Path,
diag: Diagnostics,
Expand Down Expand Up @@ -190,13 +193,13 @@ def _check_group(
probdir,
has_custom_grader,
has_default_grader,
input_validators,
input_validation,
output_validator,
work_dir,
diag,
)
else:
_check_testcase(child, metadata, input_validators, output_validator, work_dir, diag)
_check_testcase(child, metadata, input_validation, output_validator, work_dir, diag)


def _natural_sort_le(a: str, b: str) -> bool:
Expand Down Expand Up @@ -231,7 +234,7 @@ def parse_num(s: str, i: int) -> tuple[int, int]:
def _check_testcase(
testcase: TestCase,
metadata: Metadata,
input_validators: InputValidators,
input_validation: InputValidationCache,
output_validator: Program,
work_dir: Path,
diag: Diagnostics,
Expand All @@ -240,7 +243,7 @@ def _check_testcase(
_check_newlines(testcase.ansfile, diag)
_check_size_limits(testcase.infile, diag)
_check_size_limits(testcase.ansfile, diag)
check_testcase_input(input_validators, testcase, work_dir, diag)
input_validation.check(testcase, diag)
anssize = testcase.ansfile.stat().st_size / 1024.0 / 1024.0
outputlim = metadata.limits.output
if anssize > outputlim:
Expand Down
73 changes: 70 additions & 3 deletions problemtools/checks/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,13 @@
import re
import string
import tempfile
import traceback
from collections.abc import Callable
from concurrent.futures import Future
from pathlib import Path
from re import Match

from ..context import Context
from ..diagnostics import Diagnostics
from ..formatversion import FormatVersion
from ..judge import SubmissionResult, validate_output
Expand Down Expand Up @@ -173,8 +176,24 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab
os.unlink(file_name)


def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: Path, diag: Diagnostics) -> None:
"""Run the (already checked) input validators against a single test case's input file."""
def _compute_testcase_input_errors(
validators: InputValidators, testcase: TestCase, work_dir: Path
) -> list[tuple[str, str | None]]:
"""Run all input validators against a single test case's input file.

Returns the (message, additional_info) pairs to report as errors -- one per validator
that rejected (or crashed on) the input; empty if all validators accepted it.

Catches unexpected exceptions rather than letting them propagate, returning them as an
error (a bit ugly, but good enough for now, I think)."""
try:
return _run_input_validators(validators, testcase, work_dir)
except Exception as e:
return [(f'Internal error while running input validators on {testcase.infile}: {e}', traceback.format_exc())]


def _run_input_validators(validators: InputValidators, testcase: TestCase, work_dir: Path) -> list[tuple[str, str | None]]:
errors: list[tuple[str, str | None]] = []
flags = testcase.input_validator_flags

for val in validators.validators:
Expand All @@ -193,7 +212,55 @@ def check_testcase_input(validators: InputValidators, testcase: TestCase, work_d
validator_stdout = outfile.read().decode('utf-8', 'replace')
validator_stderr = errfile.read().decode('utf-8', 'replace')
validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out)
diag.error(emsg, validator_output)
errors.append((emsg, validator_output))
return errors


# This is a bit overlapping with the ResultStore in judge/. I think they are sufficiently
# different that it's best to have them as two separate implementations for now
class InputValidationCache:
"""Runs a problem's input validators against every testcase's input file.

Usage: call precompute() once, as early as convenient. Then call check() for each
testcase -- in the same order check_testdata's tree walk visits them -- to report
whatever errors were (or, on demand, are now) found.

precompute() is a no-op at --threads=1 (context.executor is None); check() then just
runs each testcase synchronously.
"""

def __init__(self, validators: InputValidators, work_dir: Path) -> None:
self._validators = validators
self._work_dir = work_dir
# Keyed by infile rather than by the TestCase itself: TestCase is a frozen dataclass
# with a list field, so it isn't hashable.
self._futures: dict[Path, Future[list[tuple[str, str | None]]]] = {}

def precompute(self, testdata: TestDataGroup, context: Context) -> None:
"""Submit one background job per testcase matching context.data_filter.

May be called at most once. A no-op if context.executor is None."""
assert not self._futures, 'precompute() called more than once'
if context.executor is None:
return
for testcase in testdata.get_all_testcases():
if testcase.matches_filter(context.data_filter):
self._futures[testcase.infile] = context.submit_background_work(
_compute_testcase_input_errors, self._validators, testcase, self._work_dir
)

def check(self, testcase: TestCase, diag: Diagnostics) -> None:
"""Report the errors (if any) input validation found for testcase.

Blocks on the background job for testcase if precompute() started one and it's
still running; otherwise (no precompute(), or testcase was filtered out of it)
computes the result synchronously."""
future = self._futures.get(testcase.infile)
errors = (
future.result() if future is not None else _compute_testcase_input_errors(self._validators, testcase, self._work_dir)
)
for msg, additional_info in errors:
diag.error(msg, additional_info)


def check_output_validators(
Expand Down
10 changes: 6 additions & 4 deletions problemtools/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from re import Pattern
from typing import ParamSpec, TypeVar
from typing import Any, ParamSpec, TypeVar

_T = TypeVar('_T')
_P = ParamSpec('_P')
Expand All @@ -29,11 +29,13 @@ def __init__(
self.fixed_timelim = fixed_timelim
self.parts: list[str] = parts if parts is not None else list(PROBLEM_PARTS)
self.executor: ThreadPoolExecutor | None = ThreadPoolExecutor(threads) if threads > 1 else None
self._background_work: list[concurrent.futures.Future[object]] = []
self._background_work: list[concurrent.futures.Future[Any]] = []

def submit_background_work(self, job: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> None:
def submit_background_work(self, job: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> concurrent.futures.Future[_T]:
assert self.executor
self._background_work.append(self.executor.submit(job, *args, **kwargs))
future = self.executor.submit(job, *args, **kwargs)
self._background_work.append(future)
return future

def cancel_background_work(self) -> None:
for future in self._background_work:
Expand Down