From a768db3eb38e77ffdecf7b272f2cfd0e39d54ae0 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 28 Aug 2026 16:48:51 +0200 Subject: [PATCH] Add multithread support for input validation #417 --- problemtools/checks/testdata.py | 17 ++++--- problemtools/checks/validators.py | 73 +++++++++++++++++++++++++++++-- problemtools/context.py | 10 +++-- 3 files changed, 86 insertions(+), 14 deletions(-) diff --git a/problemtools/checks/testdata.py b/problemtools/checks/testdata.py index c70456f8..f0382cfb 100644 --- a/problemtools/checks/testdata.py +++ b/problemtools/checks/testdata.py @@ -24,7 +24,7 @@ TestDataGroup, ) from ..run import Program -from .validators import check_testcase_input +from .validators import InputValidationCache def check_testdata( @@ -47,6 +47,9 @@ def check_testdata( has_custom_grader = graders.grader is not None has_default_grader = DEFAULT_GRADER is not None + input_validation = InputValidationCache(input_validators, work_dir) + input_validation.precompute(testdata, context) + _check_group( testdata, context, @@ -54,7 +57,7 @@ def check_testdata( probdir, has_custom_grader, has_default_grader, - input_validators, + input_validation, output_validator, work_dir, diag, @@ -68,7 +71,7 @@ def _check_group( probdir: Path, has_custom_grader: bool, has_default_grader: bool, - input_validators: InputValidators, + input_validation: InputValidationCache, output_validator: Program, work_dir: Path, diag: Diagnostics, @@ -190,13 +193,13 @@ def _check_group( probdir, has_custom_grader, has_default_grader, - input_validators, + input_validation, output_validator, work_dir, diag, ) else: - _check_testcase(child, metadata, input_validators, output_validator, work_dir, diag) + _check_testcase(child, metadata, input_validation, output_validator, work_dir, diag) def _natural_sort_le(a: str, b: str) -> bool: @@ -231,7 +234,7 @@ def parse_num(s: str, i: int) -> tuple[int, int]: def _check_testcase( testcase: TestCase, metadata: Metadata, - input_validators: InputValidators, + input_validation: InputValidationCache, output_validator: Program, work_dir: Path, diag: Diagnostics, @@ -240,7 +243,7 @@ def _check_testcase( _check_newlines(testcase.ansfile, diag) _check_size_limits(testcase.infile, diag) _check_size_limits(testcase.ansfile, diag) - check_testcase_input(input_validators, testcase, work_dir, diag) + input_validation.check(testcase, diag) anssize = testcase.ansfile.stat().st_size / 1024.0 / 1024.0 outputlim = metadata.limits.output if anssize > outputlim: diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index 3d9933af..6649fc6e 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -7,10 +7,13 @@ import re import string import tempfile +import traceback from collections.abc import Callable +from concurrent.futures import Future from pathlib import Path from re import Match +from ..context import Context from ..diagnostics import Diagnostics from ..formatversion import FormatVersion from ..judge import SubmissionResult, validate_output @@ -173,8 +176,24 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab os.unlink(file_name) -def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: Path, diag: Diagnostics) -> None: - """Run the (already checked) input validators against a single test case's input file.""" +def _compute_testcase_input_errors( + validators: InputValidators, testcase: TestCase, work_dir: Path +) -> list[tuple[str, str | None]]: + """Run all input validators against a single test case's input file. + + Returns the (message, additional_info) pairs to report as errors -- one per validator + that rejected (or crashed on) the input; empty if all validators accepted it. + + Catches unexpected exceptions rather than letting them propagate, returning them as an + error (a bit ugly, but good enough for now, I think).""" + try: + return _run_input_validators(validators, testcase, work_dir) + except Exception as e: + return [(f'Internal error while running input validators on {testcase.infile}: {e}', traceback.format_exc())] + + +def _run_input_validators(validators: InputValidators, testcase: TestCase, work_dir: Path) -> list[tuple[str, str | None]]: + errors: list[tuple[str, str | None]] = [] flags = testcase.input_validator_flags for val in validators.validators: @@ -193,7 +212,55 @@ def check_testcase_input(validators: InputValidators, testcase: TestCase, work_d validator_stdout = outfile.read().decode('utf-8', 'replace') validator_stderr = errfile.read().decode('utf-8', 'replace') validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out) - diag.error(emsg, validator_output) + errors.append((emsg, validator_output)) + return errors + + +# This is a bit overlapping with the ResultStore in judge/. I think they are sufficiently +# different that it's best to have them as two separate implementations for now +class InputValidationCache: + """Runs a problem's input validators against every testcase's input file. + + Usage: call precompute() once, as early as convenient. Then call check() for each + testcase -- in the same order check_testdata's tree walk visits them -- to report + whatever errors were (or, on demand, are now) found. + + precompute() is a no-op at --threads=1 (context.executor is None); check() then just + runs each testcase synchronously. + """ + + def __init__(self, validators: InputValidators, work_dir: Path) -> None: + self._validators = validators + self._work_dir = work_dir + # Keyed by infile rather than by the TestCase itself: TestCase is a frozen dataclass + # with a list field, so it isn't hashable. + self._futures: dict[Path, Future[list[tuple[str, str | None]]]] = {} + + def precompute(self, testdata: TestDataGroup, context: Context) -> None: + """Submit one background job per testcase matching context.data_filter. + + May be called at most once. A no-op if context.executor is None.""" + assert not self._futures, 'precompute() called more than once' + if context.executor is None: + return + for testcase in testdata.get_all_testcases(): + if testcase.matches_filter(context.data_filter): + self._futures[testcase.infile] = context.submit_background_work( + _compute_testcase_input_errors, self._validators, testcase, self._work_dir + ) + + def check(self, testcase: TestCase, diag: Diagnostics) -> None: + """Report the errors (if any) input validation found for testcase. + + Blocks on the background job for testcase if precompute() started one and it's + still running; otherwise (no precompute(), or testcase was filtered out of it) + computes the result synchronously.""" + future = self._futures.get(testcase.infile) + errors = ( + future.result() if future is not None else _compute_testcase_input_errors(self._validators, testcase, self._work_dir) + ) + for msg, additional_info in errors: + diag.error(msg, additional_info) def check_output_validators( diff --git a/problemtools/context.py b/problemtools/context.py index d5bf445b..eb87efbd 100644 --- a/problemtools/context.py +++ b/problemtools/context.py @@ -5,7 +5,7 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from re import Pattern -from typing import ParamSpec, TypeVar +from typing import Any, ParamSpec, TypeVar _T = TypeVar('_T') _P = ParamSpec('_P') @@ -29,11 +29,13 @@ def __init__( self.fixed_timelim = fixed_timelim self.parts: list[str] = parts if parts is not None else list(PROBLEM_PARTS) self.executor: ThreadPoolExecutor | None = ThreadPoolExecutor(threads) if threads > 1 else None - self._background_work: list[concurrent.futures.Future[object]] = [] + self._background_work: list[concurrent.futures.Future[Any]] = [] - def submit_background_work(self, job: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> None: + def submit_background_work(self, job: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> concurrent.futures.Future[_T]: assert self.executor - self._background_work.append(self.executor.submit(job, *args, **kwargs)) + future = self.executor.submit(job, *args, **kwargs) + self._background_work.append(future) + return future def cancel_background_work(self) -> None: for future in self._background_work: