From 9c9233013d0dc62eb29e4a805a021daf02f4fce5 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 28 Aug 2026 10:52:16 +0200 Subject: [PATCH 1/3] Create model for Problem --- problemtools/model/__init__.py | 3 ++ problemtools/model/problem.py | 80 ++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 problemtools/model/problem.py diff --git a/problemtools/model/__init__.py b/problemtools/model/__init__.py index 185c6a4c..3f54f97d 100644 --- a/problemtools/model/__init__.py +++ b/problemtools/model/__init__.py @@ -6,6 +6,7 @@ from .attachments import Attachments, load_attachments from .graders import DEFAULT_GRADER, Graders, load_graders from .includes import DEFAULT_LANGUAGE, IncludeFile, Includes, LanguageIncludes, load_includes +from .problem import Problem, load_problem from .statements import Statements, load_statements from .submissions import LegacyPolicy, Submission, Submissions, load_submissions from .testdata import DEFAULT_CONFIG, SCORING_ONLY_KEYS, TestCase, TestDataGroup, load_testdata @@ -25,6 +26,7 @@ 'LanguageIncludes', 'LegacyPolicy', 'OutputValidators', + 'Problem', 'Statements', 'Submission', 'Submissions', @@ -36,6 +38,7 @@ 'load_includes', 'load_input_validators', 'load_output_validators', + 'load_problem', 'load_statements', 'load_submissions', 'load_testdata', diff --git a/problemtools/model/problem.py b/problemtools/model/problem.py new file mode 100644 index 00000000..4933390e --- /dev/null +++ b/problemtools/model/problem.py @@ -0,0 +1,80 @@ +"""Model for a whole problem package: the aggregate of all its parts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..diagnostics import Diagnostics, VerifyError +from ..formatversion import FormatVersion +from ..languages import Languages, load_language_config +from ..metadata import Metadata, load_metadata +from .attachments import Attachments, load_attachments +from .graders import Graders, load_graders +from .includes import Includes, load_includes +from .statements import Statements, load_statements +from .submissions import Submissions, load_submissions +from .testdata import TestDataGroup, load_testdata +from .validators import InputValidators, OutputValidators, load_input_validators, load_output_validators + + +@dataclass(frozen=True) +class Problem: + """A statically loaded problem package. + + Loading is static parsing with very little verification (it does not, e.g., + compile validators or run submissions) -- see the `checks` package for the + checks that should be run on top of a loaded Problem. + """ + + probdir: Path + language_config: Languages + metadata: Metadata + statements: Statements + attachments: Attachments + input_validators: InputValidators + output_validators: OutputValidators + graders: Graders + testdata: TestDataGroup + includes: Includes + submissions: Submissions + + @property + def shortname(self) -> str: + return self.probdir.name + + @property + def format_version(self) -> FormatVersion: + return self.metadata.problem_format_version + + +def load_problem(probdir: Path, diag: Diagnostics) -> Problem: + """Loads a problem package from probdir. + + On failure, reports errors via `diag` and raises VerifyError. + """ + try: + language_config = load_language_config(probdir.parent) + problem_metadata = load_metadata(probdir) + format_version = problem_metadata.problem_format_version + + includes = load_includes(probdir, language_config) + return Problem( + probdir=probdir, + language_config=language_config, + metadata=problem_metadata, + statements=load_statements(probdir, format_version), + attachments=load_attachments(probdir), + input_validators=load_input_validators(probdir, language_config), + output_validators=load_output_validators(probdir, format_version, language_config), + graders=load_graders(probdir, language_config), + testdata=load_testdata(probdir, problem_metadata), + includes=includes, + # Submissions need includes, so includes must be loaded first. + submissions=load_submissions(probdir, language_config, includes), + ) + except VerifyError: + # A loader already reported its own diagnostic; don't double-report. + raise + except Exception as e: + diag.fatal(f'Failed to load problem: {e}') From ebf21f80c292f6b9c0f123849c5f35c25154673e Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 28 Aug 2026 12:39:44 +0200 Subject: [PATCH 2/3] Rewrite verifyproblem to use model + checkers #398 --- problemtools/context.py | 3 +- problemtools/verifyproblem.py | 705 ++++++++++------------------------ tests/test_verify_hello.py | 38 +- 3 files changed, 221 insertions(+), 525 deletions(-) diff --git a/problemtools/context.py b/problemtools/context.py index 8b781ba7..d5bf445b 100644 --- a/problemtools/context.py +++ b/problemtools/context.py @@ -10,7 +10,8 @@ _T = TypeVar('_T') _P = ParamSpec('_P') -PROBLEM_PARTS = ['config', 'data', 'graders', 'statement', 'submissions', 'validators'] +#: Listed in the order they're checked in. +PROBLEM_PARTS = ['config', 'statement', 'validators', 'graders', 'data', 'submissions'] class Context: diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index b3575fa3..6bec6fb5 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -2,478 +2,67 @@ from __future__ import annotations import argparse +import functools import logging -import os import random import re import shutil import sys import tempfile -from abc import ABC +from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path from re import Pattern from types import TracebackType -from typing import ClassVar, NoReturn, Self +from typing import Self -from pydantic import ValidationError - -from . import checks, languages, metadata, model, run +from . import checks, model, run from .context import PROBLEM_PARTS, Context from .diagnostics import Diagnostics, LoggingDiagnostics, VerifyError -from .formatversion import FormatVersion, get_format_version +from .formatversion import FormatVersion from .version import add_version_arg random.seed(42) -class ProblemAspect(ABC): - _check_res: bool | None = None - problem: Problem - _diag: Diagnostics - - def __init__(self, name: str, problem: Problem) -> None: - if self is not problem: - self._diag = problem._diag.child(name) - self.problem = problem - - @property - def errors(self) -> int: - return self._diag.errors - - @property - def warnings(self) -> int: - return self._diag.warnings - - def fatal(self, msg: str, additional_info: str | None = None) -> NoReturn: - self._check_res = False - self._diag.fatal(msg, additional_info) - - def error(self, msg: str, additional_info: str | None = None) -> None: - self._check_res = False - self._diag.error(msg, additional_info) - - def warning(self, msg: str, additional_info: str | None = None) -> None: - self._diag.warning(msg, additional_info) - - def info(self, msg: str) -> None: - self._diag.info(msg) - - def debug(self, msg: str) -> None: - self._diag.debug(msg) - - def msg(self, msg: str) -> None: - print(msg) - - -class ProblemPart(ProblemAspect): - """Baseclass for all parts that can be included in a problem-format.""" - - """Should always be overridden by the subclass. Specifies the name that will be used to refer - to the part e.g for logs. - """ - PART_NAME: ClassVar[str] - - def __init__(self, problem: Problem) -> None: - if self.PART_NAME is None: - raise NotImplementedError('Every problem-part must override PART_NAME') - super().__init__(self.PART_NAME, problem) - self.setup() - - def setup(self) -> None: - pass - - def start_background_work(self, context: Context) -> None: - pass - - def check(self, context: Context) -> bool: - return True - - -class ProblemStatement(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'statement' - - def setup(self) -> None: - self.debug(' Loading problem statement') - self.statements = model.load_statements(Path(self.problem.probdir), self.problem.format) - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_statements( - self.statements, - self.problem.metadata, - self.problem.format, - Path(self.problem.probdir), - self.problem.tmpdir, - self._diag, - ) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - def __str__(self) -> str: - return 'problem statement' - - -class ProblemConfig(ProblemPart): - PART_NAME = 'config' - - def setup(self) -> None: - self.debug(' Loading problem config') - try: - self._metadata = metadata.load_metadata(Path(self.problem.probdir)) - self.problem._set_metadata(self._metadata) - except ValidationError as e: - error_str = '\n'.join([f' {"->".join(str(loc) for loc in err["loc"])}: {err["msg"]}' for err in e.errors()]) - self.fatal(f'Failed parsing problem.yaml. Found {len(e.errors())} errors:\n{error_str}') - except Exception as e: - self.fatal(f'Failed loading problem configuration: {e}') - - def __str__(self) -> str: - return 'problem configuration' - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_config( - self._metadata, - self.problem.format, - self.problem.statement.statements, - self.problem.testdata.testdata, - self._diag, - ) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - -class Attachments(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'attachments' - - def setup(self) -> None: - self.attachments = model.load_attachments(Path(self.problem.probdir)) - self.debug(f'Adding attachments {self.attachments.paths!s}') - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_attachments(self.attachments, self._diag) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - def get_attachment_paths(self) -> list[Path]: - return self.attachments.paths - - def __str__(self) -> str: - return 'attachments' - - -class InputValidators(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'input_validator' - - def setup(self) -> None: - self.input_validators = model.load_input_validators(Path(self.problem.probdir), self.problem.language_config) - - def __str__(self) -> str: - return 'input format validators' - - def start_background_work(self, context: Context) -> None: - for val in self.input_validators.validators: - context.submit_background_work(val.compile, Path(self.problem.tmpdir)) - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_input_validators(self.input_validators, self.problem.testdata.testdata, self.problem.tmpdir, self._diag) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - -class Graders(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'grader' - - def setup(self) -> None: - self.graders = model.load_graders(Path(self.problem.probdir), self.problem.language_config) - - def __str__(self) -> str: - return 'graders' - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True +@dataclass(frozen=True) +class _CheckStep: + """One named check to run as part of verifying a problem. - errors_before = self.errors - checks.check_graders(self.graders, self.problem.metadata, self.problem.tmpdir, self._diag) - if self.errors > errors_before: - self._check_res = False + `part` is one of PROBLEM_PARTS -- it groups steps for the -p/--parts CLI + filter and for the "Checking X" progress messages. `name` scopes the + diagnostics `run` reports to (via Diagnostics.child).""" - return self._check_res + part: str + name: str + run: Callable[[Diagnostics], None] + start_background_work: Callable[[Context], None] | None = None -class OutputValidators(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" +@dataclass(frozen=True) +class CheckResult: + """Result of ProblemVerifier.check().""" - PART_NAME = 'output_validator' + errors: int + warnings: int + #: Time limit computed from accepted submissions' runtimes, if the 'submissions' + #: part was checked and had at least one accepted submission to measure. + timelim: float | None - def setup(self) -> None: - self.output_validators = model.load_output_validators( - Path(self.problem.probdir), self.problem.format, self.problem.language_config - ) - self._has_precompiled = False - - @property - def output_validator(self) -> run.Program: - validator = self.output_validators.select(self.problem.format, self.problem.metadata) - if validator is None: - self.fatal('Unable to locate default validator') - return validator - - def __str__(self) -> str: - return 'output validators' - - def start_background_work(self, context: Context) -> None: - if not self._has_precompiled: - context.submit_background_work(self.output_validator.compile, Path(self.problem.tmpdir)) - self._has_precompiled = True - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_output_validators( - self.output_validators, - self.problem.format, - self.problem.metadata, - self.problem.testdata.testdata, - Path(self.problem.probdir), - self.problem.tmpdir, - self._diag, - ) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - -class Includes(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'includes' - - def setup(self) -> None: - self.includes = model.load_includes(Path(self.problem.probdir), self.problem.language_config) - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_includes(self.includes, self.problem.language_config, self.problem.format, self._diag) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - def __str__(self) -> str: - return 'includes' - - -class Submissions(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'submission' - - def setup(self) -> None: - self.submissions = model.load_submissions( - Path(self.problem.probdir), self.problem.language_config, self.problem.includes.includes - ) - - def __str__(self) -> str: - return 'submissions' - - def start_background_work(self, context: Context) -> None: - # Send off an early background compile job for each submission and - # validator, to avoid a bottleneck step at the start of each test run. - self.problem.output_validators.start_background_work(context) - policy = self.submissions.policy - for sub in self.submissions.submissions: - if policy.matches(sub) and context.submission_filter.search(str(sub.path)): - context.submit_background_work(sub.program.compile, Path(self.problem.tmpdir)) - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - checks.check_submissions( - self.submissions, - self.problem.metadata, - self.problem.testdata.testdata, - self.problem.output_validators.output_validator, - self.problem.graders.graders, - self.problem.tmpdir, - Path(self.problem.probdir), - context, - self.problem._set_timelim, - self._diag, - ) - if self.errors > errors_before: - self._check_res = False - - return self._check_res - - -class TestData(ProblemPart): - """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" - - PART_NAME = 'data' - - def setup(self) -> None: - self.testdata = model.load_testdata(Path(self.problem.probdir), self.problem.metadata) - - def __str__(self) -> str: - return 'test data' - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - errors_before = self.errors - - checks.check_testdata( - self.testdata, - context, - self.problem.metadata, - Path(self.problem.probdir), - self.problem.graders.graders, - self.problem.input_validators.input_validators, - self.problem.output_validators.output_validators, - self.problem.format, - self.problem.tmpdir, - self._diag, - ) - if self.errors > errors_before: - self._check_res = False - - return self._check_res +class ProblemVerifier: + """Runs checks against a loaded problem. -class Problem(ProblemAspect): - """Represents a checkable problem""" + Owns the temporary work directory used to compile and run programs while + checking -- use as a context manager.""" - def __init__(self, probdir: str, diagnostics: Diagnostics): - self.probdir = os.path.realpath(probdir) - self.shortname: str = os.path.basename(self.probdir) - self._diag = diagnostics - super().__init__(self.shortname, self) - self.language_config = languages.load_language_config(Path(self.probdir).parent) - self.loaded = False - self._metadata: metadata.Metadata | None = None - self._timelim: float | None = None - - # Unfortunately must be before metadata, otherwise mypy gets confused about the type metadata.Metadata (feels like a bug) - def _set_metadata(self, metadata: metadata.Metadata) -> None: # Should only be called by ProblemConfig - assert self._metadata is None, 'Attempted to set metadata twice' - self._metadata = metadata - - @property - def metadata(self) -> metadata.Metadata: - assert self._metadata is not None, 'Attempted to access config before it was set. load() or check() first.' - return self._metadata - - @property - def timelim(self) -> float: - assert self._timelim is not None, 'Attempted to access timelim before it was set. check() first.' - return self._timelim - - def _set_timelim(self, timelim: float) -> None: # Should only be called by Submissions - assert self._timelim is None, 'Attempted to set timelim twice' - self._timelim = timelim - - def is_pass_fail(self) -> bool: - return self.metadata.is_pass_fail() - - def is_scoring(self) -> bool: - return self.metadata.is_scoring() - - def is_interactive(self) -> bool: - return self.metadata.is_interactive() - - def is_multi_pass(self) -> bool: - return self.metadata.is_multi_pass() - - def is_submit_answer(self) -> bool: - return self.metadata.is_submit_answer() - - def load(self) -> None: - """Parses the problem package statically, loading up information with very little verification. - - Call this if you want to get a usable Problem object without expensive - steps (such as compiling validators, and testing submissions). - - N.B., This api is EXPERIMENTAL. We eventually want to create a stable - API from problemtools, this is a first move in that direction. - - Raises: - VerifyError: if problem package is too broken to parse safely - """ - - if self.loaded: - return - - if not os.path.isdir(self.probdir): - self.fatal(f"Problem directory '{self.probdir}' not found") - - try: - self.format = get_format_version(Path(self.probdir)) - except Exception as e: - self.fatal(f'Failed loading problem version: {e}') - self.config = ProblemConfig(self) # Populates self.metadata as a side effect. Needs to run first. - self.statement = ProblemStatement(self) - self.attachments = Attachments(self) - self.input_validators = InputValidators(self) - self.output_validators = OutputValidators(self) - self.graders = Graders(self) - self.testdata = TestData(self) - self.includes = Includes(self) - # Submissions.setup() reads self.includes.includes, so includes must be loaded first. - self.submissions = Submissions(self) - self.loaded = True + def __init__(self, problem: model.Problem, diag: Diagnostics) -> None: + self.problem = problem + self._diag = diag + self.work_dir: Path | None = None def __enter__(self) -> Self: - self.tmpdir = tempfile.mkdtemp(prefix=f'verify-{self.shortname}-') + self.work_dir = Path(tempfile.mkdtemp(prefix=f'verify-{self.problem.shortname}-')) return self def __exit__( @@ -482,61 +71,54 @@ def __exit__( exc_value: BaseException | None, exc_traceback: TracebackType | None, ) -> None: - shutil.rmtree(self.tmpdir) + assert self.work_dir is not None + shutil.rmtree(self.work_dir) + self.work_dir = None + + def check(self, context: Context) -> CheckResult: + """Runs checks on the problem. ProblemVerifier must be entered (as a context manager) first.""" + assert self.work_dir is not None, 'ProblemVerifier.check() called before __enter__' + problem = self.problem + diag = self._diag + timelim: float | None = None + + def set_timelim(value: float) -> None: + nonlocal timelim + timelim = value + + @functools.cache + def get_output_validator() -> run.Program: + validator = problem.output_validators.select(problem.format_version, problem.metadata) + if validator is None: + diag.fatal('Unable to locate default validator') + return validator - def __str__(self) -> str: - return str(self.shortname) - - def check(self, context: Context) -> tuple[int, int]: - """Loads and checks the problem package + try: + if not re.match('^[a-z0-9]+$', problem.shortname): + diag.error(f"Invalid shortname '{problem.shortname}' (must be [a-z0-9]+)") + if problem.format_version is FormatVersion.V_2023_07: + diag.warning( + f'Support for version {problem.format_version} is very incomplete. Verification may not work as expected.' + ) - Loads the problem package and runs checks. After this has completed, - the Problem object is fully populated. You do not need to manually - run load() first. + checks.check_problem_package(problem.probdir, problem.format_version, diag) + run.limit.check_limit_capabilities(diag) - Returns: - Tuple with the number of errors, warnings found. + steps = self._build_steps(context, get_output_validator, set_timelim) + assert {step.part for step in steps} == set(PROBLEM_PARTS), 'CheckStep parts and PROBLEM_PARTS must be kept in sync' - Raises: - VerifyError: if problem package is too broken to parse safely - """ - try: - self.load() - except VerifyError: - return self.errors, self.warnings + active_parts = [part for part in PROBLEM_PARTS if part in context.parts] - try: - part_mapping: dict[str, list] = { - 'config': [self.config], - 'statement': [self.statement, self.attachments], - 'validators': [self.input_validators, self.output_validators], - 'graders': [self.graders], - 'data': [self.testdata], - 'submissions': [self.includes, self.submissions], - } - assert sorted(part_mapping.keys()) == sorted(PROBLEM_PARTS), 'part_mapping and PROBLEM_PARTS must be kept in sync' - - if not re.match('^[a-z0-9]+$', self.shortname): - self.error(f"Invalid shortname '{self.shortname}' (must be [a-z0-9]+)") - if self.format is FormatVersion.V_2023_07: - self.warning(f'Support for version {self.format} is very incomplete. Verification may not work as expected.') - - checks.check_problem_package(Path(self.probdir), self.format, self._diag) - - run.limit.check_limit_capabilities(self._diag) - - parts = [ - part for part in part_mapping if part in context.parts - ] # Parts from context in the order they appear in part_mapping if context.executor: - for part in parts: - for item in part_mapping[part]: - item.start_background_work(context) - - for part in parts: - self.msg(f'Checking {part}') - for item in part_mapping[part]: - item.check(context) + for step in steps: + if step.part in active_parts and step.start_background_work: + step.start_background_work(context) + + for part in active_parts: + print(f'Checking {part}') + for step in steps: + if step.part == part: + step.run(diag.child(step.name)) except VerifyError: pass except KeyboardInterrupt: @@ -548,7 +130,116 @@ def check(self, context: Context) -> tuple[int, int]: # Wait for background work to finish before performing an rmtree on # the directory tree it uses. context.wait_for_background_work() - return self.errors, self.warnings + + return CheckResult(errors=diag.errors, warnings=diag.warnings, timelim=timelim) + + def _build_steps( + self, + context: Context, + get_output_validator: Callable[[], run.Program], + set_timelim: Callable[[float], None], + ) -> list[_CheckStep]: + problem = self.problem + assert self.work_dir is not None + work_dir = self.work_dir + + def start_input_validators(context: Context) -> None: + for validator in problem.input_validators.validators: + context.submit_background_work(validator.compile, work_dir) + + def start_output_validator(context: Context) -> None: + context.submit_background_work(get_output_validator().compile, work_dir) + + def start_submissions(context: Context) -> None: + # Precompile the output validator here too: submissions need it, and this step + # runs even if the 'validators' part wasn't requested. Program.compile() caches + # its result, so this is harmless if 'validators' already queued the same compile. + context.submit_background_work(get_output_validator().compile, work_dir) + policy = problem.submissions.policy + for sub in problem.submissions.submissions: + if policy.matches(sub) and context.submission_filter.search(str(sub.path)): + context.submit_background_work(sub.program.compile, work_dir) + + return [ + _CheckStep( + 'config', + 'config', + lambda diag: checks.check_config( + problem.metadata, problem.format_version, problem.statements, problem.testdata, diag + ), + ), + _CheckStep( + 'statement', + 'statement', + lambda diag: checks.check_statements( + problem.statements, problem.metadata, problem.format_version, problem.probdir, str(work_dir), diag + ), + ), + _CheckStep('statement', 'attachments', lambda diag: checks.check_attachments(problem.attachments, diag)), + _CheckStep( + 'validators', + 'input_validator', + lambda diag: checks.check_input_validators(problem.input_validators, problem.testdata, str(work_dir), diag), + start_input_validators, + ), + _CheckStep( + 'validators', + 'output_validator', + lambda diag: checks.check_output_validators( + problem.output_validators, + problem.format_version, + problem.metadata, + problem.testdata, + problem.probdir, + str(work_dir), + diag, + ), + start_output_validator, + ), + _CheckStep( + 'graders', + 'grader', + lambda diag: checks.check_graders(problem.graders, problem.metadata, str(work_dir), diag), + ), + _CheckStep( + 'data', + 'data', + lambda diag: checks.check_testdata( + problem.testdata, + context, + problem.metadata, + problem.probdir, + problem.graders, + problem.input_validators, + problem.output_validators, + problem.format_version, + str(work_dir), + diag, + ), + ), + _CheckStep( + 'submissions', + 'includes', + lambda diag: checks.check_includes(problem.includes, problem.language_config, problem.format_version, diag), + ), + _CheckStep( + 'submissions', + 'submission', + lambda diag: checks.check_submissions( + problem.submissions, + problem.metadata, + problem.testdata, + get_output_validator(), + problem.graders, + str(work_dir), + problem.probdir, + context, + set_timelim, + diag, + ), + start_submissions, + ), + ] def re_argument(s: str) -> Pattern[str]: @@ -609,7 +300,7 @@ def argparser() -> argparse.ArgumentParser: type=part_argument, nargs='+', default=PROBLEM_PARTS, - help=f'only test the indicated parts of the problem. Each PROBLEM_PART can be one of {PROBLEM_PARTS}.', + help=f'only test the indicated parts of the problem. Each PROBLEM_PART can be one of {sorted(PROBLEM_PARTS)}.', ) parser.add_argument( '-j', @@ -639,7 +330,8 @@ def main() -> None: threads=args.threads, ) for problemdir in args.problemdir: - shortname = os.path.basename(os.path.realpath(problemdir)) + probdir = Path(problemdir).resolve() + shortname = probdir.name print(f'Loading problem {shortname}') diag = LoggingDiagnostics.create( shortname, @@ -648,14 +340,19 @@ def main() -> None: warnings_as_errors=args.werror, max_additional_info=args.max_additional_info, ) - with Problem(problemdir, diag) as prob: - errors, warnings = prob.check(context) - - def p(x: int) -> str: - return '' if x == 1 else 's' - - print(f'{prob.shortname} tested: {errors} error{p(errors)}, {warnings} warning{p(warnings)}') - total_errors += errors + try: + problem = model.load_problem(probdir, diag) + except VerifyError: + result = CheckResult(errors=diag.errors, warnings=diag.warnings, timelim=None) + else: + with ProblemVerifier(problem, diag) as verifier: + result = verifier.check(context) + + def p(x: int) -> str: + return '' if x == 1 else 's' + + print(f'{shortname} tested: {result.errors} error{p(result.errors)}, {result.warnings} warning{p(result.warnings)}') + total_errors += result.errors except KeyboardInterrupt: print('\naborting...') diff --git a/tests/test_verify_hello.py b/tests/test_verify_hello.py index 29bc00c3..95a48935 100644 --- a/tests/test_verify_hello.py +++ b/tests/test_verify_hello.py @@ -1,7 +1,7 @@ import logging import pathlib -import problemtools.verifyproblem as verify +from problemtools import checks, model from problemtools.diagnostics import LoggingDiagnostics @@ -10,28 +10,26 @@ def _make_diag(shortname: str) -> LoggingDiagnostics: def test_load_hello(): - directory = pathlib.Path(__file__).parent / 'hello' - string = str(directory.resolve()) + probdir = (pathlib.Path(__file__).parent / 'hello').resolve() - context = verify.Context() + diag = _make_diag('hello') + problem = model.load_problem(probdir, diag) + assert problem.shortname == 'hello' - with verify.Problem(string, _make_diag('hello')) as p: - p.load() - assert p.shortname == 'hello' - # pytest and fork don't go along very well, so just run aspects that work without run - assert p.config.check(context) - assert p.attachments.check(context) - assert p.is_pass_fail() - assert not p.is_scoring() - assert not p.is_interactive() - assert not p.is_multi_pass() - assert not p.is_submit_answer() + # pytest and fork don't go along very well, so just run checks that work without run + checks.check_config(problem.metadata, problem.format_version, problem.statements, problem.testdata, diag) + checks.check_attachments(problem.attachments, diag) + assert diag.errors == 0 + + assert problem.metadata.is_pass_fail() + assert not problem.metadata.is_scoring() + assert not problem.metadata.is_interactive() + assert not problem.metadata.is_multi_pass() + assert not problem.metadata.is_submit_answer() def test_load_twice(): - directory = pathlib.Path(__file__).parent / 'hello' - string = str(directory.resolve()) + probdir = (pathlib.Path(__file__).parent / 'hello').resolve() - with verify.Problem(string, _make_diag('hello')) as p: - p.load() - p.load() + model.load_problem(probdir, _make_diag('hello')) + model.load_problem(probdir, _make_diag('hello')) From 400d9e07cfb872dafd5681f64bfe19a8cfec953e Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 28 Aug 2026 13:15:33 +0200 Subject: [PATCH 3/3] Adjust type of work_dir to Path in checks to avoid silly casts to and from str --- problemtools/checks/graders.py | 4 ++-- problemtools/checks/statements.py | 2 +- problemtools/checks/submissions.py | 10 +++++----- problemtools/checks/testdata.py | 8 ++++---- problemtools/checks/validators.py | 20 ++++++++++---------- problemtools/verifyproblem.py | 12 ++++++------ 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/problemtools/checks/graders.py b/problemtools/checks/graders.py index 087205ea..f8920676 100644 --- a/problemtools/checks/graders.py +++ b/problemtools/checks/graders.py @@ -9,7 +9,7 @@ from ..model import Graders -def check_graders(graders: Graders, metadata: Metadata, work_dir: str, diag: Diagnostics) -> None: +def check_graders(graders: Graders, metadata: Metadata, work_dir: Path, diag: Diagnostics) -> None: """Run all checks on a problem's custom graders.""" if len(graders.graders) > 1: diag.fatal('There is more than one custom grader') @@ -21,6 +21,6 @@ def check_graders(graders: Graders, metadata: Metadata, work_dir: str, diag: Dia if metadata.is_pass_fail(): diag.fatal('There is a grader but the problem is pass-fail') - result = grader.compile(Path(work_dir)) + result = grader.compile(work_dir) if not result.success: diag.fatal(f'Compile error for {grader}', result.errmsg) diff --git a/problemtools/checks/statements.py b/problemtools/checks/statements.py index 3ebc0f0c..701ba81c 100644 --- a/problemtools/checks/statements.py +++ b/problemtools/checks/statements.py @@ -30,7 +30,7 @@ def check_statements( metadata: Metadata, format: FormatVersion, probdir: Path, - work_dir: str, + work_dir: Path, diag: Diagnostics, ) -> None: """Run all checks on a problem's statements.""" diff --git a/problemtools/checks/submissions.py b/problemtools/checks/submissions.py index 7971d6d3..182ed7ca 100644 --- a/problemtools/checks/submissions.py +++ b/problemtools/checks/submissions.py @@ -32,7 +32,7 @@ def check_submissions( testdata: TestDataGroup, output_validator: Program, graders: Graders, - tmpdir: str, + work_dir: Path, probdir: Path, context: Context, set_timelim: Callable[[float], None], @@ -79,7 +79,7 @@ def check_submissions( ) continue - result = sub.program.compile(Path(tmpdir)) + result = sub.program.compile(work_dir) if not result.success: diag.error(f'Compile error for {label} submission {sub.program}', additional_info=result.errmsg) continue @@ -94,7 +94,7 @@ def check_submissions( testdata, output_validator, graders, - tmpdir, + work_dir, probdir, seen_oob_score_groups, timelim, @@ -163,7 +163,7 @@ def _check_submission( testdata: TestDataGroup, output_validator: Program, graders: Graders, - tmpdir: str, + work_dir: Path, probdir: Path, seen_oob_score_groups: set[int], timelim: float, @@ -180,7 +180,7 @@ def _check_submission( output_validator=output_validator, metadata=metadata, root=testdata, - base_dir=Path(tmpdir), + base_dir=work_dir, context=context, graders=graders, diag=diag, diff --git a/problemtools/checks/testdata.py b/problemtools/checks/testdata.py index e167bf1b..c70456f8 100644 --- a/problemtools/checks/testdata.py +++ b/problemtools/checks/testdata.py @@ -36,7 +36,7 @@ def check_testdata( input_validators: InputValidators, output_validators: OutputValidators, format: FormatVersion, - work_dir: str, + work_dir: Path, diag: Diagnostics, ) -> None: """Run all checks on a problem's test data.""" @@ -70,7 +70,7 @@ def _check_group( has_default_grader: bool, input_validators: InputValidators, output_validator: Program, - work_dir: str, + work_dir: Path, diag: Diagnostics, ) -> None: if group.config['grading'] not in ['default', 'custom']: @@ -233,7 +233,7 @@ def _check_testcase( metadata: Metadata, input_validators: InputValidators, output_validator: Program, - work_dir: str, + work_dir: Path, diag: Diagnostics, ) -> None: _check_newlines(testcase.infile, diag) @@ -257,7 +257,7 @@ def _check_testcase( submission_output=testcase.ansfile, output_validator=output_validator, metadata=metadata, - base_dir=Path(work_dir), + base_dir=work_dir, diag=diag, ) if val_res.verdict != 'AC': diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index c862d4f9..3d9933af 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -93,7 +93,7 @@ def _error_in_2023_07(format: FormatVersion, diag: Diagnostics, msg: str, additi diag.error(msg, additional_info) -def check_input_validators(validators: InputValidators, testdata: TestDataGroup, work_dir: str, diag: Diagnostics) -> None: +def check_input_validators(validators: InputValidators, testdata: TestDataGroup, work_dir: Path, diag: Diagnostics) -> None: """Run all checks on a problem's input format validators.""" if validators.uses_old_path: diag.warning('input_format_validators is a deprecated name; please use input_validators instead') @@ -104,7 +104,7 @@ def check_input_validators(validators: InputValidators, testdata: TestDataGroup, for val in validators.validators: try: - result = val.compile(Path(work_dir)) + result = val.compile(work_dir) if not result.success: diag.error(f'Compile error for {val}', result.errmsg) except ProgramError as e: @@ -132,7 +132,7 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None: for flags_str in all_flags: flags = flags_str.split() for val in validators.validators: - status, _ = val.run(file_name, args=flags, work_dir=Path(work_dir)) + status, _ = val.run(file_name, args=flags, work_dir=work_dir) if os.WEXITSTATUS(status) != 42: break else: @@ -154,7 +154,7 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab for flags_str in all_flags: flags = flags_str.split() for val in validators.validators: - status, _ = val.run(file_name, args=flags, work_dir=Path(work_dir)) + status, _ = val.run(file_name, args=flags, work_dir=work_dir) if os.WEXITSTATUS(status) != 42: # expected behavior; validator rejects modified input return False @@ -173,17 +173,17 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab os.unlink(file_name) -def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: str, diag: Diagnostics) -> None: +def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: Path, diag: Diagnostics) -> None: """Run the (already checked) input validators against a single test case's input file.""" flags = testcase.input_validator_flags for val in validators.validators: # A validator that failed to compile was already reported by check_input_validators; skip it. - if not val.compile(Path(work_dir)).success: + if not val.compile(work_dir).success: continue with tempfile.NamedTemporaryFile() as outfile, tempfile.NamedTemporaryFile() as errfile: - status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=Path(work_dir)) + status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=work_dir) if not os.WIFEXITED(status): emsg = f'Input format validator {val} crashed on input {testcase.infile}' elif os.WEXITSTATUS(status) != 42: @@ -202,7 +202,7 @@ def check_output_validators( metadata: Metadata, testdata: TestDataGroup, probdir: Path, - work_dir: str, + work_dir: Path, diag: Diagnostics, ) -> None: """Run all checks on a problem's output validators.""" @@ -233,7 +233,7 @@ def check_output_validators( diag.fatal('problem.yaml specifies custom validator but no validator programs found') try: - result = selected.compile(Path(work_dir)) + result = selected.compile(work_dir) if not result.success: diag.fatal(f'Compile error for output validator {selected}', result.errmsg) except ProgramError as e: @@ -254,7 +254,7 @@ def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[TestCase] submission_output=Path(f.name), output_validator=selected, metadata=metadata, - base_dir=Path(work_dir), + base_dir=work_dir, diag=diag, ) results.append(result) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 6bec6fb5..d38395a6 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -172,14 +172,14 @@ def start_submissions(context: Context) -> None: 'statement', 'statement', lambda diag: checks.check_statements( - problem.statements, problem.metadata, problem.format_version, problem.probdir, str(work_dir), diag + problem.statements, problem.metadata, problem.format_version, problem.probdir, work_dir, diag ), ), _CheckStep('statement', 'attachments', lambda diag: checks.check_attachments(problem.attachments, diag)), _CheckStep( 'validators', 'input_validator', - lambda diag: checks.check_input_validators(problem.input_validators, problem.testdata, str(work_dir), diag), + lambda diag: checks.check_input_validators(problem.input_validators, problem.testdata, work_dir, diag), start_input_validators, ), _CheckStep( @@ -191,7 +191,7 @@ def start_submissions(context: Context) -> None: problem.metadata, problem.testdata, problem.probdir, - str(work_dir), + work_dir, diag, ), start_output_validator, @@ -199,7 +199,7 @@ def start_submissions(context: Context) -> None: _CheckStep( 'graders', 'grader', - lambda diag: checks.check_graders(problem.graders, problem.metadata, str(work_dir), diag), + lambda diag: checks.check_graders(problem.graders, problem.metadata, work_dir, diag), ), _CheckStep( 'data', @@ -213,7 +213,7 @@ def start_submissions(context: Context) -> None: problem.input_validators, problem.output_validators, problem.format_version, - str(work_dir), + work_dir, diag, ), ), @@ -231,7 +231,7 @@ def start_submissions(context: Context) -> None: problem.testdata, get_output_validator(), problem.graders, - str(work_dir), + work_dir, problem.probdir, context, set_timelim,