Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions problemtools/checks/graders.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from ..model import Graders


def check_graders(graders: Graders, metadata: Metadata, work_dir: str, diag: Diagnostics) -> None:
def check_graders(graders: Graders, metadata: Metadata, work_dir: Path, diag: Diagnostics) -> None:
"""Run all checks on a problem's custom graders."""
if len(graders.graders) > 1:
diag.fatal('There is more than one custom grader')
Expand All @@ -21,6 +21,6 @@ def check_graders(graders: Graders, metadata: Metadata, work_dir: str, diag: Dia
if metadata.is_pass_fail():
diag.fatal('There is a grader but the problem is pass-fail')

result = grader.compile(Path(work_dir))
result = grader.compile(work_dir)
if not result.success:
diag.fatal(f'Compile error for {grader}', result.errmsg)
2 changes: 1 addition & 1 deletion problemtools/checks/statements.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def check_statements(
metadata: Metadata,
format: FormatVersion,
probdir: Path,
work_dir: str,
work_dir: Path,
diag: Diagnostics,
) -> None:
"""Run all checks on a problem's statements."""
Expand Down
10 changes: 5 additions & 5 deletions problemtools/checks/submissions.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def check_submissions(
testdata: TestDataGroup,
output_validator: Program,
graders: Graders,
tmpdir: str,
work_dir: Path,
probdir: Path,
context: Context,
set_timelim: Callable[[float], None],
Expand Down Expand Up @@ -79,7 +79,7 @@ def check_submissions(
)
continue

result = sub.program.compile(Path(tmpdir))
result = sub.program.compile(work_dir)
if not result.success:
diag.error(f'Compile error for {label} submission {sub.program}', additional_info=result.errmsg)
continue
Expand All @@ -94,7 +94,7 @@ def check_submissions(
testdata,
output_validator,
graders,
tmpdir,
work_dir,
probdir,
seen_oob_score_groups,
timelim,
Expand Down Expand Up @@ -163,7 +163,7 @@ def _check_submission(
testdata: TestDataGroup,
output_validator: Program,
graders: Graders,
tmpdir: str,
work_dir: Path,
probdir: Path,
seen_oob_score_groups: set[int],
timelim: float,
Expand All @@ -180,7 +180,7 @@ def _check_submission(
output_validator=output_validator,
metadata=metadata,
root=testdata,
base_dir=Path(tmpdir),
base_dir=work_dir,
context=context,
graders=graders,
diag=diag,
Expand Down
8 changes: 4 additions & 4 deletions problemtools/checks/testdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def check_testdata(
input_validators: InputValidators,
output_validators: OutputValidators,
format: FormatVersion,
work_dir: str,
work_dir: Path,
diag: Diagnostics,
) -> None:
"""Run all checks on a problem's test data."""
Expand Down Expand Up @@ -70,7 +70,7 @@ def _check_group(
has_default_grader: bool,
input_validators: InputValidators,
output_validator: Program,
work_dir: str,
work_dir: Path,
diag: Diagnostics,
) -> None:
if group.config['grading'] not in ['default', 'custom']:
Expand Down Expand Up @@ -233,7 +233,7 @@ def _check_testcase(
metadata: Metadata,
input_validators: InputValidators,
output_validator: Program,
work_dir: str,
work_dir: Path,
diag: Diagnostics,
) -> None:
_check_newlines(testcase.infile, diag)
Expand All @@ -257,7 +257,7 @@ def _check_testcase(
submission_output=testcase.ansfile,
output_validator=output_validator,
metadata=metadata,
base_dir=Path(work_dir),
base_dir=work_dir,
diag=diag,
)
if val_res.verdict != 'AC':
Expand Down
20 changes: 10 additions & 10 deletions problemtools/checks/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ def _error_in_2023_07(format: FormatVersion, diag: Diagnostics, msg: str, additi
diag.error(msg, additional_info)


def check_input_validators(validators: InputValidators, testdata: TestDataGroup, work_dir: str, diag: Diagnostics) -> None:
def check_input_validators(validators: InputValidators, testdata: TestDataGroup, work_dir: Path, diag: Diagnostics) -> None:
"""Run all checks on a problem's input format validators."""
if validators.uses_old_path:
diag.warning('input_format_validators is a deprecated name; please use input_validators instead')
Expand All @@ -104,7 +104,7 @@ def check_input_validators(validators: InputValidators, testdata: TestDataGroup,

for val in validators.validators:
try:
result = val.compile(Path(work_dir))
result = val.compile(work_dir)
if not result.success:
diag.error(f'Compile error for {val}', result.errmsg)
except ProgramError as e:
Expand Down Expand Up @@ -132,7 +132,7 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None:
for flags_str in all_flags:
flags = flags_str.split()
for val in validators.validators:
status, _ = val.run(file_name, args=flags, work_dir=Path(work_dir))
status, _ = val.run(file_name, args=flags, work_dir=work_dir)
if os.WEXITSTATUS(status) != 42:
break
else:
Expand All @@ -154,7 +154,7 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab
for flags_str in all_flags:
flags = flags_str.split()
for val in validators.validators:
status, _ = val.run(file_name, args=flags, work_dir=Path(work_dir))
status, _ = val.run(file_name, args=flags, work_dir=work_dir)
if os.WEXITSTATUS(status) != 42:
# expected behavior; validator rejects modified input
return False
Expand All @@ -173,17 +173,17 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab
os.unlink(file_name)


def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: str, diag: Diagnostics) -> None:
def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: Path, diag: Diagnostics) -> None:
"""Run the (already checked) input validators against a single test case's input file."""
flags = testcase.input_validator_flags

for val in validators.validators:
# A validator that failed to compile was already reported by check_input_validators; skip it.
if not val.compile(Path(work_dir)).success:
if not val.compile(work_dir).success:
continue

with tempfile.NamedTemporaryFile() as outfile, tempfile.NamedTemporaryFile() as errfile:
status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=Path(work_dir))
status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=work_dir)
if not os.WIFEXITED(status):
emsg = f'Input format validator {val} crashed on input {testcase.infile}'
elif os.WEXITSTATUS(status) != 42:
Expand All @@ -202,7 +202,7 @@ def check_output_validators(
metadata: Metadata,
testdata: TestDataGroup,
probdir: Path,
work_dir: str,
work_dir: Path,
diag: Diagnostics,
) -> None:
"""Run all checks on a problem's output validators."""
Expand Down Expand Up @@ -233,7 +233,7 @@ def check_output_validators(
diag.fatal('problem.yaml specifies custom validator but no validator programs found')

try:
result = selected.compile(Path(work_dir))
result = selected.compile(work_dir)
if not result.success:
diag.fatal(f'Compile error for output validator {selected}', result.errmsg)
except ProgramError as e:
Expand All @@ -254,7 +254,7 @@ def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[TestCase]
submission_output=Path(f.name),
output_validator=selected,
metadata=metadata,
base_dir=Path(work_dir),
base_dir=work_dir,
diag=diag,
)
results.append(result)
Expand Down
3 changes: 2 additions & 1 deletion problemtools/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
_T = TypeVar('_T')
_P = ParamSpec('_P')

PROBLEM_PARTS = ['config', 'data', 'graders', 'statement', 'submissions', 'validators']
#: Listed in the order they're checked in.
PROBLEM_PARTS = ['config', 'statement', 'validators', 'graders', 'data', 'submissions']


class Context:
Expand Down
3 changes: 3 additions & 0 deletions problemtools/model/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .attachments import Attachments, load_attachments
from .graders import DEFAULT_GRADER, Graders, load_graders
from .includes import DEFAULT_LANGUAGE, IncludeFile, Includes, LanguageIncludes, load_includes
from .problem import Problem, load_problem
from .statements import Statements, load_statements
from .submissions import LegacyPolicy, Submission, Submissions, load_submissions
from .testdata import DEFAULT_CONFIG, SCORING_ONLY_KEYS, TestCase, TestDataGroup, load_testdata
Expand All @@ -25,6 +26,7 @@
'LanguageIncludes',
'LegacyPolicy',
'OutputValidators',
'Problem',
'Statements',
'Submission',
'Submissions',
Expand All @@ -36,6 +38,7 @@
'load_includes',
'load_input_validators',
'load_output_validators',
'load_problem',
'load_statements',
'load_submissions',
'load_testdata',
Expand Down
80 changes: 80 additions & 0 deletions problemtools/model/problem.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""Model for a whole problem package: the aggregate of all its parts."""

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path

from ..diagnostics import Diagnostics, VerifyError
from ..formatversion import FormatVersion
from ..languages import Languages, load_language_config
from ..metadata import Metadata, load_metadata
from .attachments import Attachments, load_attachments
from .graders import Graders, load_graders
from .includes import Includes, load_includes
from .statements import Statements, load_statements
from .submissions import Submissions, load_submissions
from .testdata import TestDataGroup, load_testdata
from .validators import InputValidators, OutputValidators, load_input_validators, load_output_validators


@dataclass(frozen=True)
class Problem:
"""A statically loaded problem package.

Loading is static parsing with very little verification (it does not, e.g.,
compile validators or run submissions) -- see the `checks` package for the
checks that should be run on top of a loaded Problem.
"""

probdir: Path
language_config: Languages
metadata: Metadata
statements: Statements
attachments: Attachments
input_validators: InputValidators
output_validators: OutputValidators
graders: Graders
testdata: TestDataGroup
includes: Includes
submissions: Submissions

@property
def shortname(self) -> str:
return self.probdir.name

@property
def format_version(self) -> FormatVersion:
return self.metadata.problem_format_version


def load_problem(probdir: Path, diag: Diagnostics) -> Problem:
"""Loads a problem package from probdir.

On failure, reports errors via `diag` and raises VerifyError.
"""
try:
language_config = load_language_config(probdir.parent)
problem_metadata = load_metadata(probdir)
format_version = problem_metadata.problem_format_version

includes = load_includes(probdir, language_config)
return Problem(
probdir=probdir,
language_config=language_config,
metadata=problem_metadata,
statements=load_statements(probdir, format_version),
attachments=load_attachments(probdir),
input_validators=load_input_validators(probdir, language_config),
output_validators=load_output_validators(probdir, format_version, language_config),
graders=load_graders(probdir, language_config),
testdata=load_testdata(probdir, problem_metadata),
includes=includes,
# Submissions need includes, so includes must be loaded first.
submissions=load_submissions(probdir, language_config, includes),
)
except VerifyError:
# A loader already reported its own diagnostic; don't double-report.
raise
except Exception as e:
diag.fatal(f'Failed to load problem: {e}')
Loading