Add check for judge-specific limits

README.md

@@ -224,6 +224,30 @@ problemtools' configuration:
        memory: 2048 # (unit is MiB)
    ```
 
+3. `judge.yaml`.  Describes what the judge system you intend to run the
+   problem on supports, such as maximum limits, maximum file sizes, and
+   whether minimization problems or negative scores are allowed.
+   `verifyproblem` gives warnings (not errors) for problems exceeding this.
+   The default, found in [judge.yaml](problemtools/config/judge.yaml),
+   describes what Kattis supports. Settings are merged with the
+   defaults, so you only need to list what you want to change, and setting
+   a value to `null` disables that check.  For instance, if you are
+   targeting a judge system called Foo which supports 8 GiB of memory and
+   negative scores, you could use:
+
+   ```yaml
+   name: Foo
+   limits:
+       memory: 8192 # (unit is MiB)
+   allow_negative_scores: true
+   ```
+
+   To turn off all of these checks, set `enabled: false`. To skip them for a
+   single run, leave out the `judge` part when passing `-p` to `verifyproblem`.
+
+   Like `languages.yaml`, `judge.yaml` may also be placed as a sibling to the
+   problem package being checked.
+
 
 ## Requirements and compatibility
 

problemtools/checks/__init__.py

@@ -2,6 +2,7 @@
 from .config import check_config
 from .graders import check_graders
 from .includes import check_includes
+from .judge import check_judge
 from .problem_package import check_problem_package
 from .statements import check_statements
 from .submissions import check_submissions
@@ -14,6 +15,7 @@
     'check_graders',
     'check_includes',
     'check_input_validators',
+    'check_judge',
     'check_output_validators',
     'check_problem_package',
     'check_statements',

problemtools/checks/config.py

@@ -105,12 +105,3 @@ def check_config(
             for param in validation_params:
                 if param not in ['score', 'interactive']:
                     diag.error(f"Invalid parameter '{param}' for custom validation")
-
-    if metadata.limits.time_limit is not None and not metadata.limits.time_limit.is_integer():
-        diag.warning(
-            'Time limit configured to non-integer value. This can be fragile, and may not be supported by your CCS (Kattis does not).'
-        )
-    if not metadata.limits.time_resolution.is_integer():
-        diag.warning(
-            'Time resolution is not an integer. This can be fragile, and may not be supported by your CCS (Kattis does not).'
-        )

problemtools/checks/judge.py

@@ -0,0 +1,134 @@
+"""Checks that a problem is supported by the judge system described by judge.yaml.
+
+These are not checks against the problem format specification, so they only give warnings.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from ..config import ConfigError
+from ..diagnostics import Diagnostics
+from ..judge_config import JudgeConfig, LimitMaxima, load_judge_config
+from ..metadata import Metadata
+from ..model import EntryKind, Files, TestDataGroup
+
+_MIB = 1024 * 1024
+
+_LIMITS = {
+    'memory': ('Memory limit', 'MiB'),
+    'output': ('Output limit', 'MiB'),
+    'code': ('Code size limit', 'KiB'),
+    'compilation_time': ('Compilation time limit', 's'),
+    'compilation_memory': ('Compilation memory limit', 'MiB'),
+    'validation_time': ('Validation time limit', 's'),
+    'validation_memory': ('Validation memory limit', 'MiB'),
+    'validation_output': ('Validation output limit', 'MiB'),
+}
+assert set(_LIMITS) == set(LimitMaxima.model_fields)
+
+
+def check_judge(metadata: Metadata, files: Files, testdata: TestDataGroup, diag: Diagnostics) -> None:
+    """Run all checks that the problem is supported by the judge system.
+
+    The judge system is described by judge.yaml, using the problem's parent directory as
+    an extra override location."""
+    try:
+        judge_config = load_judge_config(files.root.parent)
+    except ConfigError as e:
+        diag.error(str(e))
+        return
+    if not judge_config.enabled:
+        diag.msg('Not checking judge compatibility (disabled in judge.yaml)')
+        return
+    diag.msg(f'Checking compatibility with {judge_config.name}')
+    shortname = files.root.name
+    if judge_config.max_shortname_length is not None and len(shortname) > judge_config.max_shortname_length:
+        diag.warning(
+            f"Problem short name '{shortname}' is {len(shortname)} characters long, "
+            f'above what {judge_config.name} normally supports ({judge_config.max_shortname_length} characters)'
+        )
+    _check_file_sizes(judge_config, files, testdata, diag)
+    _check_config(judge_config, metadata, testdata, diag)
+
+
+def _check_file_sizes(judge_config: JudgeConfig, files: Files, testdata: TestDataGroup, diag: Diagnostics) -> None:
+    if judge_config.max_file_size is not None:
+        for path in files.walk(files.root, EntryKind.FILE):
+            size = path.stat().st_size
+            if size > judge_config.max_file_size * _MIB:
+                diag.warning(
+                    f'File {path.relative_to(files.root.parent)} is {size / _MIB:.1f} MiB, '
+                    f'above what {judge_config.name} normally supports ({judge_config.max_file_size} MiB)'
+                )
+
+    if judge_config.max_testdata_size is not None:
+        # Count files with identical content once.
+        size_by_digest: dict[bytes, int] = {}
+        for testcase in testdata.get_all_testcases():
+            size_by_digest[testcase.input_sha256] = testcase.infile.stat().st_size
+            size_by_digest[testcase.answer_sha256] = testcase.ansfile.stat().st_size
+        total = sum(size_by_digest.values())
+        if total > judge_config.max_testdata_size * _MIB:
+            diag.warning(
+                f'Test data input and answer files are {total / _MIB:.1f} MiB in total (counting identical files once), '
+                f'above what {judge_config.name} normally supports ({judge_config.max_testdata_size} MiB)'
+            )
+
+
+def _check_config(judge_config: JudgeConfig, metadata: Metadata, testdata: TestDataGroup, diag: Diagnostics) -> None:
+    name = judge_config.name
+
+    for key, (description, unit) in _LIMITS.items():
+        maximum = getattr(judge_config.limits, key)
+        value = getattr(metadata.limits, key)
+        if maximum is not None and value > maximum:
+            diag.warning(
+                f'{description} (limits.{key}) is {value} {unit}, above what {name} normally supports ({maximum} {unit})'
+            )
+
+    if judge_config.integer_time_limit:
+        if metadata.limits.time_limit is not None and not metadata.limits.time_limit.is_integer():
+            diag.warning(f'Time limit configured to non-integer value, which is not supported by {name}')
+        if not metadata.limits.time_resolution.is_integer():
+            diag.warning(f'Time resolution is not an integer, which is not supported by {name}')
+
+    if metadata.is_pass_fail():
+        return
+
+    if not judge_config.allow_minimization and metadata.legacy_grading.objective == 'min':
+        diag.warning(f'Minimization problems (objective: min) are not supported by {name}')
+
+    if not judge_config.allow_negative_scores:
+        _check_negative_scores(judge_config, testdata, None, diag)
+
+
+def _check_negative_scores(
+    judge_config: JudgeConfig, group: TestDataGroup, parent_config: dict[str, Any] | None, diag: Diagnostics
+) -> None:
+    """Heuristic check for negative scores, based only on the score configuration in testdata.yaml.
+
+    This is not a complete check: a grader or output validator can still produce negative scores.
+    Catching those would require checking the scores actually seen when running submissions."""
+    # Configuration is inherited, so only report values in the group where they are set.
+    for key in ['accept_score', 'reject_score', 'range']:
+        value = group.config.get(key)
+        if value is None or (parent_config is not None and parent_config.get(key) == value):
+            continue
+        if _allows_negative_score(key, value):
+            diag.warning(f'{group} has {key} {value}, but negative scores are not supported by {judge_config.name}')
+    for subgroup in group.get_subgroups():
+        _check_negative_scores(judge_config, subgroup, group.config, diag)
+
+
+def _allows_negative_score(key: str, value: Any) -> bool:
+    # Malformed values are reported by the test data checks, so we just ignore them here.
+    try:
+        if key == 'range':
+            low, high = map(float, str(value).split())
+            # The default range is unbounded, which we accept.
+            return low < 0 and not (low == -math.inf and high == math.inf)
+        return float(value) < 0
+    except ValueError:
+        return False

problemtools/checks/testdata.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import collections
-import hashlib
 from collections.abc import Callable
 from pathlib import Path
 
@@ -275,9 +274,7 @@ def _check_group(
         for testcase in group.get_all_testcases():
             if testcase.infile.is_symlink():
                 continue
-            with open(testcase.infile, 'rb') as f:
-                digest = hashlib.file_digest(f, lambda: hashlib.sha256(usedforsecurity=False)).digest()
-            hashes[digest].append(testcase.infile.relative_to(files.root))
+            hashes[testcase.input_sha256].append(testcase.infile.relative_to(files.root))
         for identical in hashes.values():
             if len(identical) > 1:
                 diag.warning(

problemtools/config/judge.yaml

@@ -0,0 +1,33 @@
+# Capabilities of the judge system problems are verified against. Exceeding
+# these gives warnings, not errors. Set a value to null to disable that check.
+
+# Set to false to disable all judge compatibility checks.
+enabled: true
+
+# Name of the judge system, used in warnings.
+name: Kattis
+
+# Maximum values for limits in problem.yaml. Same keys and units as there
+# (memory and output in MiB, code in KiB, times in seconds).
+limits:
+  memory: 4096
+  output: 50
+  code: 512
+  compilation_time: 120
+  compilation_memory: 4096
+  validation_time: 120
+  validation_memory: 4096
+  validation_output: 8
+
+# Maximum length of the problem short name (the package directory name).
+max_shortname_length: 40
+
+# Maximum size of any single file in the package, in MiB.
+max_file_size: 100
+# Maximum total size of test data input and answer files, in MiB. Files with
+# identical content are only counted once.
+max_testdata_size: 512
+
+integer_time_limit: true
+allow_minimization: false
+allow_negative_scores: false

problemtools/context.py

@@ -16,7 +16,7 @@
 _P = ParamSpec('_P')
 
 #: Listed in the order they're checked in.
-PROBLEM_PARTS = ['config', 'statement', 'validators', 'graders', 'data', 'submissions']
+PROBLEM_PARTS = ['config', 'judge', 'statement', 'validators', 'graders', 'data', 'submissions']
 
 
 class Context:

problemtools/judge/cache.py

@@ -1,11 +1,8 @@
 from __future__ import annotations
 
 import copy
-import hashlib
 from concurrent.futures import Future
 from dataclasses import dataclass
-from functools import cache
-from pathlib import Path
 from threading import Lock
 
 from ..model import TestCase
@@ -19,20 +16,14 @@ class CacheKey:
     validator_flags: tuple[str, ...]
 
 
-@cache
-def _compute_reuse_key(infile: Path, ansfile: Path, output_validator_flags: tuple[str, ...]) -> CacheKey:
-    return CacheKey(
-        input_hash=hashlib.sha256(infile.read_bytes()).digest(),
-        ans_hash=hashlib.sha256(ansfile.read_bytes()).digest(),
-        validator_flags=output_validator_flags,
-    )
-
-
 def compute_reuse_key(testcase: TestCase) -> CacheKey:
     """A key identifying testcase for result-reuse purposes: same input/answer file contents
-    and same output validator flags means a result can be reused. Memoized, since this hashes
-    file contents on first computation for a given testcase."""
-    return _compute_reuse_key(testcase.infile, testcase.ansfile, tuple(testcase.output_validator_flags))
+    and same output validator flags means a result can be reused."""
+    return CacheKey(
+        input_hash=testcase.input_sha256,
+        ans_hash=testcase.answer_sha256,
+        validator_flags=tuple(testcase.output_validator_flags),
+    )
 
 
 @dataclass

problemtools/judge_config.py

@@ -0,0 +1,53 @@
+"""Configuration describing what the judge system problems are verified against supports.
+
+Loaded from judge.yaml, with the same override paths as languages.yaml. All fields
+are optional and permissive by default, so a config only needs to list what it restricts.
+"""
+
+from pathlib import Path
+
+import pydantic
+from pydantic import BaseModel, ConfigDict
+
+from . import config
+
+
+class LimitMaxima(BaseModel):
+    """Maximum values for problem.yaml limits. Same keys and units as there. None means unchecked."""
+
+    memory: int | None = None
+    output: int | None = None
+    code: int | None = None
+    compilation_time: int | None = None
+    compilation_memory: int | None = None
+    validation_time: int | None = None
+    validation_memory: int | None = None
+    validation_output: int | None = None
+
+    model_config = ConfigDict(extra='forbid')
+
+
+class JudgeConfig(BaseModel):
+    enabled: bool = True
+    name: str = 'your judge system'
+    limits: LimitMaxima = LimitMaxima()
+    max_shortname_length: int | None = None  # length of the problem package directory name
+    max_file_size: int | None = None  # MiB, any file in the package
+    max_testdata_size: int | None = None  # MiB, de-duplicated test data .in and .ans files
+    integer_time_limit: bool = False
+    allow_minimization: bool = True
+    allow_negative_scores: bool = True
+
+    model_config = ConfigDict(extra='forbid')
+
+
+def load_judge_config(probdir_parent: Path) -> JudgeConfig:
+    """Load judge configuration.
+
+    :raises config.ConfigError: if the configuration is invalid.
+    """
+    data = config.load_config('judge.yaml', [probdir_parent])
+    try:
+        return JudgeConfig.model_validate(data)
+    except pydantic.ValidationError as e:
+        raise config.ConfigError(f'Invalid judge configuration (judge.yaml): {e}') from e

problemtools/model/testdata.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import os
 from dataclasses import dataclass, field
+from functools import cached_property
 from re import Pattern
 from typing import Any
 
@@ -32,6 +34,16 @@ class TestCase:
     def __str__(self) -> str:
         return f'testcase {self.path}'
 
+    @cached_property
+    def input_sha256(self) -> bytes:
+        """sha256 of the input file contents. Computed on first access."""
+        return _file_digest(self.infile)
+
+    @cached_property
+    def answer_sha256(self) -> bytes:
+        """sha256 of the answer file contents. Computed on first access."""
+        return _file_digest(self.ansfile)
+
     def matches_filter(self, filter_re: Pattern[str]) -> bool:
         return filter_re.search(str(self.path)) is not None
 
@@ -42,6 +54,11 @@ def get_all_testcases(self) -> list[TestCase]:
         return [self]
 
 
+def _file_digest(path: AbsolutePath) -> bytes:
+    with open(path, 'rb') as f:
+        return hashlib.file_digest(f, lambda: hashlib.sha256(usedforsecurity=False)).digest()
+
+
 @dataclass(frozen=True)
 class TestDataGroup:
     """A group of test cases and/or nested test case groups, configured by testdata.yaml.