feat: extract translatable strings from the authz schema

.gitignore

@@ -64,5 +64,8 @@ docs/openedx_authz.*.rst
 *.sqlite3
 *.db
 
+# Generated by `make extract_translations` / `extract_schema_translations`
+src/openedx_authz/engine/schema/_generated_translations.py
+
 # IDE
 .vscode/

Makefile

@@ -75,6 +75,7 @@ selfcheck: ## check that the Makefile is well-formed
 
 extract_translations: ## extract strings to be translated, outputting .mo files
 	rm -rf docs/_build
+	python manage.py extract_schema_translations
 	cd src/openedx_authz && i18n_tool extract --no-segment
 
 compile_translations: ## compile translation files, outputting .po files for each supported language

pyproject.toml

@@ -114,6 +114,7 @@ test-base = [
     "code-annotations",
     "coverage",
     "ddt",
+    "polib",
     "pytest-cov",
     "pytest-django",
 ]

src/openedx_authz/engine/schema/translation.py

@@ -0,0 +1,257 @@
+"""Extract translatable strings from the static authz schema (ADR 0020).
+
+The authz schema's ``display_name`` and ``description`` fields are source-language
+text that end users see through APIs and frontend clients, so they need to go
+through the Open edX translation process (OEP-58). Standard extraction tools only
+scan Python and JavaScript source, so they can't find strings written in YAML.
+
+This module reads every discovered schema resource directly with ``yaml.safe_load``,
+independent of the schema ``load``/``validate``/``compile`` pipeline: it only needs
+the raw ``display_name``/``description`` fields, not the richer typed documents
+those steps build, so it stays usable (and testable) against ad hoc or malformed
+schema directories, not just a fully valid one. It does, however, resolve conflicts
+between files the same way ``engine.schema.compilation`` does: by each document's
+``priority`` (higher wins; identical content merges; a disagreement at equal
+priority is an error), so the generated catalog reflects the same winner the
+compiled schema would use at runtime. Re-deriving that one rule here (rather than
+depending on the loader/compiler) is what keeps this module usable before, or
+independently of, that pipeline.
+
+It emits a generated Python module containing one ``pgettext()`` call per
+translatable field, which ``i18n_tool extract`` (a thin wrapper around Django's
+``makemessages``) then picks up like any other source file. ``pgettext`` is used
+instead of plain ``gettext`` because ADR 0020 requires a permission's display name
+and description to remain separate translation messages even when they contain
+the same English text; a plain ``gettext`` call would merge identical msgids into
+one catalog entry. The first ``pgettext()`` argument is gettext's ``msgctxt``: it
+carries the field's stable identifier, so two different roles that happen to share
+a one-word name like "Admin" stay separate messages too.
+
+This generated module is never imported or executed: its only purpose is to be
+valid Python source for ``makemessages`` to scan. At runtime, the schema catalog
+API looks up a field's source-language value directly with the same
+``pgettext(context, value)`` pair used here, so the generated calls must stay in
+sync with however that lookup builds its context string.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import yaml
+
+from openedx_authz.engine.schema.discovery import DiscoveredResource, SchemaDiscovery, SchemaDiscoveryError
+
+GENERATED_FILE_HEADER = '''"""Generated by extract_schema_translations. Do not edit, do not import.
+
+This file exists only so that ``i18n_tool extract`` (Django's ``makemessages``)
+can find the authz schema's source-language display_name/description strings,
+which otherwise live in YAML and are invisible to it. Regenerate it with:
+
+    python manage.py extract_schema_translations
+
+See openedx_authz.engine.schema.translation for why pgettext is used here.
+"""
+from django.utils.translation import pgettext
+'''
+
+# gettext's own internal glue byte between msgctxt and msgid. If it ends up inside
+# either half, tools downstream of extraction (msguniq, as part of `makemessages`)
+# fail outright trying to parse the result back apart, so it can never appear in
+# generated content.
+GETTEXT_CONTEXT_SEPARATOR = "\x04"
+
+
+@dataclass(frozen=True)
+class TranslatableMessage:
+    """One ``pgettext(context, message)`` pair extracted from the schema.
+
+    Attributes:
+        context: gettext's ``msgctxt``. Disambiguates otherwise-identical
+            messages, formatted as ``"{field_kind}.{field_name}:{stable_id}"``,
+            e.g. ``"permission.display_name:courses.view_course"``.
+        message: gettext's ``msgid``, the source-language (English) string to
+            translate.
+    """
+
+    context: str
+    message: str
+
+
+class SchemaTranslationExtractionError(Exception):
+    """A schema resource could not be read, parsed, or resolved for extraction."""
+
+
+@dataclass(frozen=True)
+class _Candidate:
+    """A :class:`TranslatableMessage` plus the provenance needed to resolve conflicts."""
+
+    message: TranslatableMessage
+    priority: int
+    source: str
+
+
+def _permission_stable_id(entry: dict) -> str:
+    """Build a permission's stable id from its namespace and name, both required."""
+    namespace = entry.get("namespace")
+    name = entry.get("name")
+    if not namespace or not name:
+        return ""
+    return f"{namespace}.{name}"
+
+
+# Each entry is (the document key holding this section's list, the field_kind used
+# in the generated context string, a function pulling this entry's stable id out of
+# its raw mapping). All four sections share the same display_name/description
+# extraction logic (_messages_for_entry); only these three things differ between
+# them, so a data table covers it without a class per section.
+_SECTIONS: tuple[tuple[str, str, Callable[[dict], str]], ...] = (
+    ("permission_categories", "permission_category", lambda entry: str(entry.get("id") or "")),
+    ("permissions", "permission", _permission_stable_id),
+    ("roles", "role", lambda entry: str(entry.get("id") or "")),
+    ("role_extensions", "role_extension", lambda entry: str(entry.get("role") or "")),
+)
+
+
+def extract_messages(resources: list[DiscoveredResource] | None = None) -> list[TranslatableMessage]:
+    """Extract every translatable schema string from the given (or discovered) resources.
+
+    Args:
+        resources: Resources to extract from. Defaults to everything
+            ``SchemaDiscovery`` finds (every registered schema directory), which
+            is what the management command uses.
+
+    Returns:
+        Deduplicated messages, sorted by context, so the generated file's
+        diffs are stable across runs regardless of discovery order.
+
+    Raises:
+        SchemaTranslationExtractionError: A resource's bytes couldn't be read, it
+            wasn't valid YAML, a section wasn't shaped as the schema requires, an
+            entry was missing its identifying field, a translatable string
+            collided with gettext's own context separator, or two documents
+            disagreed on the same field at equal priority. A translation build
+            must fail loudly here (ADR 0020) rather than silently produce a
+            catalog that doesn't match what the compiled schema would serve.
+    """
+    if resources is None:
+        try:
+            resources = SchemaDiscovery().discover()
+        except SchemaDiscoveryError as exc:
+            raise SchemaTranslationExtractionError(str(exc)) from exc
+
+    resolved: dict[str, _Candidate] = {}
+    for resource in resources:
+        for candidate in _extract_from_resource(resource):
+            existing = resolved.get(candidate.message.context)
+            resolved[candidate.message.context] = (
+                candidate if existing is None else _resolve_conflict(existing, candidate)
+            )
+
+    return sorted((candidate.message for candidate in resolved.values()), key=lambda m: m.context)
+
+
+def _resolve_conflict(existing: _Candidate, incoming: _Candidate) -> _Candidate:
+    """Resolve two competing messages for the same context, the way compilation.py does.
+
+    Identical messages merge (both files contributed the same text); otherwise
+    higher ``priority`` wins; equal priority with disagreeing text is unresolvable.
+    """
+    if existing.message == incoming.message:
+        return existing
+    if existing.priority == incoming.priority:
+        raise SchemaTranslationExtractionError(
+            f"Conflicting translation for {existing.message.context!r} at equal priority "
+            f"{existing.priority} ({existing.source} vs {incoming.source}): "
+            f"{existing.message.message!r} vs {incoming.message.message!r}."
+        )
+    return existing if existing.priority > incoming.priority else incoming
+
+
+def _extract_from_resource(resource: DiscoveredResource) -> list[_Candidate]:
+    """Parse one resource's YAML and pull its translatable fields out."""
+    source = f"{resource.package}:{resource.resource_path}"
+    try:
+        contents = resource.read_bytes()
+    except SchemaDiscoveryError as exc:
+        raise SchemaTranslationExtractionError(str(exc)) from exc
+
+    try:
+        document = yaml.safe_load(contents)
+    except yaml.YAMLError as exc:
+        raise SchemaTranslationExtractionError(f"Invalid YAML in {source}: {exc}") from exc
+
+    if document is None:
+        return []
+    if not isinstance(document, dict):
+        raise SchemaTranslationExtractionError(
+            f"Schema file {source} must be a mapping at the top level, got {type(document).__name__}."
+        )
+
+    # Required per authz-schema-v1.json; enforcing its presence is validation's job,
+    # not extraction's (same split loading.py makes), so a missing priority reads as
+    # 0, the lowest precedence. A malformed one can't be resolved at all, so that
+    # raises now rather than producing a nonsensical conflict winner later.
+    try:
+        priority = int(document.get("priority", 0))
+    except (TypeError, ValueError) as exc:
+        raise SchemaTranslationExtractionError(
+            f"{source}: 'priority' must be an integer, got {document.get('priority')!r}."
+        ) from exc
+
+    candidates: list[_Candidate] = []
+    for section_key, field_kind, stable_id_of in _SECTIONS:
+        entries = document.get(section_key) or []
+        if not isinstance(entries, list):
+            raise SchemaTranslationExtractionError(
+                f"{source}: {section_key!r} must be a list, got {type(entries).__name__}."
+            )
+        for entry in entries:
+            if not isinstance(entry, dict):
+                raise SchemaTranslationExtractionError(
+                    f"{source}: an entry in {section_key!r} must be a mapping, got {type(entry).__name__}."
+                )
+            stable_id = stable_id_of(entry)
+            if not stable_id:
+                raise SchemaTranslationExtractionError(
+                    f"{source}: a {field_kind} entry in {section_key!r} is missing its identifying field(s)."
+                )
+            for message in _messages_for_entry(source, field_kind, stable_id, entry):
+                candidates.append(_Candidate(message=message, priority=priority, source=source))
+
+    return candidates
+
+
+def _messages_for_entry(source: str, field_kind: str, stable_id: str, entry: dict) -> list[TranslatableMessage]:
+    """Build the display_name/description messages for one schema entry.
+
+    A field that's absent (common on a ``role_extension``, which only overrides
+    what it mentions, but just as possible on any other entry) is skipped rather
+    than emitting an empty message: requiring these fields is validation's job,
+    not extraction's, so there's nothing useful to translate either way.
+    """
+    messages = []
+    for field_name in ("display_name", "description"):
+        value = entry.get(field_name)
+        if value is None:
+            continue
+        context = f"{field_kind}.{field_name}:{stable_id}"
+        message = str(value)
+        for part, label in ((context, "context"), (message, "text")):
+            if GETTEXT_CONTEXT_SEPARATOR in part:
+                raise SchemaTranslationExtractionError(
+                    f"{source}: the {label} for {context!r} contains U+0004, which gettext reserves as its "
+                    "own context separator and cannot appear in translatable content."
+                )
+        messages.append(TranslatableMessage(context=context, message=message))
+    return messages
+
+
+def render_module(messages: list[TranslatableMessage]) -> str:
+    """Render the extracted messages as a generated Python module's source text."""
+    lines = [GENERATED_FILE_HEADER]
+    for extracted in messages:
+        lines.append(f"pgettext({extracted.context!r}, {extracted.message!r})")
+    lines.append("")  # Trailing newline.
+    return "\n".join(lines)

src/openedx_authz/management/commands/extract_schema_translations.py

@@ -0,0 +1,59 @@
+"""Django management command to extract translatable strings from the authz schema.
+
+See openedx_authz.engine.schema.translation for why this exists and how it works.
+"""
+
+import os
+
+from django.core.management.base import BaseCommand, CommandError
+
+from openedx_authz import ROOT_DIRECTORY
+from openedx_authz.engine.schema.translation import (
+    SchemaTranslationExtractionError,
+    extract_messages,
+    render_module,
+)
+
+DEFAULT_OUTPUT_PATH = os.path.join(ROOT_DIRECTORY, "engine", "schema", "_generated_translations.py")
+
+
+class Command(BaseCommand):
+    """Generate a Python module with one pgettext() call per translatable schema string.
+
+    Run this before ``i18n_tool extract`` (see the ``extract_translations`` Makefile
+    target), so the generated calls are on disk for ``makemessages`` to scan.
+
+    Example Usage:
+        python manage.py extract_schema_translations
+        python manage.py extract_schema_translations --output /path/to/file.py
+    """
+
+    help = "Extract translatable display_name/description strings from the authz schema into a generated module."
+
+    def add_arguments(self, parser) -> None:
+        """Add command-line arguments to the argument parser.
+
+        Args:
+            parser: The Django argument parser instance to configure.
+        """
+        parser.add_argument(
+            "--output",
+            type=str,
+            default=DEFAULT_OUTPUT_PATH,
+            help="Path to write the generated module to.",
+        )
+
+    def handle(self, *args, **options) -> None:
+        """Extract schema translation messages and write the generated module."""
+        try:
+            messages = extract_messages()
+        except SchemaTranslationExtractionError as exc:
+            raise CommandError(str(exc)) from exc
+
+        output_path = options["output"]
+        with open(output_path, "w", encoding="utf-8") as output_file:
+            output_file.write(render_module(messages))
+
+        self.stdout.write(
+            self.style.SUCCESS(f"Extracted {len(messages)} translatable schema string(s) to {output_path}")
+        )