feat: deterministic moderation triage scoring

.env.example

@@ -5,6 +5,15 @@ APP_DEBUG=true
 APP_URL=http://localhost:8000
 ADMINS=
 
+# Moderation triage (deterministic always on; AI risk scoring off unless explicitly enabled)
+# The existing OpenAI auto-moderation (ModerateCommit) is separate and keyed on OPENAI_API_KEY
+MODERATION_AI_ENABLED=false
+# MODERATION_AI_PROVIDER=
+# MODERATION_AI_API_KEY=
+# MODERATION_AUTHOR_FLOOD_THRESHOLD=3
+# MODERATION_BURST_COUNT=5
+# MODERATION_BURST_MINUTES=10
+
 APP_LOCALE=en
 APP_FALLBACK_LOCALE=en
 APP_FAKER_LOCALE=en_US

app/Actions/CreateCommit.php

@@ -42,6 +42,7 @@ public function __construct(
         private MentionedOrganizationsQuery $mentionedOrganizations,
         private CreateCommitTopics $createCommitTopics,
         private TomQuery $tomQuery,
+        private UpdateCommitModeration $updateCommitModeration,
     ) {
         //
     }
@@ -89,6 +90,10 @@ public function handle(User $user, string $body, ?Commit $parent = null, ?Commit
                 $this->updateCommitHotScores->handle([$commit->id]);
             }
 
+            if ($commit->status === CommitStatus::Pending) {
+                $this->updateCommitModeration->handle($commit);
+            }
+
             $this->createReplyNotifications->handle($commit);
             $this->createQuoteNotifications->handle($commit);
             $this->createMentionNotifications->handle($commit);

app/Actions/CreateCommitTranslation.php

@@ -11,6 +11,7 @@
 use App\Services\CommitTranslations;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
+use Laravel\Ai\Responses\StructuredAgentResponse;
 use RuntimeException;
 use Throwable;
 
@@ -37,11 +38,12 @@ public function handle(Commit $commit, ?string $language = null): void
             try {
                 $response = new CommitTranslator($language)->prompt($commit->body);
 
-                $source = $response['source_language'] ?? null;
-                $body = $response['body'] ?? null;
+                $structured = $response instanceof StructuredAgentResponse ? $response->structured : [];
+                $source = $structured['source_language'] ?? null;
+                $body = $structured['body'] ?? null;
 
                 $reason = match (true) {
-                    $response->structured === [] => 'empty_or_malformed_response',
+                    $structured === [] => 'empty_or_malformed_response',
                     ! is_string($source), ! is_string($body) => 'invalid_response_shape',
                     ! in_array($source, [...$this->translations->languages(), 'und'], true) => 'unsupported_source_language',
                     $language !== null && $source !== 'und' && blank($body) => 'empty_translation',

app/Actions/UpdateCommitModeration.php

@@ -0,0 +1,67 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Actions;
+
+use App\Contracts\Moderation\AiModerationProvider;
+use App\Models\Commit;
+use App\Moderation\DeterministicModerationSignals;
+use Illuminate\Support\Facades\Log;
+use Throwable;
+
+final readonly class UpdateCommitModeration
+{
+    public function __construct(private DeterministicModerationSignals $deterministic)
+    {
+        //
+    }
+
+    public function handle(Commit $commit): void
+    {
+        $bodyHash = $this->deterministic->bodyHash($commit->body);
+
+        $commit->forceFill(['body_hash' => $bodyHash])->saveQuietly();
+
+        $signals = $this->deterministic->handle($commit->refresh());
+        $signals = [...$signals, ...$this->aiSignals($commit)];
+
+        $score = array_sum(array_column($signals, 'weight'));
+        $cap = config('moderation.score_cap');
+        $cap = is_numeric($cap) ? (int) $cap : 100;
+
+        $commit->forceFill([
+            'moderation_score' => min($cap, $score),
+            'moderation_signals' => $signals === [] ? null : $signals,
+        ])->saveQuietly();
+    }
+
+    /**
+     * @return list<array{code: string, weight: int}>
+     */
+    private function aiSignals(Commit $commit): array
+    {
+        if (! config('moderation.ai.enabled')) {
+            return [];
+        }
+
+        if (! app()->bound(AiModerationProvider::class)) {
+            Log::warning('Moderation AI is enabled but no provider is bound.', [
+                'commit_id' => $commit->id,
+            ]);
+
+            return [];
+        }
+
+        try {
+            return resolve(AiModerationProvider::class)->signals($commit);
+        } catch (Throwable $throwable) {
+            Log::warning('Moderation AI scoring failed; keeping deterministic score.', [
+                'commit_id' => $commit->id,
+                'exception' => $throwable->getMessage(),
+            ]);
+
+            return [];
+        }
+    }
+}

app/Actions/UpdateCommitStatuses.php

@@ -13,6 +13,7 @@
 use App\Models\Notification;
 use App\Models\Poll;
 use App\Models\User;
+use App\Moderation\DeterministicModerationSignals;
 use App\Queries\CommitBuilder;
 use App\Queries\TomQuery;
 use Illuminate\Support\Facades\DB;
@@ -28,6 +29,7 @@ public function __construct(
         private CreateWorldReply $createWorldReply,
         private UpdateCommitHotScores $updateCommitHotScores,
         private TomQuery $tomQuery,
+        private DeterministicModerationSignals $moderationSignals,
     ) {
         //
     }
@@ -60,6 +62,13 @@ public function handle(array $commitIds, CommitStatus $status, ?string $reason =
                 ->delete();
 
             if ($status === CommitStatus::Rejected) {
+                // Commits approved at creation were never hashed, so a pending copy would miss them as duplicates.
+                $commits
+                    ->filter(fn (Commit $commit): bool => $commit->body_hash === null)
+                    ->each(fn (Commit $commit): int => Commit::query()
+                        ->whereKey($commit->id)
+                        ->update(['body_hash' => $this->moderationSignals->bodyHash($commit->body)]));
+
                 Notification::query()->insert($commits->map(fn (Commit $commit): array => [
                     'id' => Str::uuid7()->toString(),
                     'recipient_id' => $commit->user_id,

app/Ai/Tools/CommunityStats.php

@@ -40,7 +40,7 @@ public function handle(Request $request): string
             return 'Provide one location name between 2 and 80 characters.';
         }
 
-        if ($metric === 'bio_language' && ! array_key_exists($term, CommunityStatsQuery::Languages)) {
+        if ($metric === 'bio_language' && ! array_key_exists((string) $term, CommunityStatsQuery::Languages)) {
             return 'Unsupported language. Choose one of: '.implode(', ', array_keys(CommunityStatsQuery::Languages)).'.';
         }
 

app/Contracts/Moderation/AiModerationProvider.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Contracts\Moderation;
+
+use App\Models\Commit;
+
+interface AiModerationProvider
+{
+    /**
+     * @return list<array{code: string, weight: int}>
+     */
+    public function signals(Commit $commit): array;
+}

app/Http/Controllers/ModerationController.php

@@ -35,10 +35,11 @@ public function show(ShowModerationRequest $request, #[CurrentUser] User $user):
                 ->withStarsFor($user)
                 ->withPollFor($user)
                 ->afterQuery(fn (Collection $commits) => resolve(CommitTranslations::class)->attach($commits, includePending: true))
+                ->orderByDesc('moderation_score')
                 ->oldest()
                 ->oldest('id')
                 ->cursorPaginate(20)
-                ->through($this->withAiDecision(...))),
+                ->through($this->withModeration(...))),
         ]);
     }
 
@@ -56,10 +57,10 @@ public function update(UpdateModerationRequest $request, UpdateCommitStatuses $a
     }
 
     /**
-     * The AI's decision is hidden everywhere else, but moderators see it.
+     * The AI's decision and the risk score are hidden everywhere else, but moderators see them.
      */
-    private function withAiDecision(Commit $commit): Commit
+    private function withModeration(Commit $commit): Commit
     {
-        return $commit->makeVisible(['ai_decision', 'ai_categories']);
+        return $commit->makeVisible(['ai_decision', 'ai_categories', 'moderation_score', 'moderation_signals']);
     }
 }

app/Models/Commit.php

@@ -33,6 +33,9 @@
  * @property-read array<string, string>|null $mentions
  * @property-read array<string, string>|null $organization_mentions
  * @property-read CommitStatus $status
+ * @property-read int $moderation_score
+ * @property-read list<array{code: string, weight: int}>|null $moderation_signals
+ * @property-read string|null $body_hash
  * @property-read string|null $link_preview_id
  * @property-read string|null $quoted_commit_id
  * @property-read CommitStatus|null $ai_decision
@@ -61,6 +64,9 @@
     'ai_decision',
     'ai_categories',
     'embedding',
+    'moderation_score',
+    'moderation_signals',
+    'body_hash',
 ])]
 #[UseEloquentBuilder(CommitBuilder::class)]
 final class Commit extends Model
@@ -118,6 +124,9 @@ public function casts(): array
             'mentions' => 'array',
             'organization_mentions' => 'array',
             'status' => CommitStatus::class,
+            'moderation_score' => 'integer',
+            'moderation_signals' => 'array',
+            'body_hash' => 'string',
             'link_preview_id' => 'string',
             'quoted_commit_id' => 'string',
             'ai_decision' => CommitStatus::class,

app/Moderation/DeterministicModerationSignals.php

@@ -0,0 +1,127 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Moderation;
+
+use App\Enums\CommitStatus;
+use App\Models\Commit;
+use App\Queries\CommitBuilder;
+use App\Services\Urls;
+
+final readonly class DeterministicModerationSignals
+{
+    public function __construct(private Urls $urls)
+    {
+        //
+    }
+
+    /**
+     * @return list<array{code: string, weight: int}>
+     */
+    public function handle(Commit $commit): array
+    {
+        $signals = [];
+
+        if ($this->isDuplicate($commit)) {
+            $signals[] = $this->signal('duplicate');
+        }
+
+        if ($this->isAuthorFlood($commit)) {
+            $signals[] = $this->signal('author_flood');
+        }
+
+        if ($this->isBurst($commit)) {
+            $signals[] = $this->signal('burst');
+        }
+
+        if ($this->urls->first($commit->body) !== null) {
+            $signals[] = $this->signal('has_url');
+        }
+
+        if ($this->isInviteLike($commit->body)) {
+            $signals[] = $this->signal('invite_like');
+        }
+
+        return $signals;
+    }
+
+    public function bodyHash(string $body): string
+    {
+        $normalized = mb_strtolower(mb_trim((string) preg_replace('/\s+/u', ' ', $body)));
+
+        return hash('sha256', $normalized);
+    }
+
+    /**
+     * @return array{code: string, weight: int}
+     */
+    private function signal(string $code): array
+    {
+        return [
+            'code' => $code,
+            'weight' => $this->integerConfig("moderation.weights.{$code}", 0),
+        ];
+    }
+
+    private function isDuplicate(Commit $commit): bool
+    {
+        // Image-only commits share the empty body's hash, so there is no text to compare.
+        if (mb_trim($commit->body) === '') {
+            return false;
+        }
+
+        $hash = $commit->body_hash ?? $this->bodyHash($commit->body);
+
+        // Short replies like "thanks!" match across users all the time, so another user's text only counts once a
+        // moderator has rejected it.
+        return Commit::query()
+            ->where('body_hash', $hash)
+            ->whereKeyNot($commit->id)
+            ->where(fn (CommitBuilder $query): CommitBuilder => $query
+                ->where('status', CommitStatus::Rejected)
+                ->orWhere(fn (CommitBuilder $query): CommitBuilder => $query
+                    ->where('user_id', $commit->user_id)
+                    ->pending()))
+            ->exists();
+    }
+
+    private function isAuthorFlood(Commit $commit): bool
+    {
+        $threshold = $this->integerConfig('moderation.author_flood_threshold', 3);
+
+        return Commit::query()
+            ->where('user_id', $commit->user_id)
+            ->pending()
+            ->count() >= $threshold;
+    }
+
+    private function isBurst(Commit $commit): bool
+    {
+        $count = $this->integerConfig('moderation.burst.count', 5);
+        $minutes = $this->integerConfig('moderation.burst.minutes', 10);
+
+        // The window ends when the commit was posted, so a backfill scores the burst as it was at submission. Commits
+        // posted after this one in the same second are left out, just as they were not there yet.
+        return Commit::query()
+            ->where('user_id', $commit->user_id)
+            ->where('created_at', '>=', $commit->created_at->copy()->subMinutes($minutes))
+            ->whereNot(fn (CommitBuilder $query): CommitBuilder => $query->newerThan($commit))
+            ->count() >= $count;
+    }
+
+    private function isInviteLike(string $body): bool
+    {
+        return preg_match(
+            '/\b(join\s+(my|our)\s+(discord|telegram|whatsapp)|dm\s+me|send\s+me\s+(a\s+)?dm|click\s+(the\s+)?link|free\s+crypto|whatsapp\.me|t\.me\/)/iu',
+            $body,
+        ) === 1;
+    }
+
+    private function integerConfig(string $key, int $default): int
+    {
+        $value = config($key);
+
+        return is_numeric($value) ? (int) $value : $default;
+    }
+}

app/Queries/CommitBuilder.php

@@ -31,6 +31,7 @@ final class CommitBuilder extends Builder
     private const array COLUMNS_BUT_EMBEDDING = [
         'id', 'user_id', 'body', 'created_at', 'updated_at', 'status', 'parent_id', 'deleted_at', 'link_preview_id',
         'quoted_commit_id', 'mentions', 'ai_decision', 'ai_categories', 'root_id', 'hot_score', 'pasteis_count',
+        'moderation_score', 'moderation_signals', 'body_hash',
     ];
 
     /**