diff --git a/.env.example b/.env.example index 7144cdd72..d4770ba06 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,15 @@ APP_DEBUG=true APP_URL=http://localhost:8000 ADMINS= +# Moderation triage (deterministic always on; AI risk scoring off unless explicitly enabled) +# The existing OpenAI auto-moderation (ModerateCommit) is separate and keyed on OPENAI_API_KEY +MODERATION_AI_ENABLED=false +# MODERATION_AI_PROVIDER= +# MODERATION_AI_API_KEY= +# MODERATION_AUTHOR_FLOOD_THRESHOLD=3 +# MODERATION_BURST_COUNT=5 +# MODERATION_BURST_MINUTES=10 + APP_LOCALE=en APP_FALLBACK_LOCALE=en APP_FAKER_LOCALE=en_US diff --git a/app/Actions/CreateCommit.php b/app/Actions/CreateCommit.php index fe8a0259e..88e4726b6 100644 --- a/app/Actions/CreateCommit.php +++ b/app/Actions/CreateCommit.php @@ -42,6 +42,7 @@ public function __construct( private MentionedOrganizationsQuery $mentionedOrganizations, private CreateCommitTopics $createCommitTopics, private TomQuery $tomQuery, + private UpdateCommitModeration $updateCommitModeration, ) { // } @@ -89,6 +90,10 @@ public function handle(User $user, string $body, ?Commit $parent = null, ?Commit $this->updateCommitHotScores->handle([$commit->id]); } + if ($commit->status === CommitStatus::Pending) { + $this->updateCommitModeration->handle($commit); + } + $this->createReplyNotifications->handle($commit); $this->createQuoteNotifications->handle($commit); $this->createMentionNotifications->handle($commit); diff --git a/app/Actions/CreateCommitTranslation.php b/app/Actions/CreateCommitTranslation.php index 83607841f..d058f14aa 100644 --- a/app/Actions/CreateCommitTranslation.php +++ b/app/Actions/CreateCommitTranslation.php @@ -11,6 +11,7 @@ use App\Services\CommitTranslations; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; +use Laravel\Ai\Responses\StructuredAgentResponse; use RuntimeException; use Throwable; @@ -37,11 +38,12 @@ public function handle(Commit $commit, ?string $language = null): void try { $response = new CommitTranslator($language)->prompt($commit->body); - $source = $response['source_language'] ?? null; - $body = $response['body'] ?? null; + $structured = $response instanceof StructuredAgentResponse ? $response->structured : []; + $source = $structured['source_language'] ?? null; + $body = $structured['body'] ?? null; $reason = match (true) { - $response->structured === [] => 'empty_or_malformed_response', + $structured === [] => 'empty_or_malformed_response', ! is_string($source), ! is_string($body) => 'invalid_response_shape', ! in_array($source, [...$this->translations->languages(), 'und'], true) => 'unsupported_source_language', $language !== null && $source !== 'und' && blank($body) => 'empty_translation', diff --git a/app/Actions/UpdateCommitModeration.php b/app/Actions/UpdateCommitModeration.php new file mode 100644 index 000000000..2524e2901 --- /dev/null +++ b/app/Actions/UpdateCommitModeration.php @@ -0,0 +1,67 @@ +deterministic->bodyHash($commit->body); + + $commit->forceFill(['body_hash' => $bodyHash])->saveQuietly(); + + $signals = $this->deterministic->handle($commit->refresh()); + $signals = [...$signals, ...$this->aiSignals($commit)]; + + $score = array_sum(array_column($signals, 'weight')); + $cap = config('moderation.score_cap'); + $cap = is_numeric($cap) ? (int) $cap : 100; + + $commit->forceFill([ + 'moderation_score' => min($cap, $score), + 'moderation_signals' => $signals === [] ? null : $signals, + ])->saveQuietly(); + } + + /** + * @return list + */ + private function aiSignals(Commit $commit): array + { + if (! config('moderation.ai.enabled')) { + return []; + } + + if (! app()->bound(AiModerationProvider::class)) { + Log::warning('Moderation AI is enabled but no provider is bound.', [ + 'commit_id' => $commit->id, + ]); + + return []; + } + + try { + return resolve(AiModerationProvider::class)->signals($commit); + } catch (Throwable $throwable) { + Log::warning('Moderation AI scoring failed; keeping deterministic score.', [ + 'commit_id' => $commit->id, + 'exception' => $throwable->getMessage(), + ]); + + return []; + } + } +} diff --git a/app/Actions/UpdateCommitStatuses.php b/app/Actions/UpdateCommitStatuses.php index 162615799..f8be1f1bc 100644 --- a/app/Actions/UpdateCommitStatuses.php +++ b/app/Actions/UpdateCommitStatuses.php @@ -13,6 +13,7 @@ use App\Models\Notification; use App\Models\Poll; use App\Models\User; +use App\Moderation\DeterministicModerationSignals; use App\Queries\CommitBuilder; use App\Queries\TomQuery; use Illuminate\Support\Facades\DB; @@ -28,6 +29,7 @@ public function __construct( private CreateWorldReply $createWorldReply, private UpdateCommitHotScores $updateCommitHotScores, private TomQuery $tomQuery, + private DeterministicModerationSignals $moderationSignals, ) { // } @@ -60,6 +62,13 @@ public function handle(array $commitIds, CommitStatus $status, ?string $reason = ->delete(); if ($status === CommitStatus::Rejected) { + // Commits approved at creation were never hashed, so a pending copy would miss them as duplicates. + $commits + ->filter(fn (Commit $commit): bool => $commit->body_hash === null) + ->each(fn (Commit $commit): int => Commit::query() + ->whereKey($commit->id) + ->update(['body_hash' => $this->moderationSignals->bodyHash($commit->body)])); + Notification::query()->insert($commits->map(fn (Commit $commit): array => [ 'id' => Str::uuid7()->toString(), 'recipient_id' => $commit->user_id, diff --git a/app/Ai/Tools/CommunityStats.php b/app/Ai/Tools/CommunityStats.php index 2ed1bf74e..9f958cc03 100644 --- a/app/Ai/Tools/CommunityStats.php +++ b/app/Ai/Tools/CommunityStats.php @@ -40,7 +40,7 @@ public function handle(Request $request): string return 'Provide one location name between 2 and 80 characters.'; } - if ($metric === 'bio_language' && ! array_key_exists($term, CommunityStatsQuery::Languages)) { + if ($metric === 'bio_language' && ! array_key_exists((string) $term, CommunityStatsQuery::Languages)) { return 'Unsupported language. Choose one of: '.implode(', ', array_keys(CommunityStatsQuery::Languages)).'.'; } diff --git a/app/Contracts/Moderation/AiModerationProvider.php b/app/Contracts/Moderation/AiModerationProvider.php new file mode 100644 index 000000000..bee3f7596 --- /dev/null +++ b/app/Contracts/Moderation/AiModerationProvider.php @@ -0,0 +1,15 @@ + + */ + public function signals(Commit $commit): array; +} diff --git a/app/Http/Controllers/ModerationController.php b/app/Http/Controllers/ModerationController.php index e9327401c..f83aad10e 100644 --- a/app/Http/Controllers/ModerationController.php +++ b/app/Http/Controllers/ModerationController.php @@ -35,10 +35,11 @@ public function show(ShowModerationRequest $request, #[CurrentUser] User $user): ->withStarsFor($user) ->withPollFor($user) ->afterQuery(fn (Collection $commits) => resolve(CommitTranslations::class)->attach($commits, includePending: true)) + ->orderByDesc('moderation_score') ->oldest() ->oldest('id') ->cursorPaginate(20) - ->through($this->withAiDecision(...))), + ->through($this->withModeration(...))), ]); } @@ -56,10 +57,10 @@ public function update(UpdateModerationRequest $request, UpdateCommitStatuses $a } /** - * The AI's decision is hidden everywhere else, but moderators see it. + * The AI's decision and the risk score are hidden everywhere else, but moderators see them. */ - private function withAiDecision(Commit $commit): Commit + private function withModeration(Commit $commit): Commit { - return $commit->makeVisible(['ai_decision', 'ai_categories']); + return $commit->makeVisible(['ai_decision', 'ai_categories', 'moderation_score', 'moderation_signals']); } } diff --git a/app/Models/Commit.php b/app/Models/Commit.php index e91a35d4c..f6019b046 100644 --- a/app/Models/Commit.php +++ b/app/Models/Commit.php @@ -33,6 +33,9 @@ * @property-read array|null $mentions * @property-read array|null $organization_mentions * @property-read CommitStatus $status + * @property-read int $moderation_score + * @property-read list|null $moderation_signals + * @property-read string|null $body_hash * @property-read string|null $link_preview_id * @property-read string|null $quoted_commit_id * @property-read CommitStatus|null $ai_decision @@ -61,6 +64,9 @@ 'ai_decision', 'ai_categories', 'embedding', + 'moderation_score', + 'moderation_signals', + 'body_hash', ])] #[UseEloquentBuilder(CommitBuilder::class)] final class Commit extends Model @@ -118,6 +124,9 @@ public function casts(): array 'mentions' => 'array', 'organization_mentions' => 'array', 'status' => CommitStatus::class, + 'moderation_score' => 'integer', + 'moderation_signals' => 'array', + 'body_hash' => 'string', 'link_preview_id' => 'string', 'quoted_commit_id' => 'string', 'ai_decision' => CommitStatus::class, diff --git a/app/Moderation/DeterministicModerationSignals.php b/app/Moderation/DeterministicModerationSignals.php new file mode 100644 index 000000000..05b89f5ae --- /dev/null +++ b/app/Moderation/DeterministicModerationSignals.php @@ -0,0 +1,127 @@ + + */ + public function handle(Commit $commit): array + { + $signals = []; + + if ($this->isDuplicate($commit)) { + $signals[] = $this->signal('duplicate'); + } + + if ($this->isAuthorFlood($commit)) { + $signals[] = $this->signal('author_flood'); + } + + if ($this->isBurst($commit)) { + $signals[] = $this->signal('burst'); + } + + if ($this->urls->first($commit->body) !== null) { + $signals[] = $this->signal('has_url'); + } + + if ($this->isInviteLike($commit->body)) { + $signals[] = $this->signal('invite_like'); + } + + return $signals; + } + + public function bodyHash(string $body): string + { + $normalized = mb_strtolower(mb_trim((string) preg_replace('/\s+/u', ' ', $body))); + + return hash('sha256', $normalized); + } + + /** + * @return array{code: string, weight: int} + */ + private function signal(string $code): array + { + return [ + 'code' => $code, + 'weight' => $this->integerConfig("moderation.weights.{$code}", 0), + ]; + } + + private function isDuplicate(Commit $commit): bool + { + // Image-only commits share the empty body's hash, so there is no text to compare. + if (mb_trim($commit->body) === '') { + return false; + } + + $hash = $commit->body_hash ?? $this->bodyHash($commit->body); + + // Short replies like "thanks!" match across users all the time, so another user's text only counts once a + // moderator has rejected it. + return Commit::query() + ->where('body_hash', $hash) + ->whereKeyNot($commit->id) + ->where(fn (CommitBuilder $query): CommitBuilder => $query + ->where('status', CommitStatus::Rejected) + ->orWhere(fn (CommitBuilder $query): CommitBuilder => $query + ->where('user_id', $commit->user_id) + ->pending())) + ->exists(); + } + + private function isAuthorFlood(Commit $commit): bool + { + $threshold = $this->integerConfig('moderation.author_flood_threshold', 3); + + return Commit::query() + ->where('user_id', $commit->user_id) + ->pending() + ->count() >= $threshold; + } + + private function isBurst(Commit $commit): bool + { + $count = $this->integerConfig('moderation.burst.count', 5); + $minutes = $this->integerConfig('moderation.burst.minutes', 10); + + // The window ends when the commit was posted, so a backfill scores the burst as it was at submission. Commits + // posted after this one in the same second are left out, just as they were not there yet. + return Commit::query() + ->where('user_id', $commit->user_id) + ->where('created_at', '>=', $commit->created_at->copy()->subMinutes($minutes)) + ->whereNot(fn (CommitBuilder $query): CommitBuilder => $query->newerThan($commit)) + ->count() >= $count; + } + + private function isInviteLike(string $body): bool + { + return preg_match( + '/\b(join\s+(my|our)\s+(discord|telegram|whatsapp)|dm\s+me|send\s+me\s+(a\s+)?dm|click\s+(the\s+)?link|free\s+crypto|whatsapp\.me|t\.me\/)/iu', + $body, + ) === 1; + } + + private function integerConfig(string $key, int $default): int + { + $value = config($key); + + return is_numeric($value) ? (int) $value : $default; + } +} diff --git a/app/Queries/CommitBuilder.php b/app/Queries/CommitBuilder.php index e52544324..415cc94f1 100644 --- a/app/Queries/CommitBuilder.php +++ b/app/Queries/CommitBuilder.php @@ -31,6 +31,7 @@ final class CommitBuilder extends Builder private const array COLUMNS_BUT_EMBEDDING = [ 'id', 'user_id', 'body', 'created_at', 'updated_at', 'status', 'parent_id', 'deleted_at', 'link_preview_id', 'quoted_commit_id', 'mentions', 'ai_decision', 'ai_categories', 'root_id', 'hot_score', 'pasteis_count', + 'moderation_score', 'moderation_signals', 'body_hash', ]; /** diff --git a/config/moderation.php b/config/moderation.php new file mode 100644 index 000000000..7c98c9f78 --- /dev/null +++ b/config/moderation.php @@ -0,0 +1,50 @@ + [ + 'duplicate' => 50, + 'author_flood' => 40, + 'burst' => 25, + 'has_url' => 10, + 'invite_like' => 30, + ], + + 'score_cap' => 100, + + 'author_flood_threshold' => (int) env('MODERATION_AUTHOR_FLOOD_THRESHOLD', 3), + + 'burst' => [ + 'count' => (int) env('MODERATION_BURST_COUNT', 5), + 'minutes' => (int) env('MODERATION_BURST_MINUTES', 10), + ], + + /* + |-------------------------------------------------------------------------- + | AI Moderation (optional) + |-------------------------------------------------------------------------- + | + | AI scoring is off by default. Enable only when you have budget and a + | provider bound. The app boots and moderates without any AI credentials. + | + */ + + 'ai' => [ + 'enabled' => (bool) env('MODERATION_AI_ENABLED', false), + 'provider' => env('MODERATION_AI_PROVIDER'), + 'api_key' => env('MODERATION_AI_API_KEY'), + ], + +]; diff --git a/database/factories/CommitFactory.php b/database/factories/CommitFactory.php index 1f19fb2b6..51247ea99 100644 --- a/database/factories/CommitFactory.php +++ b/database/factories/CommitFactory.php @@ -26,6 +26,9 @@ public function definition(): array 'body' => fake()->realText(280), 'status' => CommitStatus::Approved, 'mentions' => [], + 'moderation_score' => 0, + 'moderation_signals' => null, + 'body_hash' => null, ]; } @@ -58,4 +61,15 @@ public function rejected(): self 'status' => CommitStatus::Rejected, ]); } + + /** + * @param list|null $signals + */ + public function scored(int $score, ?array $signals = null): self + { + return $this->state(fn (array $attributes): array => [ + 'moderation_score' => $score, + 'moderation_signals' => $signals, + ]); + } } diff --git a/database/migrations/2026_10_18_160000_add_moderation_scoring_to_commits_table.php b/database/migrations/2026_10_18_160000_add_moderation_scoring_to_commits_table.php new file mode 100644 index 000000000..443893c21 --- /dev/null +++ b/database/migrations/2026_10_18_160000_add_moderation_scoring_to_commits_table.php @@ -0,0 +1,22 @@ +unsignedSmallInteger('moderation_score')->default(0); + $table->json('moderation_signals')->nullable(); + $table->char('body_hash', 64)->nullable(); + + $table->index('body_hash'); + $table->index(['status', 'moderation_score', 'created_at']); + }); + } +}; diff --git a/database/migrations/2026_10_18_200000_backfill_moderation_scoring_on_commits.php b/database/migrations/2026_10_18_200000_backfill_moderation_scoring_on_commits.php new file mode 100644 index 000000000..abafe5e34 --- /dev/null +++ b/database/migrations/2026_10_18_200000_backfill_moderation_scoring_on_commits.php @@ -0,0 +1,42 @@ +toBase()->whereIn('status', [CommitStatus::Pending, CommitStatus::Rejected])->exists()) { + return; + } + + // Hash first so every pending commit can match the pending and rejected ones, whichever order they are scored in. + $signals = resolve(DeterministicModerationSignals::class); + + Commit::query() + ->whereIn('status', [CommitStatus::Pending, CommitStatus::Rejected]) + ->chunkById(500, function ($commits) use ($signals): void { + foreach ($commits as $commit) { + $commit->forceFill(['body_hash' => $signals->bodyHash($commit->body)])->saveQuietly(); + } + }); + + Commit::query() + ->where('status', CommitStatus::Pending) + ->chunkById(100, function ($commits): void { + foreach ($commits as $commit) { + resolve(UpdateCommitModeration::class)->handle($commit); + } + }); + } +}; diff --git a/resources/js/pages/moderation/show.tsx b/resources/js/pages/moderation/show.tsx index 8da5ad834..66b71a65d 100644 --- a/resources/js/pages/moderation/show.tsx +++ b/resources/js/pages/moderation/show.tsx @@ -264,7 +264,7 @@ export default function Show({

Moderation

- Oldest first + Highest risk first

@@ -324,6 +324,23 @@ export default function Show({ actions={(commit) => ( <> + {commit.moderation_signals && + commit.moderation_signals.length > 0 && ( +
    + {commit.moderation_signals.map((signal) => ( +
  • + {signal.code} +
  • + ))} +
+ )} assertRedirectToRoute('log.show', Log::Following); - expect(array_column(DB::getQueryLog(), 'query')) - ->not->toBeEmpty() - ->each->not->toContain('select count(*) as "aggregate" from "follows"'); + $queries = array_column(DB::getQueryLog(), 'query'); + + expect($queries)->not->toBeEmpty(); + + expect(array_values(array_filter( + $queries, + fn (string $query): bool => str_contains($query, 'count(*)') + && (str_contains($query, 'follows') || str_contains($query, '"users"')), + )))->toBeEmpty(); }); it('rejects non-text commit bodies with a validation error', function (mixed $body): void { diff --git a/tests/Feature/Controllers/ModerationControllerTest.php b/tests/Feature/Controllers/ModerationControllerTest.php index fc395a6ed..64e90f1ce 100644 --- a/tests/Feature/Controllers/ModerationControllerTest.php +++ b/tests/Feature/Controllers/ModerationControllerTest.php @@ -18,11 +18,24 @@ config(['app.admins' => ['admin']]); }); -it('renders pending commits oldest first', function (): void { - $admin = User::factory()->create(['handle' => 'admin']); +it('renders pending commits highest risk first then oldest', function (): void { + $admin = User::factory()->create(['username' => 'admin']); - Commit::factory()->pending()->create(['body' => 'Newer']); - Commit::factory()->pending()->create(['body' => 'Older', 'created_at' => now()->subMinute()]); + Commit::factory()->pending()->scored(10)->create([ + 'body' => 'Low risk newer', + 'created_at' => now(), + ]); + Commit::factory()->pending()->scored(10)->create([ + 'body' => 'Low risk older', + 'created_at' => now()->subMinute(), + ]); + Commit::factory()->pending()->scored(80, [ + ['code' => 'duplicate', 'weight' => 50], + ['code' => 'has_url', 'weight' => 10], + ])->create([ + 'body' => 'High risk', + 'created_at' => now()->subSeconds(30), + ]); Commit::factory()->create(); Commit::factory()->rejected()->create(); @@ -32,11 +45,14 @@ $response->assertOk() ->assertInertia(fn ($page) => $page ->component('moderation/show') - ->has('commits.data', 2) - ->where('commits.data.0.body', 'Older') + ->has('commits.data', 3) + ->where('commits.data.0.body', 'High risk') + ->where('commits.data.0.moderation_score', 80) + ->where('commits.data.0.moderation_signals.0.code', 'duplicate') ->where('commits.data.0.stars_count', 0) ->where('commits.data.0.starred', false) - ->where('commits.data.1.body', 'Newer') + ->where('commits.data.1.body', 'Low risk older') + ->where('commits.data.2.body', 'Low risk newer') ->has('commits.data.0.user')); }); diff --git a/tests/Unit/Actions/CreateCommitTest.php b/tests/Unit/Actions/CreateCommitTest.php index b6f119c46..279a71bc6 100644 --- a/tests/Unit/Actions/CreateCommitTest.php +++ b/tests/Unit/Actions/CreateCommitTest.php @@ -42,6 +42,36 @@ expect($commit->status)->toBe(CommitStatus::Pending); }); +it('scores pending commits for moderation triage', function (): void { + Queue::fake(); + + $action = resolve(CreateCommit::class); + + $commit = $action->handle(User::factory()->create(), 'Join my discord https://example.com'); + + expect($commit->refresh()) + ->status->toBe(CommitStatus::Pending) + ->body_hash->not->toBeNull() + ->moderation_score->toBeGreaterThan(0) + ->and(array_column($commit->moderation_signals ?? [], 'code')) + ->toContain('invite_like') + ->toContain('has_url'); +}); + +it('does not score admin commits that are approved immediately', function (): void { + config(['app.admins' => ['nunomaduro']]); + + $action = resolve(CreateCommit::class); + + $commit = $action->handle(User::factory()->create(['username' => 'nunomaduro']), 'Join my discord'); + + expect($commit->refresh()) + ->status->toBe(CommitStatus::Approved) + ->moderation_score->toBe(0) + ->moderation_signals->toBeNull() + ->body_hash->toBeNull(); +}); + it('stores emoji shortcodes as their emoji', function (): void { $action = resolve(CreateCommit::class); diff --git a/tests/Unit/Actions/UpdateCommitModerationTest.php b/tests/Unit/Actions/UpdateCommitModerationTest.php new file mode 100644 index 000000000..d976890ad --- /dev/null +++ b/tests/Unit/Actions/UpdateCommitModerationTest.php @@ -0,0 +1,111 @@ +pending()->create([ + 'body' => 'Join my telegram channel now', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + $commit->refresh(); + + expect($commit->body_hash)->toBe(hash('sha256', 'join my telegram channel now')) + ->and($commit->moderation_score)->toBeGreaterThan(0) + ->and($commit->moderation_signals)->toBeArray() + ->and(array_column($commit->moderation_signals, 'code'))->toContain('invite_like'); +}); + +it('caps the score', function (): void { + config([ + 'moderation.score_cap' => 15, + 'moderation.weights.invite_like' => 30, + 'moderation.weights.has_url' => 10, + ]); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'Join my discord https://example.com', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + expect($commit->refresh()->moderation_score)->toBe(15); +}); + +it('does not call the AI provider when AI is disabled', function (): void { + config(['moderation.ai.enabled' => false]); + + $this->mock(AiModerationProvider::class, function (MockInterface $mock): void { + $mock->shouldNotReceive('signals'); + }); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'Ordinary commit body', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + expect($commit->refresh()->moderation_score)->toBe(0); +}); + +it('keeps deterministic scoring when AI is enabled without a provider', function (): void { + config(['moderation.ai.enabled' => true]); + + expect(app()->bound(AiModerationProvider::class))->toBeFalse(); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'Click the link https://example.com', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + $codes = array_column($commit->refresh()->moderation_signals ?? [], 'code'); + + expect($codes)->toContain('has_url') + ->and($codes)->toContain('invite_like'); +}); + +it('merges AI signals when AI is enabled', function (): void { + config(['moderation.ai.enabled' => true]); + + $this->mock(AiModerationProvider::class, function (MockInterface $mock): void { + $mock->shouldReceive('signals') + ->once() + ->andReturn([['code' => 'ai_threat', 'weight' => 20]]); + }); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'Ordinary commit body', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + expect($commit->refresh()->moderation_score)->toBe(20) + ->and(array_column($commit->moderation_signals, 'code'))->toContain('ai_threat'); +}); + +it('keeps deterministic scoring when the AI provider throws', function (): void { + config(['moderation.ai.enabled' => true]); + + $this->mock(AiModerationProvider::class, function (MockInterface $mock): void { + $mock->shouldReceive('signals')->once()->andThrow(new RuntimeException('provider down')); + }); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'Click the link https://example.com', + ]); + + resolve(UpdateCommitModeration::class)->handle($commit); + + $codes = array_column($commit->refresh()->moderation_signals ?? [], 'code'); + + expect($codes)->toContain('has_url') + ->and($codes)->toContain('invite_like') + ->and($codes)->not->toContain('ai_threat'); +}); diff --git a/tests/Unit/Actions/UpdateCommitStatusesTest.php b/tests/Unit/Actions/UpdateCommitStatusesTest.php index 67fc69ae9..528ed19fe 100644 --- a/tests/Unit/Actions/UpdateCommitStatusesTest.php +++ b/tests/Unit/Actions/UpdateCommitStatusesTest.php @@ -10,6 +10,7 @@ use App\Jobs\ReplyAsTom; use App\Models\Commit; use App\Models\User; +use App\Moderation\DeterministicModerationSignals; use App\Services\TheAlgorithm; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; @@ -71,6 +72,19 @@ ->and($rejected->refresh()->status)->toBe(CommitStatus::Rejected); }); +it('hashes the bodies of commits it rejects so pending copies match them', function (): void { + $approved = Commit::factory()->create(['body' => 'Buy my stuff', 'body_hash' => null]); + + resolve(UpdateCommitStatuses::class)->handle([$approved->id], CommitStatus::Rejected); + + $copy = Commit::factory()->pending()->create(['body' => 'buy my stuff']); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($copy), 'code'); + + expect($approved->refresh()->body_hash)->not->toBeNull() + ->and($codes)->toContain('duplicate'); +}); + it('notifies the parent authors of the approved replies', function (): void { $parent = Commit::factory()->create(); $replies = Commit::factory()->pending()->for($parent, 'parent')->count(2)->create(); diff --git a/tests/Unit/DeterministicModerationSignalsTest.php b/tests/Unit/DeterministicModerationSignalsTest.php new file mode 100644 index 000000000..c22288078 --- /dev/null +++ b/tests/Unit/DeterministicModerationSignalsTest.php @@ -0,0 +1,171 @@ +bodyHash(" Hello World\n")) + ->toBe($signals->bodyHash('hello world')); +}); + +it('does not flag the same text pending from another user', function (): void { + $hash = resolve(DeterministicModerationSignals::class)->bodyHash('thanks!'); + + Commit::factory()->pending()->create(['body' => 'Thanks!', 'body_hash' => $hash]); + $commit = Commit::factory()->pending()->create(['body' => 'thanks!', 'body_hash' => $hash]); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->not->toContain('duplicate'); +}); + +it('flags duplicates of the same author pending bodies', function (): void { + $hash = resolve(DeterministicModerationSignals::class)->bodyHash('same message'); + $user = User::factory()->create(); + + Commit::factory()->pending()->for($user)->create([ + 'body' => 'same message', + 'body_hash' => $hash, + ]); + + $commit = Commit::factory()->pending()->for($user)->create([ + 'body' => 'Same Message', + 'body_hash' => $hash, + ]); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('duplicate'); +}); + +it('flags duplicates of rejected bodies', function (): void { + $hash = resolve(DeterministicModerationSignals::class)->bodyHash('rejected twin'); + + Commit::factory()->rejected()->create([ + 'body' => 'rejected twin', + 'body_hash' => $hash, + ]); + + $commit = Commit::factory()->pending()->create([ + 'body' => 'rejected twin', + 'body_hash' => $hash, + ]); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('duplicate'); +}); + +it('flags author flood when pending commits meet the threshold', function (): void { + config(['moderation.author_flood_threshold' => 3]); + + $user = User::factory()->create(); + Commit::factory()->pending()->for($user)->count(3)->create(); + + $commit = Commit::query()->where('user_id', $user->id)->pending()->latest('id')->firstOrFail(); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('author_flood'); +}); + +it('flags burst posting', function (): void { + config([ + 'moderation.burst.count' => 3, + 'moderation.burst.minutes' => 10, + ]); + + $user = User::factory()->create(); + Commit::factory()->for($user)->count(3)->create([ + 'status' => CommitStatus::Pending, + 'created_at' => now()->subMinute(), + ]); + + $commit = Commit::query()->where('user_id', $user->id)->latest('id')->firstOrFail(); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('burst'); +}); + +it('measures bursts from when the commit was posted', function (): void { + config([ + 'moderation.burst.count' => 3, + 'moderation.burst.minutes' => 10, + ]); + + $user = User::factory()->create(); + Commit::factory()->for($user)->count(3)->create([ + 'status' => CommitStatus::Pending, + 'created_at' => now()->subDay(), + ]); + + $old = Commit::query()->where('user_id', $user->id)->latest('id')->firstOrFail(); + $recent = Commit::factory()->for($user)->pending()->create(['created_at' => now()]); + + $signals = resolve(DeterministicModerationSignals::class); + + expect(array_column($signals->handle($old), 'code'))->toContain('burst') + ->and(array_column($signals->handle($recent), 'code'))->not->toContain('burst'); +}); + +it('does not count commits posted after it in the same second as a burst', function (): void { + config([ + 'moderation.burst.count' => 3, + 'moderation.burst.minutes' => 10, + ]); + + $user = User::factory()->create(); + $commits = Commit::factory()->for($user)->count(3)->create([ + 'status' => CommitStatus::Pending, + 'created_at' => now()->subMinute(), + ]); + + $signals = resolve(DeterministicModerationSignals::class); + + expect(array_column($signals->handle($commits->first()), 'code'))->not->toContain('burst') + ->and(array_column($signals->handle($commits->last()), 'code'))->toContain('burst'); +}); + +it('flags urls', function (): void { + $commit = Commit::factory()->pending()->create([ + 'body' => 'Check https://example.com/path', + ]); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('has_url'); +}); + +it('flags invite-like copy', function (): void { + $commit = Commit::factory()->pending()->create([ + 'body' => 'Join my discord for free tips', + ]); + + $codes = array_column(resolve(DeterministicModerationSignals::class)->handle($commit), 'code'); + + expect($codes)->toContain('invite_like'); +}); + +it('returns no signals for ordinary text', function (): void { + $commit = Commit::factory()->pending()->create([ + 'body' => 'Shipped a quiet refactor of the log query.', + ]); + + expect(resolve(DeterministicModerationSignals::class)->handle($commit))->toBe([]); +}); + +it('does not flag image-only commits as duplicates of each other', function (): void { + $signals = resolve(DeterministicModerationSignals::class); + + Commit::factory()->pending()->create(['body' => '', 'body_hash' => $signals->bodyHash('')]); + $commit = Commit::factory()->pending()->create(['body' => '', 'body_hash' => $signals->bodyHash('')]); + + expect(array_column($signals->handle($commit), 'code'))->not->toContain('duplicate'); +}); diff --git a/tests/Unit/Models/CommitTest.php b/tests/Unit/Models/CommitTest.php index 576130e08..d25401026 100644 --- a/tests/Unit/Models/CommitTest.php +++ b/tests/Unit/Models/CommitTest.php @@ -33,6 +33,12 @@ 'body_segments', 'preview_segments', 'is_checking', + 'moderation_score', + 'moderation_signals', + 'body_hash', + 'body_segments', + 'preview_segments', + 'is_checking', ]); }); @@ -452,3 +458,9 @@ expect($tombstone->toArray())->not->toHaveKey('images'); }); + +it('hides its moderation metadata by default', function (): void { + $commit = Commit::factory()->create(['moderation_score' => 50, 'moderation_signals' => [['code' => 'duplicate', 'weight' => 50]], 'body_hash' => 'hash']); + + expect($commit->toArray())->not->toHaveKeys(['moderation_score', 'moderation_signals', 'body_hash']); +}); diff --git a/tests/draft-images.config.ts b/tests/draft-images.config.ts index 4a56c462f..b91094292 100644 --- a/tests/draft-images.config.ts +++ b/tests/draft-images.config.ts @@ -12,5 +12,6 @@ export default defineConfig({ VITE_APP_NAME: '/dev/null', }, include: ['tests/Unit/*.test.tsx'], + }, });