Content Moderation System Implementation for Website

Content Moderation System Implementation

A content moderation system checks user content before or after publication. Includes manual moderation, automatic filtering (stop-words, links, ML-models), user complaints, task queue for moderators.

Architecture

[User posts content]
         ↓
[Auto-filter: spam/toxicity check]
    ↙           ↘
[Auto-approve]  [Send to moderation queue]
                         ↓
               [Moderator dashboard]
               ↙          ↘       ↘
         [Approve]    [Reject]  [Shadow-ban]
              ↓           ↓
       [Publish]    [Notify user]

Model with Moderation States

// ModerationStatus: pending, approved, rejected, spam, shadow_banned

class Comment extends Model
{
    use HasModerationStatus;

    protected $casts = [
        'moderation_metadata' => 'array',
        'moderated_at'        => 'datetime',
    ];

    public function scopeVisible(Builder $query): Builder
    {
        return $query->where('status', 'approved');
    }

    public function scopePendingModeration(Builder $query): Builder
    {
        return $query->where('status', 'pending');
    }
}

Automatic Filtering

class ContentModerationService
{
    private array $spamPatterns = [
        '/https?:\/\/[^\s]+/i',         // external links
        '/\b(casino|viagra|loan|crypto)\b/i',
        '/(.)\1{5,}/',                  // repeating characters (aaaaaaa)
    ];

    private array $bannedWords;  // loaded from DB/config

    public function analyze(string $content, User|null $author): ModerationResult
    {
        $signals = [];
        $spamScore = 0;

        // Check stop-words
        foreach ($this->bannedWords as $word) {
            if (stripos($content, $word) !== false) {
                $signals[] = "banned_word:{$word}";
                $spamScore += 40;
            }
        }

        // Check patterns
        foreach ($this->spamPatterns as $pattern) {
            if (preg_match($pattern, $content)) {
                $signals[] = "pattern_match:{$pattern}";
                $spamScore += 30;
            }
        }

        // Author history
        if ($author) {
            $authorSpamRate = $author->comments()
                ->where('status', 'spam')
                ->count() / max($author->comments()->count(), 1);

            if ($authorSpamRate > 0.3) {
                $signals[] = 'author_spam_history';
                $spamScore += 50;
            }
        } else {
            $spamScore += 10;  // anonyme — higher risk
        }

        // Content length
        if (strlen($content) < 5) {
            $signals[] = 'too_short';
            $spamScore += 20;
        }

        return new ModerationResult(
            score: $spamScore,
            signals: $signals,
            decision: match (true) {
                $spamScore >= 80 => ModerationDecision::SPAM,
                $spamScore >= 50 => ModerationDecision::PENDING,
                default          => ModerationDecision::APPROVE,
            }
        );
    }
}

Moderator Dashboard

class ModerationController extends Controller
{
    public function queue(Request $request): JsonResponse
    {
        $items = Comment::pendingModeration()
            ->with('user:id,name,email', 'entity')
            ->when($request->type, fn($q) => $q->where('entity_type', $request->type))
            ->orderBy('moderation_score', 'desc')  // most suspicious first
            ->paginate(50);

        return response()->json($items);
    }

    public function decision(Request $request, Comment $comment): JsonResponse
    {
        $request->validate([
            'action' => 'required|in:approve,reject,spam,shadow_ban',
            'reason' => 'nullable|string|max:500',
        ]);

        $comment->update([
            'status'         => match ($request->action) {
                'approve'     => 'approved',
                'reject'      => 'rejected',
                'spam'        => 'spam',
                'shadow_ban'  => 'shadow_banned',
            },
            'moderated_by'   => auth()->id(),
            'moderated_at'   => now(),
            'rejection_reason' => $request->reason,
        ]);

        if ($request->action === 'approve') {
            event(new CommentApprovedEvent($comment));
        }

        if (in_array($request->action, ['reject', 'spam'])) {
            $comment->user?->notify(new CommentRejectedNotification($comment));
        }

        UpdateAuthorReputationJob::dispatch($comment->user, $request->action);

        return response()->json(['status' => 'ok']);
    }
}

User Complaints

class ReportController extends Controller
{
    public function store(Request $request, Comment $comment): JsonResponse
    {
        $request->validate(['reason' => 'required|in:spam,abuse,misinformation,other']);

        Report::firstOrCreate(
            ['reporter_id' => auth()->id(), 'comment_id' => $comment->id],
            ['reason' => $request->reason]
        );

        $reportCount = Report::where('comment_id', $comment->id)->count();

        if ($reportCount >= 3 && $comment->status === 'approved') {
            $comment->update(['status' => 'pending', 'auto_flagged' => true]);
        }

        return response()->json(['reported' => true]);
    }
}

Implementation Timeline