Add content moderation support via OpenAI moderations endpoint

config/ai.php

@@ -19,6 +19,7 @@
     'default_for_transcription' => 'openai',
     'default_for_embeddings' => 'openai',
     'default_for_reranking' => 'cohere',
+    'default_for_moderation' => 'openai',
 
     /*
     |--------------------------------------------------------------------------

src/AiManager.php

@@ -9,6 +9,7 @@
 use Laravel\Ai\Contracts\Providers\EmbeddingProvider;
 use Laravel\Ai\Contracts\Providers\FileProvider;
 use Laravel\Ai\Contracts\Providers\ImageProvider;
+use Laravel\Ai\Contracts\Providers\ModerationProvider;
 use Laravel\Ai\Contracts\Providers\RerankingProvider;
 use Laravel\Ai\Contracts\Providers\StoreProvider;
 use Laravel\Ai\Contracts\Providers\TextProvider;
@@ -39,6 +40,7 @@ class AiManager extends MultipleInstanceManager
     use Concerns\InteractsWithFakeEmbeddings;
     use Concerns\InteractsWithFakeFiles;
     use Concerns\InteractsWithFakeImages;
+    use Concerns\InteractsWithFakeModeration;
     use Concerns\InteractsWithFakeReranking;
     use Concerns\InteractsWithFakeStores;
     use Concerns\InteractsWithFakeTranscriptions;
@@ -134,6 +136,34 @@ public function fakeableRerankingProvider(?string $name = null): RerankingProvid
             : $provider;
     }
 
+    /**
+     * Get a moderation provider instance by name.
+     *
+     * @throws LogicException
+     */
+    public function moderationProvider(?string $name = null): ModerationProvider
+    {
+        return tap($this->instance($name), function ($instance) {
+            if (! $instance instanceof ModerationProvider) {
+                throw new LogicException('Provider ['.$instance::class.'] does not support moderation.');
+            }
+        });
+    }
+
+    /**
+     * Get a moderation provider instance, using a fake gateway if moderation is faked.
+     *
+     * @throws LogicException
+     */
+    public function fakeableModerationProvider(?string $name = null): ModerationProvider
+    {
+        $provider = $this->moderationProvider($name);
+
+        return $this->moderationIsFaked()
+            ? (clone $provider)->useModerationGateway($this->fakeModerationGateway())
+            : $provider;
+    }
+
     /**
      * Get a provider instance by name.
      *

src/AiServiceProvider.php

@@ -56,6 +56,14 @@ public function boot(): void
             return $request->generate(provider: $provider, model: $model)->embeddings[0];
         });
 
+        // Moderation macro...
+        Stringable::macro('moderate', function (
+            ?string $provider = null,
+            ?string $model = null,
+        ) {
+            return Moderation::check($this->value, provider: $provider, model: $model);
+        });
+
         // Reranking macro...
         Collection::macro('rerank', function (
             Closure|array|string $by,

src/Concerns/InteractsWithFakeModeration.php

@@ -0,0 +1,99 @@
+<?php
+
+namespace Laravel\Ai\Concerns;
+
+use Closure;
+use Illuminate\Support\Collection;
+use Laravel\Ai\Gateway\FakeModerationGateway;
+use Laravel\Ai\Prompts\ModerationPrompt;
+use PHPUnit\Framework\Assert as PHPUnit;
+
+trait InteractsWithFakeModeration
+{
+    /**
+     * The fake moderation gateway instance.
+     */
+    protected ?FakeModerationGateway $fakeModerationGateway = null;
+
+    /**
+     * All of the recorded moderations.
+     */
+    protected array $recordedModerations = [];
+
+    /**
+     * Fake moderation operations.
+     */
+    public function fakeModeration(Closure|array $responses = []): FakeModerationGateway
+    {
+        return $this->fakeModerationGateway = new FakeModerationGateway($responses);
+    }
+
+    /**
+     * Record a moderation.
+     */
+    public function recordModeration(ModerationPrompt $prompt): self
+    {
+        $this->recordedModerations[] = $prompt;
+
+        return $this;
+    }
+
+    /**
+     * Assert that a moderation was performed matching a given truth test.
+     */
+    public function assertChecked(Closure $callback): self
+    {
+        PHPUnit::assertTrue(
+            (new Collection($this->recordedModerations))->contains(function (ModerationPrompt $prompt) use ($callback) {
+                return $callback($prompt);
+            }),
+            'An expected moderation check was not recorded.'
+        );
+
+        return $this;
+    }
+
+    /**
+     * Assert that a moderation was not performed matching a given truth test.
+     */
+    public function assertNotChecked(Closure $callback): self
+    {
+        PHPUnit::assertTrue(
+            (new Collection($this->recordedModerations))->doesntContain(function (ModerationPrompt $prompt) use ($callback) {
+                return $callback($prompt);
+            }),
+            'An unexpected moderation check was recorded.'
+        );
+
+        return $this;
+    }
+
+    /**
+     * Assert that no moderations were performed.
+     */
+    public function assertNothingChecked(): self
+    {
+        PHPUnit::assertEmpty(
+            $this->recordedModerations,
+            'Unexpected moderation checks were recorded.'
+        );
+
+        return $this;
+    }
+
+    /**
+     * Determine if moderation is faked.
+     */
+    public function moderationIsFaked(): bool
+    {
+        return $this->fakeModerationGateway !== null;
+    }
+
+    /**
+     * Get the fake moderation gateway.
+     */
+    public function fakeModerationGateway(): ?FakeModerationGateway
+    {
+        return $this->fakeModerationGateway;
+    }
+}

src/Contracts/Gateway/ModerationGateway.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace Laravel\Ai\Contracts\Gateway;
+
+use Laravel\Ai\Contracts\Providers\ModerationProvider;
+use Laravel\Ai\Responses\ModerationResponse;
+
+interface ModerationGateway
+{
+    /**
+     * Check the given input for content that may violate usage policies.
+     */
+    public function moderate(
+        ModerationProvider $provider,
+        string $model,
+        string $input
+    ): ModerationResponse;
+}

src/Contracts/Providers/ModerationProvider.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Laravel\Ai\Contracts\Providers;
+
+use Laravel\Ai\Contracts\Gateway\ModerationGateway;
+use Laravel\Ai\Responses\ModerationResponse;
+
+interface ModerationProvider
+{
+    /**
+     * Check the given input for content that may violate usage policies.
+     */
+    public function moderate(string $input, ?string $model = null): ModerationResponse;
+
+    /**
+     * Get the provider's moderation gateway.
+     */
+    public function moderationGateway(): ModerationGateway;
+
+    /**
+     * Set the provider's moderation gateway.
+     */
+    public function useModerationGateway(ModerationGateway $gateway): self;
+
+    /**
+     * Get the name of the default moderation model.
+     */
+    public function defaultModerationModel(): string;
+}

src/Events/Moderated.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace Laravel\Ai\Events;
+
+use Laravel\Ai\Prompts\ModerationPrompt;
+use Laravel\Ai\Providers\Provider;
+use Laravel\Ai\Responses\ModerationResponse;
+
+class Moderated
+{
+    public function __construct(
+        public string $invocationId,
+        public Provider $provider,
+        public string $model,
+        public ModerationPrompt $prompt,
+        public ModerationResponse $response,
+    ) {}
+}

src/Events/Moderating.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace Laravel\Ai\Events;
+
+use Laravel\Ai\Prompts\ModerationPrompt;
+use Laravel\Ai\Providers\Provider;
+
+class Moderating
+{
+    public function __construct(
+        public string $invocationId,
+        public Provider $provider,
+        public string $model,
+        public ModerationPrompt $prompt,
+    ) {}
+}

src/Gateway/FakeModerationGateway.php

@@ -0,0 +1,125 @@
+<?php
+
+namespace Laravel\Ai\Gateway;
+
+use Closure;
+use Laravel\Ai\Contracts\Gateway\ModerationGateway;
+use Laravel\Ai\Contracts\Providers\ModerationProvider;
+use Laravel\Ai\Prompts\ModerationPrompt;
+use Laravel\Ai\Responses\Data\Meta;
+use Laravel\Ai\Responses\Data\ModerationCategory;
+use Laravel\Ai\Responses\ModerationResponse;
+use RuntimeException;
+
+class FakeModerationGateway implements ModerationGateway
+{
+    protected int $currentResponseIndex = 0;
+
+    protected bool $preventStrayModerations = false;
+
+    public function __construct(
+        protected Closure|array $responses = [],
+    ) {}
+
+    /**
+     * Check the given input for content that may violate usage policies.
+     */
+    public function moderate(
+        ModerationProvider $provider,
+        string $model,
+        string $input
+    ): ModerationResponse {
+        $prompt = new ModerationPrompt($input, $provider, $model);
+
+        return $this->nextResponse($provider, $model, $prompt);
+    }
+
+    /**
+     * Get the next response instance.
+     */
+    protected function nextResponse(
+        ModerationProvider $provider,
+        string $model,
+        ModerationPrompt $prompt
+    ): ModerationResponse {
+        $response = is_array($this->responses)
+            ? ($this->responses[$this->currentResponseIndex] ?? null)
+            : call_user_func($this->responses, $prompt);
+
+        return tap($this->marshalResponse(
+            $response, $provider, $model, $prompt
+        ), fn () => $this->currentResponseIndex++);
+    }
+
+    /**
+     * Marshal the given response into a full response instance.
+     */
+    protected function marshalResponse(
+        mixed $response,
+        ModerationProvider $provider,
+        string $model,
+        ModerationPrompt $prompt
+    ): ModerationResponse {
+        if ($response instanceof Closure) {
+            $response = $response($prompt);
+        }
+
+        if (is_null($response)) {
+            if ($this->preventStrayModerations) {
+                throw new RuntimeException('Attempted moderation without a fake response.');
+            }
+
+            $response = $this->generateFakeModeration();
+        }
+
+        if ($response instanceof ModerationResponse) {
+            return $response;
+        }
+
+        if (is_array($response) && isset($response[0]) && $response[0] instanceof ModerationCategory) {
+            $flagged = array_any($response, fn (ModerationCategory $category) => $category->flagged);
+
+            return new ModerationResponse(
+                $flagged,
+                $response,
+                new Meta($provider->name(), $model),
+            );
+        }
+
+        return $response;
+    }
+
+    /**
+     * Generate a fake moderation response.
+     *
+     * @return array<int, ModerationCategory>
+     */
+    protected function generateFakeModeration(): array
+    {
+        return [
+            new ModerationCategory('hate', false, 0.0001),
+            new ModerationCategory('hate/threatening', false, 0.0001),
+            new ModerationCategory('harassment', false, 0.0001),
+            new ModerationCategory('harassment/threatening', false, 0.0001),
+            new ModerationCategory('illicit', false, 0.0001),
+            new ModerationCategory('illicit/violent', false, 0.0001),
+            new ModerationCategory('self-harm', false, 0.0001),
+            new ModerationCategory('self-harm/intent', false, 0.0001),
+            new ModerationCategory('self-harm/instructions', false, 0.0001),
+            new ModerationCategory('sexual', false, 0.0001),
+            new ModerationCategory('sexual/minors', false, 0.0001),
+            new ModerationCategory('violence', false, 0.0001),
+            new ModerationCategory('violence/graphic', false, 0.0001),
+        ];
+    }
+
+    /**
+     * Indicate that an exception should be thrown if any moderation is not faked.
+     */
+    public function preventStrayModerations(bool $prevent = true): self
+    {
+        $this->preventStrayModerations = $prevent;
+
+        return $this;
+    }
+}