Add document processing pipeline to Indexer class

- Add optional loader and transformers to Indexer constructor
- Add __invoke method to process sources through full pipeline
- Update example to use enhanced Indexer directly instead of DocumentProcessor
- Maintain backward compatibility with existing index() method

Author: OskarStark

examples/indexer/movies.php

@@ -12,7 +12,6 @@
 use Symfony\AI\Platform\Bridge\OpenAi\Embeddings;
 use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
 use Symfony\AI\Store\Bridge\Local\InMemoryStore;
-use Symfony\AI\Store\Document\DocumentProcessor;
 use Symfony\AI\Store\Document\Loader\TextFileLoader;
 use Symfony\AI\Store\Document\Transformer\ReplaceTextTransformer;
 use Symfony\AI\Store\Document\Transformer\TextSplitTransformer;
@@ -23,13 +22,14 @@
 
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $store = new InMemoryStore();
-$processor = new DocumentProcessor(
+$indexer = new Indexer(
+    new Vectorizer($platform, new Embeddings('text-embedding-3-small')),
+    $store,
     new TextFileLoader(),
     [
         new ReplaceTextTransformer(search: '## Plot', replace: '## Synopsis'),
         new TextSplitTransformer(chunkSize: 500, overlap: 100),
-    ],
-    new Indexer(new Vectorizer($platform, new Embeddings('text-embedding-3-small')), $store)
+    ]
 );
 
 $movies = [
@@ -38,7 +38,7 @@
     dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
 ];
 
-$processor->process($movies);
+$indexer($movies);
 
 $vector = $platform->invoke(new Embeddings('text-embedding-3-small'), 'Roman gladiator revenge')->asVectors()[0];
 $results = $store->search($vector, 2);

src/store/src/Indexer.php

@@ -13,17 +13,24 @@
 
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
+use Symfony\AI\Store\Document\LoaderInterface;
 use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\TransformerInterface;
 use Symfony\AI\Store\Document\VectorizerInterface;
 
 /**
  * @author Christopher Hertel <mail@christopher-hertel.de>
  */
 final readonly class Indexer implements IndexerInterface
 {
+    /**
+     * @param TransformerInterface[] $transformers
+     */
     public function __construct(
         private VectorizerInterface $vectorizer,
         private StoreInterface $store,
+        private ?LoaderInterface $loader = null,
+        private array $transformers = [],
         private LoggerInterface $logger = new NullLogger(),
     ) {
     }
@@ -56,4 +63,46 @@ public function index(TextDocument|iterable $documents, int $chunkSize = 50): vo
 
         $this->logger->debug(0 === $counter ? 'No documents to index' : \sprintf('Indexed %d documents', $counter));
     }
+
+    /**
+     * Process sources through the complete document pipeline: load → transform → vectorize → store.
+     * 
+     * @param string|array<string> $source  Source identifier (file path, URL, etc.) or array of sources
+     * @param array<string, mixed> $options Processing options
+     */
+    public function __invoke(string|array $source, array $options = []): void
+    {
+        if (null === $this->loader) {
+            throw new \LogicException('Cannot process sources without a loader. Either provide documents directly to index() or configure a loader in the constructor.');
+        }
+
+        $this->logger->debug('Starting document processing', [
+            'source' => $source,
+            'options' => $options,
+        ]);
+
+        $sources = (array) $source;
+        $allDocuments = [];
+
+        // Load documents from all sources
+        foreach ($sources as $singleSource) {
+            $documents = ($this->loader)($singleSource, $options['loader'] ?? []);
+            foreach ($documents as $document) {
+                $allDocuments[] = $document;
+            }
+        }
+
+        // Transform documents through all transformers
+        $transformedDocuments = $allDocuments;
+        foreach ($this->transformers as $transformer) {
+            $transformedDocuments = ($transformer)($transformedDocuments, $options['transformer'] ?? []);
+        }
+
+        // Vectorize and store documents
+        $this->index($transformedDocuments, $options['chunk_size'] ?? 50);
+
+        $this->logger->debug('Document processing completed', [
+            'source' => $source,
+        ]);
+    }
 }