Fix validation

.github/workflows/codeql-analysis.yml

@@ -16,5 +16,5 @@ jobs:
 
       - name: Run CodeQL
         run: |
-          docker run --rm -v $PWD:/app composer sh -c \
+          docker run --rm -v $PWD:/app -w /app phpswoole/swoole:5.1.8-php8.3-alpine sh -c \
           "composer install --profile --ignore-platform-reqs && composer check"

.github/workflows/linter.yml

@@ -16,5 +16,5 @@ jobs:
 
       - name: Run Linter
         run: |
-          docker run --rm -v $PWD:/app composer sh -c \
+          docker run --rm -v $PWD:/app -w /app phpswoole/swoole:5.1.8-php8.3-alpine sh -c \
           "composer install --profile --ignore-platform-reqs && composer lint"

bin/tasks/operators.php

@@ -8,8 +8,11 @@
  *
  * @example
  * docker compose exec tests bin/operators --adapter=mariadb --iterations=1000
- * docker compose exec tests bin/operators --adapter=postgres --iterations=1000
- * docker compose exec tests bin/operators --adapter=sqlite --iterations=1000
+ * docker compose exec tests bin/operators --adapter=postgres --iterations=1000 --seed=10000
+ * docker compose exec tests bin/operators --adapter=sqlite --iterations=1000 --seed=5000
+ *
+ * The --seed parameter allows you to pre-populate the collection with a specified
+ * number of documents to test how operators perform with varying amounts of existing data.
  */
 
 global $cli;
@@ -38,8 +41,9 @@
     ->desc('Benchmark operator performance vs traditional read-modify-write')
     ->param('adapter', '', new Text(0), 'Database adapter (mariadb, postgres, sqlite)')
     ->param('iterations', 1000, new Integer(true), 'Number of iterations per test', true)
+    ->param('seed', 0, new Integer(true), 'Number of documents to pre-seed the collection with', true)
     ->param('name', 'operator_benchmark_' . uniqid(), new Text(0), 'Name of test database', true)
-    ->action(function (string $adapter, int $iterations, string $name) {
+    ->action(function (string $adapter, int $iterations, int $seed, string $name) {
         $namespace = '_ns';
         $cache = new Cache(new NoCache());
 
@@ -48,6 +52,7 @@
         Console::info("=============================================================");
         Console::info("Adapter: {$adapter}");
         Console::info("Iterations: {$iterations}");
+        Console::info("Seed Documents: {$seed}");
         Console::info("Database: {$name}");
         Console::info("=============================================================\n");
 
@@ -110,13 +115,13 @@
                 ->setNamespace($namespace);
 
             // Setup test environment
-            setupTestEnvironment($database, $name);
+            setupTestEnvironment($database, $name, $seed);
 
             // Run all benchmarks
             $results = runAllBenchmarks($database, $iterations);
 
             // Display results
-            displayResults($results, $adapter, $iterations);
+            displayResults($results, $adapter, $iterations, $seed);
 
             // Cleanup
             cleanup($database, $name);
@@ -133,7 +138,7 @@
 /**
  * Setup test environment with collections and sample data
  */
-function setupTestEnvironment(Database $database, string $name): void
+function setupTestEnvironment(Database $database, string $name, int $seed): void
 {
     Console::info("Setting up test environment...");
 
@@ -179,9 +184,69 @@ function setupTestEnvironment(Database $database, string $name): void
     $database->createAttribute('operators_test', 'created_at', Database::VAR_DATETIME, 0, false, null, false, false, null, [], ['datetime']);
     $database->createAttribute('operators_test', 'updated_at', Database::VAR_DATETIME, 0, false, null, false, false, null, [], ['datetime']);
 
+    // Seed documents if requested
+    if ($seed > 0) {
+        seedDocuments($database, $seed);
+    }
+
     Console::success("Test environment setup complete.\n");
 }
 
+/**
+ * Seed the collection with a specified number of documents
+ */
+function seedDocuments(Database $database, int $count): void
+{
+    Console::info("Seeding {$count} documents...");
+
+    $batchSize = 100; // Insert in batches for better performance
+    $batches = (int) ceil($count / $batchSize);
+
+    $seedStart = microtime(true);
+
+    for ($batch = 0; $batch < $batches; $batch++) {
+        $docs = [];
+        $remaining = min($batchSize, $count - ($batch * $batchSize));
+
+        for ($i = 0; $i < $remaining; $i++) {
+            $docNum = ($batch * $batchSize) + $i;
+            $docs[] = new Document([
+                '$id' => 'seed_' . $docNum,
+                '$permissions' => [
+                    Permission::read(Role::any()),
+                    Permission::update(Role::any()),
+                ],
+                'counter' => rand(0, 1000),
+                'score' => round(rand(0, 10000) / 100, 2),
+                'multiplier' => round(rand(50, 200) / 100, 2),
+                'divider' => round(rand(5000, 15000) / 100, 2),
+                'modulo_val' => rand(50, 200),
+                'power_val' => round(rand(100, 300) / 100, 2),
+                'name' => 'seed_doc_' . $docNum,
+                'text' => 'Seed text for document ' . $docNum,
+                'description' => 'This is seed document ' . $docNum . ' with some foo bar baz content',
+                'active' => (bool) rand(0, 1),
+                'tags' => ['seed', 'tag' . ($docNum % 10), 'category' . ($docNum % 5)],
+                'numbers' => [rand(1, 10), rand(11, 20), rand(21, 30)],
+                'items' => ['item' . ($docNum % 3), 'item' . ($docNum % 7)],
+                'created_at' => DateTime::now(),
+                'updated_at' => DateTime::now(),
+            ]);
+        }
+
+        // Bulk insert documents
+        $database->createDocuments('operators_test', $docs);
+
+        // Show progress
+        $progress = (($batch + 1) * $batchSize);
+        $current = min($progress, $count);
+        Console::log("  Seeded {$current}/{$count} documents...");
+    }
+
+    $seedTime = microtime(true) - $seedStart;
+    Console::success("Seeding completed in " . number_format($seedTime, 2) . "s\n");
+}
+
 /**
  * Run all operator benchmarks
  */
@@ -848,13 +913,14 @@ function benchmarkOperatorAcrossOperations(
 /**
  * Display formatted results table
  */
-function displayResults(array $results, string $adapter, int $iterations): void
+function displayResults(array $results, string $adapter, int $iterations, int $seed): void
 {
     Console::info("\n=============================================================");
     Console::info("  BENCHMARK RESULTS");
     Console::info("=============================================================");
     Console::info("Adapter: {$adapter}");
     Console::info("Iterations per test: {$iterations}");
+    Console::info("Seeded documents: {$seed}");
     Console::info("=============================================================\n");
 
     // ==================================================================

src/Database/Connection.php

@@ -21,7 +21,6 @@ class Connection
      */
     public static function hasError(\Throwable $e): bool
     {
-        /** @phpstan-ignore-next-line can't find static method */
         if (DetectsLostConnections::causedByLostConnection($e)) {
             return true;
         }

src/Database/Database.php

@@ -27,7 +27,6 @@
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\Index as IndexValidator;
 use Utopia\Database\Validator\IndexDependency as IndexDependencyValidator;
-use Utopia\Database\Validator\Operator as OperatorValidator;
 use Utopia\Database\Validator\PartialStructure;
 use Utopia\Database\Validator\Permissions;
 use Utopia\Database\Validator\Queries\Document as DocumentValidator;
@@ -4952,20 +4951,7 @@ public function updateDocument(string $collection, string $id, Document $documen
             }
             $createdAt = $document->getCreatedAt();
 
-            // Extract operators from the document before merging
-            $documentArray = $document->getArrayCopy();
-            $extracted = Operator::extractOperators($documentArray);
-            $operators = $extracted['operators'];
-            $updates = $extracted['updates'];
-
-            $operatorValidator = new OperatorValidator($collection, $old);
-            foreach ($operators as $attribute => $operator) {
-                if (!$operatorValidator->isValid($operator)) {
-                    throw new StructureException($operatorValidator->getDescription());
-                }
-            }
-
-            $document = \array_merge($old->getArrayCopy(), $updates);
+            $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy());
             $document['$collection'] = $old->getAttribute('$collection'); // Make sure user doesn't switch collection ID
             $document['$createdAt'] = ($createdAt === null || !$this->preserveDates) ? $old->getCreatedAt() : $createdAt;
 
@@ -4989,8 +4975,11 @@ public function updateDocument(string $collection, string $id, Document $documen
                     $relationships[$relationship->getAttribute('key')] = $relationship;
                 }
 
-                if (!empty($operators)) {
-                    $shouldUpdate = true;
+                foreach ($document as $key => $value) {
+                    if (Operator::isOperator($value)) {
+                        $shouldUpdate = true;
+                        break;
+                    }
                 }
 
                 // Compare if the document has any changes
@@ -5110,7 +5099,8 @@ public function updateDocument(string $collection, string $id, Document $documen
                 $this->adapter->getIdAttributeType(),
                 $this->adapter->getMinDateTime(),
                 $this->adapter->getMaxDateTime(),
-                $this->adapter->getSupportForAttributes()
+                $this->adapter->getSupportForAttributes(),
+                $old
             );
             if (!$structureValidator->isValid($document)) { // Make sure updated structure still apply collection rules (if any)
                 throw new StructureException($structureValidator->getDescription());
@@ -5120,22 +5110,24 @@ public function updateDocument(string $collection, string $id, Document $documen
                 $document = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
             }
 
-
             $document = $this->adapter->castingBefore($collection, $document);
 
-            // Re-add operators to document for adapter processing
-            foreach ($operators as $key => $operator) {
-                $document->setAttribute($key, $operator);
-            }
-
             $this->adapter->updateDocument($collection, $id, $document, $skipPermissionsUpdate);
 
             $document = $this->adapter->castingAfter($collection, $document);
 
             $this->purgeCachedDocument($collection->getId(), $id);
 
             // If operators were used, refetch document to get computed values
-            if (!empty($operators)) {
+            $hasOperators = false;
+            foreach ($document->getArrayCopy() as $value) {
+                if (Operator::isOperator($value)) {
+                    $hasOperators = true;
+                    break;
+                }
+            }
+
+            if ($hasOperators) {
                 $refetched = $this->refetchDocuments($collection, [$document]);
                 $document = $refetched[0];
             }
@@ -5258,24 +5250,17 @@ public function updateDocuments(
             applyDefaults: false
         );
 
-        // Separate operators from regular updates for validation
-        $extracted = Operator::extractOperators($updates->getArrayCopy());
-        $operators = $extracted['operators'];
-        $regularUpdates = $extracted['updates'];
-
-        // Only validate regular updates, not operators
-        if (!empty($regularUpdates)) {
-            $validator = new PartialStructure(
-                $collection,
-                $this->adapter->getIdAttributeType(),
-                $this->adapter->getMinDateTime(),
-                $this->adapter->getMaxDateTime(),
-                $this->adapter->getSupportForAttributes()
-            );
+        $validator = new PartialStructure(
+            $collection,
+            $this->adapter->getIdAttributeType(),
+            $this->adapter->getMinDateTime(),
+            $this->adapter->getMaxDateTime(),
+            $this->adapter->getSupportForAttributes(),
+            null // No old document available in bulk updates
+        );
 
-            if (!$validator->isValid(new Document($regularUpdates))) {
-                throw new StructureException($validator->getDescription());
-            }
+        if (!$validator->isValid($updates)) {
+            throw new StructureException($validator->getDescription());
         }
 
         $originalLimit = $limit;
@@ -5311,17 +5296,8 @@ public function updateDocuments(
             $currentPermissions = $updates->getPermissions();
             sort($currentPermissions);
 
-            $this->withTransaction(function () use ($collection, $updates, &$batch, $currentPermissions, $operators) {
+            $this->withTransaction(function () use ($collection, $updates, &$batch, $currentPermissions) {
                 foreach ($batch as $index => $document) {
-                    if (!empty($operators)) {
-                        $operatorValidator = new OperatorValidator($collection, $document);
-                        foreach ($operators as $attribute => $operator) {
-                            if (!$operatorValidator->isValid($operator)) {
-                                throw new StructureException($operatorValidator->getDescription());
-                            }
-                        }
-                    }
-
                     $skipPermissionsUpdate = true;
 
                     if ($updates->offsetExists('$permissions')) {
@@ -5369,7 +5345,15 @@ public function updateDocuments(
 
             $updates = $this->adapter->castingBefore($collection, $updates);
 
-            if (!empty($operators)) {
+            $hasOperators = false;
+            foreach ($updates->getArrayCopy() as $value) {
+                if (Operator::isOperator($value)) {
+                    $hasOperators = true;
+                    break;
+                }
+            }
+
+            if ($hasOperators) {
                 $batch = $this->refetchDocuments($collection, $batch);
             }
 
@@ -6035,45 +6019,19 @@ public function upsertDocumentsWithIncrease(
                 }
             }
 
-            // Extract operators for validation
-            $documentArray = $document->getArrayCopy();
-            $extracted = Operator::extractOperators($documentArray);
-            $operators = $extracted['operators'];
-            $regularUpdates = $extracted['updates'];
-
-            $operatorValidator = new OperatorValidator($collection, $old->isEmpty() ? null : $old);
-            foreach ($operators as $attribute => $operator) {
-                if (!$operatorValidator->isValid($operator)) {
-                    throw new StructureException($operatorValidator->getDescription());
-                }
-            }
-
-            // Create a temporary document with only regular updates for encoding and validation
-            $tempDocument = new Document($regularUpdates);
-            $tempDocument->setAttribute('$id', $document->getId());
-            $tempDocument->setAttribute('$collection', $document->getAttribute('$collection'));
-            $tempDocument->setAttribute('$createdAt', $document->getAttribute('$createdAt'));
-            $tempDocument->setAttribute('$updatedAt', $document->getAttribute('$updatedAt'));
-            $tempDocument->setAttribute('$permissions', $document->getAttribute('$permissions'));
-            if ($this->adapter->getSharedTables()) {
-                $tempDocument->setAttribute('$tenant', $document->getAttribute('$tenant'));
-            }
-
-            $encodedTemp = $this->encode($collection, $tempDocument);
-
             $validator = new Structure(
                 $collection,
                 $this->adapter->getIdAttributeType(),
                 $this->adapter->getMinDateTime(),
                 $this->adapter->getMaxDateTime(),
-                $this->adapter->getSupportForAttributes()
+                $this->adapter->getSupportForAttributes(),
+                $old->isEmpty() ? null : $old
             );
 
-            if (!$validator->isValid($encodedTemp)) {
+            if (!$validator->isValid($document)) {
                 throw new StructureException($validator->getDescription());
             }
 
-            // Now encode the full document with operators for the adapter
             $document = $this->encode($collection, $document);
 
             if (!$old->isEmpty()) {

src/Database/Validator/Operator.php

@@ -60,8 +60,12 @@ public function getDescription(): string
     public function isValid($value): bool
     {
         if (!$value instanceof DatabaseOperator) {
-            $this->message = 'Value must be an instance of Operator';
-            return false;
+            try {
+                $value = DatabaseOperator::parse($value);
+            } catch (\Throwable $e) {
+                $this->message = 'Invalid operator: ' . $e->getMessage();
+                return false;
+            }
         }
 
         $method = $value->getMethod();