Skip to content

Commit bfc66eb

Browse files
abnegateabnegate
authored
Merge pull request #618 from utopia-php/bulk-sequences
Refactor getSequences
2 parents 83278d6 + bf7404b commit bfc66eb

File tree

6 files changed

+62
-63
lines changed

6 files changed

+62
-63
lines changed

‎src/Database/Adapter.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -733,6 +733,13 @@ abstract public function createOrUpdateDocuments(
733733
array $changes
734734
): array;
735735

736+
/**
737+
* @param string $collection
738+
* @param array<Document> $documents
739+
* @return array<Document>
740+
*/
741+
abstract public function getSequences(string $collection, array $documents): array;
742+
736743
/**
737744
* Delete Document
738745
*

‎src/Database/Adapter/MariaDB.php‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1171,8 +1171,6 @@ public function createOrUpdateDocuments(
11711171
$bindIndex = 0;
11721172
$batchKeys = [];
11731173
$bindValues = [];
1174-
$documentIds = [];
1175-
$documentTenants = [];
11761174

11771175
foreach ($changes as $change) {
11781176
$document = $change->getNew();
@@ -1184,14 +1182,10 @@ public function createOrUpdateDocuments(
11841182

11851183
if (!empty($document->getSequence())) {
11861184
$attributes['_id'] = $document->getSequence();
1187-
} else {
1188-
$documentIds[] = $document->getId();
11891185
}
11901186

11911187
if ($this->sharedTables) {
1192-
$attributes['_tenant']
1193-
= $documentTenants[]
1194-
= $document->getTenant();
1188+
$attributes['_tenant'] = $document->getTenant();
11951189
}
11961190

11971191
\ksort($attributes);
@@ -1349,18 +1343,6 @@ public function createOrUpdateDocuments(
13491343
}
13501344
$stmtAddPermissions->execute();
13511345
}
1352-
1353-
$sequences = $this->getSequences(
1354-
$collection,
1355-
$documentIds,
1356-
$documentTenants
1357-
);
1358-
1359-
foreach ($changes as $change) {
1360-
if (isset($sequences[$change->getNew()->getId()])) {
1361-
$change->getNew()->setAttribute('$sequence', $sequences[$change->getNew()->getId()]);
1362-
}
1363-
}
13641346
} catch (PDOException $e) {
13651347
throw $this->processException($e);
13661348
}

‎src/Database/Adapter/Pool.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,4 +494,9 @@ protected function execute(mixed $stmt): bool
494494
{
495495
return $this->delegate(__FUNCTION__, \func_get_args());
496496
}
497+
498+
public function getSequences(string $collection, array $documents): array
499+
{
500+
return $this->delegate(__FUNCTION__, \func_get_args());
501+
}
497502
}

‎src/Database/Adapter/SQL.php‎

Lines changed: 43 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -710,49 +710,64 @@ public function deleteDocuments(string $collection, array $sequences, array $per
710710
}
711711

712712
/**
713-
* Get internal IDs for the given documents
713+
* Assign internal IDs for the given documents
714714
*
715715
* @param string $collection
716-
* @param array<string> $documentIds
717-
* @param array<?int> $documentTenants
718-
* @return array<string>
716+
* @param array<Document> $documents
717+
* @return array<Document>
719718
* @throws DatabaseException
720719
*/
721-
protected function getSequences(string $collection, array $documentIds, array $documentTenants = []): array
720+
public function getSequences(string $collection, array $documents): array
722721
{
723-
$sequences = [];
722+
$documentIds = [];
723+
$keys = [];
724+
$binds = [];
724725

725-
/**
726-
* UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query
727-
*/
728-
foreach (\array_chunk($documentIds, 1000) as $documentIdsChunk) {
729-
$sql = "
730-
SELECT _uid, _id
731-
FROM {$this->getSQLTable($collection)}
732-
WHERE {$this->quote('_uid')} IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ")
733-
{$this->getTenantQuery($collection, tenantCount: \count($documentIdsChunk))}
734-
";
726+
foreach ($documents as $i => $document) {
727+
if (empty($document->getSequence())) {
728+
$documentIds[] = $document->getId();
735729

736-
$stmt = $this->getPDO()->prepare($sql);
730+
$key = ":uid_{$i}";
737731

738-
foreach ($documentIdsChunk as $index => $id) {
739-
$stmt->bindValue(":_key_{$index}", $id);
740-
}
732+
$binds[$key] = $document->getId();
733+
$keys[] = $key;
741734

742-
if ($this->sharedTables) {
743-
foreach ($documentIdsChunk as $index => $id) {
744-
$stmt->bindValue(":_tenant_{$index}", \array_shift($documentTenants));
735+
if ($this->sharedTables) {
736+
$binds[':_tenant_'.$i] = $document->getTenant();
745737
}
746738
}
739+
}
747740

748-
$stmt->execute();
749-
$results = $stmt->fetchAll(\PDO::FETCH_KEY_PAIR); // Fetch as [documentId => sequence]
750-
$stmt->closeCursor();
741+
if (empty($documentIds)) {
742+
return $documents;
743+
}
751744

752-
$sequences = [...$sequences, ...$results];
745+
$placeholders = implode(',', array_values($keys));
746+
747+
$sql = "
748+
SELECT _uid, _id
749+
FROM {$this->getSQLTable($collection)}
750+
WHERE {$this->quote('_uid')} IN ({$placeholders})
751+
{$this->getTenantQuery($collection, tenantCount: \count($documentIds))}
752+
";
753+
754+
$stmt = $this->getPDO()->prepare($sql);
755+
756+
foreach ($binds as $key => $value) {
757+
$stmt->bindValue($key, $value);
753758
}
754759

755-
return $sequences;
760+
$stmt->execute();
761+
$sequences = $stmt->fetchAll(\PDO::FETCH_KEY_PAIR); // Fetch as [documentId => sequence]
762+
$stmt->closeCursor();
763+
764+
foreach ($documents as $document) {
765+
if (isset($sequences[$document->getId()])) {
766+
$document['$sequence'] = $sequences[$document->getId()];
767+
}
768+
}
769+
770+
return $documents;
756771
}
757772

758773
/**
@@ -1808,8 +1823,6 @@ public function createDocuments(string $collection, array $documents): array
18081823
$batchKeys = [];
18091824
$bindValues = [];
18101825
$permissions = [];
1811-
$documentIds = [];
1812-
$documentTenants = [];
18131826

18141827
foreach ($documents as $index => $document) {
18151828
$attributes = $document->getAttributes();
@@ -1820,13 +1833,10 @@ public function createDocuments(string $collection, array $documents): array
18201833

18211834
if (!empty($document->getSequence())) {
18221835
$attributes['_id'] = $document->getSequence();
1823-
} else {
1824-
$documentIds[] = $document->getId();
18251836
}
18261837

18271838
if ($this->sharedTables) {
18281839
$attributes['_tenant'] = $document->getTenant();
1829-
$documentTenants[] = $document->getTenant();
18301840
}
18311841

18321842
$bindKeys = [];
@@ -1889,17 +1899,6 @@ public function createDocuments(string $collection, array $documents): array
18891899
$this->execute($stmtPermissions);
18901900
}
18911901

1892-
$sequences = $this->getSequences(
1893-
$collection,
1894-
$documentIds,
1895-
$documentTenants
1896-
);
1897-
1898-
foreach ($documents as $document) {
1899-
if (isset($sequences[$document->getId()])) {
1900-
$document['$sequence'] = $sequences[$document->getId()];
1901-
}
1902-
}
19031902
} catch (PDOException $e) {
19041903
throw $this->processException($e);
19051904
}

‎src/Database/Database.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3759,6 +3759,8 @@ public function createDocuments(
37593759
return $this->adapter->createDocuments($collection->getId(), $chunk);
37603760
});
37613761

3762+
$batch = $this->adapter->getSequences($collection->getId(), $batch);
3763+
37623764
foreach ($batch as $document) {
37633765
if ($this->resolveRelationships) {
37643766
$document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
@@ -5074,6 +5076,8 @@ public function createOrUpdateDocumentsWithIncrease(
50745076
$chunk
50755077
)));
50765078

5079+
$batch = $this->adapter->getSequences($collection->getId(), $batch);
5080+
50775081
foreach ($chunk as $change) {
50785082
if ($change->getOld()->isEmpty()) {
50795083
$created++;

‎tests/e2e/Adapter/Scopes/DocumentTests.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ public function testUpsertDocuments(): void
445445
foreach ($results as $index => $document) {
446446
$createdAt[$index] = $document->getCreatedAt();
447447
$this->assertNotEmpty(true, $document->getId());
448+
$this->assertNotEmpty(true, $document->getSequence());
448449
$this->assertIsString($document->getAttribute('string'));
449450
$this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working
450451
$this->assertIsInt($document->getAttribute('integer'));
@@ -481,6 +482,7 @@ public function testUpsertDocuments(): void
481482

482483
foreach ($results as $document) {
483484
$this->assertNotEmpty(true, $document->getId());
485+
$this->assertNotEmpty(true, $document->getSequence());
484486
$this->assertIsString($document->getAttribute('string'));
485487
$this->assertEquals('new text📝', $document->getAttribute('string')); // Also makes sure an emoji is working
486488
$this->assertIsInt($document->getAttribute('integer'));

0 commit comments

Comments
 (0)