feat(filecache): Scale DB query created when deleting file from filecache

apps/files/lib/AppInfo/Application.php

@@ -36,7 +36,7 @@
 use OCP\AppFramework\Bootstrap\IRegistrationContext;
 use OCP\Collaboration\Reference\RenderReferenceEvent;
 use OCP\Collaboration\Resources\IProviderManager;
-use OCP\Files\Cache\CacheEntryRemovedEvent;
+use OCP\Files\Cache\CacheEntriesRemovedEvent;
 use OCP\Files\Events\Node\BeforeNodeCopiedEvent;
 use OCP\Files\Events\Node\BeforeNodeDeletedEvent;
 use OCP\Files\Events\Node\BeforeNodeRenamedEvent;
@@ -114,7 +114,7 @@ public function register(IRegistrationContext $context): void {
 		$context->registerEventListener(RenderReferenceEvent::class, RenderReferenceEventListener::class);
 		$context->registerEventListener(BeforeNodeRenamedEvent::class, SyncLivePhotosListener::class);
 		$context->registerEventListener(BeforeNodeDeletedEvent::class, SyncLivePhotosListener::class);
-		$context->registerEventListener(CacheEntryRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted.
+		$context->registerEventListener(CacheEntriesRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted.
 		$context->registerEventListener(BeforeNodeCopiedEvent::class, SyncLivePhotosListener::class);
 		$context->registerEventListener(NodeCopiedEvent::class, SyncLivePhotosListener::class);
 		$context->registerEventListener(LoadSearchPlugins::class, LoadSearchPluginsListener::class);

apps/files/lib/Listener/SyncLivePhotosListener.php

@@ -17,7 +17,7 @@
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
 use OCP\Exceptions\AbortedEventException;
-use OCP\Files\Cache\CacheEntryRemovedEvent;
+use OCP\Files\Cache\CacheEntriesRemovedEvent;
 use OCP\Files\Events\Node\BeforeNodeCopiedEvent;
 use OCP\Files\Events\Node\BeforeNodeDeletedEvent;
 use OCP\Files\Events\Node\BeforeNodeRenamedEvent;
@@ -63,8 +63,8 @@ public function handle(Event $event): void {
 				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource()->getId());
 			} elseif ($event instanceof BeforeNodeDeletedEvent) {
 				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getNode()->getId());
-			} elseif ($event instanceof CacheEntryRemovedEvent) {
-				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getFileId());
+			} elseif ($event instanceof CacheEntriesRemovedEvent) {
+				$this->handleCacheEntriesRemovedEvent($event);
 			}
 
 			if ($peerFileId === null) {
@@ -83,12 +83,30 @@ public function handle(Event $event): void {
 				$this->handleMove($event->getSource(), $event->getTarget(), $peerFile);
 			} elseif ($event instanceof BeforeNodeDeletedEvent) {
 				$this->handleDeletion($event, $peerFile);
-			} elseif ($event instanceof CacheEntryRemovedEvent) {
-				$peerFile->delete();
 			}
 		}
 	}
 
+	public function handleCacheEntriesRemovedEvent(CacheEntriesRemovedEvent $cacheEntriesRemovedEvent): void {
+		$entries = $cacheEntriesRemovedEvent->getCacheEntryRemovedEvents();
+		$fileIds = [];
+		foreach ($entries as $entry) {
+			$fileIds[] = $entry->getFileId();
+		}
+
+		$peerFileIds = $this->livePhotosService->getLivePhotoPeerIds($fileIds);
+
+		foreach ($peerFileIds as $peerFileId) {
+			// Check the user's folder.
+			$peerFile = $this->userFolder->getFirstNodeById($peerFileId);
+
+			if ($peerFile === null) {
+				return; // Peer file not found.
+			}
+			$peerFile->delete();
+		}
+	}
+
 	private function runMoveOrCopyChecks(Node $sourceFile, Node $targetFile, Node $peerFile): void {
 		$targetParent = $targetFile->getParent();
 		$sourceExtension = $sourceFile->getExtension();

apps/files/lib/Service/LivePhotosService.php

@@ -33,4 +33,22 @@ public function getLivePhotoPeerId(int $fileId): ?int {
 
 		return (int)$metadata->getString('files-live-photo');
 	}
+
+	/**
+	 * Get the associated live photo for multiple file ids
+	 * @param int[] $fileIds
+	 * @return int[]
+	 */
+	public function getLivePhotoPeerIds(array $fileIds): array {
+		$metadata = $this->filesMetadataManager->getMetadataForFiles($fileIds);
+		$peersIds = [];
+		foreach ($metadata as $item) {
+			if (!$item->hasKey('files-live-photo')) {
+				continue;
+			}
+
+			$peersIds[] = (int)$item->getString('files-live-photo');
+		}
+		return $peersIds;
+	}
 }

lib/composer/composer/autoload_classmap.php

@@ -396,6 +396,7 @@
     'OCP\\Files\\AlreadyExistsException' => $baseDir . '/lib/public/Files/AlreadyExistsException.php',
     'OCP\\Files\\AppData\\IAppDataFactory' => $baseDir . '/lib/public/Files/AppData/IAppDataFactory.php',
     'OCP\\Files\\Cache\\AbstractCacheEvent' => $baseDir . '/lib/public/Files/Cache/AbstractCacheEvent.php',
+    'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php',
     'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php',
     'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php',
     'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php',

lib/composer/composer/autoload_static.php

@@ -437,6 +437,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OCP\\Files\\AlreadyExistsException' => __DIR__ . '/../../..' . '/lib/public/Files/AlreadyExistsException.php',
         'OCP\\Files\\AppData\\IAppDataFactory' => __DIR__ . '/../../..' . '/lib/public/Files/AppData/IAppDataFactory.php',
         'OCP\\Files\\Cache\\AbstractCacheEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/AbstractCacheEvent.php',
+        'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php',
         'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php',
         'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php',
         'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php',

lib/private/Files/Cache/Cache.php

@@ -18,6 +18,7 @@
 use OCP\DB\Exception;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\Cache\CacheEntriesRemovedEvent;
 use OCP\Files\Cache\CacheEntryInsertedEvent;
 use OCP\Files\Cache\CacheEntryRemovedEvent;
 use OCP\Files\Cache\CacheEntryUpdatedEvent;
@@ -620,13 +621,17 @@ private function removeChildren(ICacheEntry $entry) {
 			$query->executeStatement();
 		}
 
+		$cacheEntryRemovedEvents = [];
 		foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
-			$cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
+			$cacheEntryRemovedEvents[] = new CacheEntryRemovedEvent(
 				$this->storage,
 				$filePath,
 				$fileId,
 				$this->getNumericStorageId()
 			);
+		}
+		$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents));
+		foreach ($cacheEntryRemovedEvents as $cacheEntryRemovedEvent) {
 			$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
 		}
 	}
@@ -784,7 +789,10 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
 			$this->connection->commit();
 
 			if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
-				$this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
+				$event = new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId());
+				$this->eventDispatcher->dispatchTyped($event);
+				$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent([$event]));
+
 				$event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
 				$this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
 				$this->eventDispatcher->dispatchTyped($event);

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -20,7 +20,7 @@
 use OCP\DB\Exception as DBException;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\EventDispatcher\IEventDispatcher;
-use OCP\Files\Cache\CacheEntryRemovedEvent;
+use OCP\Files\Cache\CacheEntriesRemovedEvent;
 use OCP\Files\Events\Node\NodeWrittenEvent;
 use OCP\Files\InvalidPathException;
 use OCP\Files\Node;
@@ -214,6 +214,20 @@ public function deleteMetadata(int $fileId): void {
 		}
 	}
 
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void {
+		try {
+			$this->metadataRequestService->dropMetadataForFiles($storage, $fileIds);
+		} catch (Exception $e) {
+			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
+		}
+
+		try {
+			$this->indexRequestService->dropIndexForFiles($fileIds);
+		} catch (Exception $e) {
+			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
+		}
+	}
+
 	/**
 	 * @param IQueryBuilder $qb
 	 * @param string $fileTableAlias alias of the table that contains data about files
@@ -301,6 +315,6 @@ public function initMetadata(
 	 */
 	public static function loadListeners(IEventDispatcher $eventDispatcher): void {
 		$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
-		$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
+		$eventDispatcher->addServiceListener(CacheEntriesRemovedEvent::class, MetadataDelete::class);
 	}
 }

lib/private/FilesMetadata/Listener/MetadataDelete.php

@@ -11,14 +11,14 @@
 use Exception;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
-use OCP\Files\Cache\CacheEntryRemovedEvent;
+use OCP\Files\Cache\CacheEntriesRemovedEvent;
 use OCP\FilesMetadata\IFilesMetadataManager;
 use Psr\Log\LoggerInterface;
 
 /**
  * Handle file deletion event and remove stored metadata related to the deleted file
  *
- * @template-implements IEventListener<CacheEntryRemovedEvent>
+ * @template-implements IEventListener<CacheEntriesRemovedEvent>
  */
 class MetadataDelete implements IEventListener {
 	public function __construct(
@@ -28,14 +28,25 @@ public function __construct(
 	}
 
 	public function handle(Event $event): void {
-		if (!($event instanceof CacheEntryRemovedEvent)) {
+		if (!($event instanceof CacheEntriesRemovedEvent)) {
 			return;
 		}
 
+		$entries = $event->getCacheEntryRemovedEvents();
+		$storageToFileIds = [];
+
+		foreach ($entries as $entry) {
+			try {
+				$storageToFileIds[$entry->getStorageId()] ??= [];
+				$storageToFileIds[$entry->getStorageId()][] = $entry->getFileId();
+			} catch (Exception $e) {
+				$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
+			}
+		}
+
 		try {
-			$nodeId = $event->getFileId();
-			if ($nodeId > 0) {
-				$this->filesMetadataManager->deleteMetadata($nodeId);
+			foreach ($storageToFileIds as $storageId => $fileIds) {
+				$this->filesMetadataManager->deleteMetadataForFiles($storageId, $fileIds);
 			}
 		} catch (Exception $e) {
 			$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);

lib/private/FilesMetadata/Service/IndexRequestService.php

@@ -175,4 +175,30 @@ public function dropIndex(int $fileId, string $key = ''): void {
 
 		$qb->executeStatement();
 	}
+
+	/**
+	 * Drop indexes related to multiple file ids
+	 * if a key is specified, only drop entries related to it
+	 *
+	 * @param int[] $fileIds file ids
+	 * @param string $key metadata key
+	 *
+	 * @throws DbException
+	 */
+	public function dropIndexForFiles(array $fileIds, string $key = ''): void {
+		$chunks = array_chunk($fileIds, 1000);
+
+		foreach ($chunks as $chunk) {
+			$qb = $this->dbConnection->getQueryBuilder();
+			$expr = $qb->expr();
+			$qb->delete(self::TABLE_METADATA_INDEX)
+				->where($expr->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
+
+			if ($key !== '') {
+				$qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key)));
+			}
+
+			$qb->executeStatement();
+		}
+	}
 }

lib/private/FilesMetadata/Service/MetadataRequestService.php

@@ -108,8 +108,10 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata {
 	 */
 	public function getMetadataFromFileIds(array $fileIds): array {
 		$qb = $this->dbConnection->getQueryBuilder();
-		$qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA);
-		$qb->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
+		$qb->select('file_id', 'json', 'sync_token')
+			->from(self::TABLE_METADATA)
+			->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+			->runAcrossAllShards();
 
 		$list = [];
 		$result = $qb->executeQuery();
@@ -143,6 +145,22 @@ public function dropMetadata(int $fileId): void {
 		$qb->executeStatement();
 	}
 
+	/**
+	 * @param int[] $fileIds
+	 * @throws Exception
+	 */
+	public function dropMetadataForFiles(int $storage, array $fileIds): void {
+		$chunks = array_chunk($fileIds, 1000);
+
+		foreach ($chunks as $chunk) {
+			$qb = $this->dbConnection->getQueryBuilder();
+			$qb->delete(self::TABLE_METADATA)
+				->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+				->hintShardKey('storage', $storage);
+			$qb->executeStatement();
+		}
+	}
+
 	/**
 	 * update metadata in the database
 	 *

lib/public/Files/Cache/CacheEntriesRemovedEvent.php

@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OCP\Files\Cache;
+
+use OCP\EventDispatcher\Event;
+
+/**
+ * Meta-event wrapping multiple CacheEntryRemovedEvent for when an existing
+ * entry in the cache gets removed.
+ *
+ * @since 32.0.0
+ */
+#[\OCP\AppFramework\Attribute\Listenable(since: '32.0.0')]
+class CacheEntriesRemovedEvent extends Event {
+	/**
+	 * @param CacheEntryRemovedEvent[] $cacheEntryRemovedEvents
+	 */
+	public function __construct(
+		private readonly array $cacheEntryRemovedEvents,
+	) {
+	}
+
+	/**
+	 * @return CacheEntryRemovedEvent[]
+	 */
+	public function getCacheEntryRemovedEvents(): array {
+		return $this->cacheEntryRemovedEvents;
+	}
+}

lib/public/Files/Cache/CacheEntryRemovedEvent.php

@@ -11,7 +11,11 @@
 /**
  * Event for when an existing entry in the cache gets removed
  *
+ * Prefer using CacheEntriesRemovedEvent as it is more efficient when deleting
+ * multiple files at the same time.
+ *
  * @since 21.0.0
+ * @see CacheEntriesRemovedEvent
  */
 class CacheEntryRemovedEvent extends AbstractCacheEvent implements ICacheEvent {
 }

lib/public/FilesMetadata/IFilesMetadataManager.php

@@ -8,6 +8,7 @@
 
 namespace OCP\FilesMetadata;
 
+use OCP\AppFramework\Attribute\Consumable;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Node;
 use OCP\FilesMetadata\Exceptions\FilesMetadataException;
@@ -20,6 +21,7 @@
  *
  * @since 28.0.0
  */
+#[Consumable(since: '28.0.0')]
 interface IFilesMetadataManager {
 	/** @since 28.0.0 */
 	public const PROCESS_LIVE = 1;
@@ -98,6 +100,16 @@ public function saveMetadata(IFilesMetadata $filesMetadata): void;
 	 */
 	public function deleteMetadata(int $fileId): void;
 
+	/**
+	 * Delete metadata and its indexes of multiple file ids
+	 *
+	 * @param int $storage The storage id coresponding to the $fileIds
+	 * @param array<int> $fileIds file ids
+	 * @return void
+	 * @since 32.0.0
+	 */
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void;
+
 	/**
 	 * generate and return a MetadataQuery to help building sql queries
 	 *