feat(dav): introduce paginate with custom headers

Signed-off-by: Benjamin Gaussorgues <benjamin.gaussorgues@nextcloud.com>

apps/dav/appinfo/info.xml

@@ -10,7 +10,7 @@
 	<name>WebDAV</name>
 	<summary>WebDAV endpoint</summary>
 	<description>WebDAV endpoint</description>
-	<version>1.32.0</version>
+	<version>1.33.0</version>
 	<licence>agpl</licence>
 	<author>owncloud.org</author>
 	<namespace>DAV</namespace>
@@ -27,6 +27,7 @@
 		<job>OCA\DAV\BackgroundJob\CleanupDirectLinksJob</job>
 		<job>OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob</job>
 		<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
+		<job>OCA\DAV\BackgroundJob\CleanupPaginateCacheJob</job>
 		<job>OCA\DAV\BackgroundJob\EventReminderJob</job>
 		<job>OCA\DAV\BackgroundJob\CalendarRetentionJob</job>
 		<job>OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob</job>

apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\BackgroundJob;
+
+use OCA\DAV\Paginate\PaginateCache;
+use OCP\BackgroundJob\Job;
+
+class CleanupPaginateCacheJob extends Job {
+	/** @var PaginateCache */
+	private $cache;
+
+	public function __construct(PaginateCache $cache) {
+		$this->cache = $cache;
+	}
+
+	public function run($argument): void {
+		$this->cache->cleanup();
+	}
+}

apps/dav/lib/Migration/Version1032Date20241011093632.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version1032Date20241011093632 extends SimpleMigrationStep {
+	public function name(): string {
+		return 'Add dav_page_cache table';
+	}
+
+	public function description(): string {
+		return 'Add table to cache webdav multistatus responses for pagination purpose';
+	}
+
+	public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('dav_page_cache')) {
+			$table = $schema->createTable('dav_page_cache');
+
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true
+			]);
+			$table->addColumn('url_hash', Types::STRING, [
+				'notnull' => true,
+				'length' => 32,
+			]);
+			$table->addColumn('token', Types::STRING, [
+				'notnull' => true,
+				'length' => 32
+			]);
+			$table->addColumn('result_index', Types::INTEGER, [
+				'notnull' => true
+			]);
+			$table->addColumn('result_value', Types::TEXT, [
+				'notnull' => false,
+			]);
+			$table->addColumn('insert_time', Types::DATETIME, [
+				'notnull' => true,
+			]);
+
+			$table->setPrimaryKey(['id'], 'dav_page_cache_id_index');
+			$table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url');
+			$table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index');
+			$table->addIndex(['result_index'], 'dav_page_cache_index');
+			$table->addIndex(['insert_time'], 'dav_page_cache_time');
+		}
+
+		return $schema;
+	}
+}

apps/dav/lib/Paginate/LimitedCopyIterator.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+/**
+ * Save a copy of the first X items into a separate iterator
+ *
+ * This allows us to pass the iterator to the cache while keeping a copy
+ * of the required items.
+ *
+ * @template-implements AppendIterator<array>
+ */
+class LimitedCopyIterator extends \AppendIterator {
+	private array $skipped = [];
+	private array $copy = [];
+
+	public function __construct(\Traversable $iterator, int $count, int $offset = 0) {
+		parent::__construct();
+
+		if (!$iterator instanceof \Iterator) {
+			$iterator = new \IteratorIterator($iterator);
+		}
+		$iterator = new \NoRewindIterator($iterator);
+
+		$i = 0;
+		while ($iterator->valid() && ++$i <= $offset) {
+			$this->skipped[] = $iterator->current();
+			$iterator->next();
+		}
+
+		while ($iterator->valid() && count($this->copy) < $count) {
+			$this->copy[] = $iterator->current();
+			$iterator->next();
+		}
+
+		$this->append(new \ArrayIterator($this->skipped));
+		$this->append($this->getRequestedItems());
+		$this->append($iterator);
+	}
+
+	public function getRequestedItems(): \Iterator {
+		return new \ArrayIterator($this->copy);
+	}
+}

apps/dav/lib/Paginate/PaginateCache.php

@@ -0,0 +1,98 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use OCP\Security\ISecureRandom;
+
+class PaginateCache {
+	public const TTL = '1 hour';
+
+	public function __construct(
+		private IDBConnection $database,
+		private ISecureRandom $random,
+		private ITimeFactory $timeFactory,
+	) {
+	}
+
+	/**
+	 * @param string $uri
+	 * @param \Iterator $items
+	 * @return array{'token': string, 'count': int}
+	 */
+	public function store(string $uri, \Iterator $items): array {
+		$token = $this->random->generate(32);
+		$now = $this->timeFactory->getDateTime();
+
+		$query = $this->database->getQueryBuilder();
+		$query->insert('dav_page_cache')
+			->values([
+				'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR),
+				'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR),
+				'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_DATETIME_MUTABLE),
+				'result_index' => $query->createParameter('index'),
+				'result_value' => $query->createParameter('value'),
+			]);
+
+		$count = 0;
+		foreach ($items as $item) {
+			$value = serialize($item);
+			$query->setParameter('index', $count, IQueryBuilder::PARAM_INT);
+			$query->setParameter('value', $value);
+			$query->executeStatement();
+			$count++;
+		}
+
+		return ['token' => $token, 'count' => $count];
+	}
+
+	public function get(string $url, string $token, int $offset, int $count): array {
+		$query = $this->database->getQueryBuilder();
+		$query->select(['result_value'])
+			->from('dav_page_cache')
+			->where($query->expr()->eq('token', $query->createNamedParameter($token)))
+			->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url))))
+			->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT)))
+			->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT)));
+
+		$result = $query->executeQuery();
+		return array_map(function (string $entry) {
+			return unserialize($entry);
+		}, $result->fetchAll(\PDO::FETCH_COLUMN));
+	}
+
+	public function exists(string $token): bool {
+		$query = $this->database->getQueryBuilder();
+		return (bool)$query->select('id')
+			->from('dav_page_cache')
+			->where($query->expr()->eq('token', $query->createNamedParameter($token)))
+			->setMaxResults(1)
+			->executeQuery()
+			->fetchOne();
+	}
+
+	public function cleanup(): void {
+		$now = $this->timeFactory->getDateTime();
+		$minDate = $now->sub(\DateInterval::createFromDateString(self::TTL));
+
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache')
+			->where($query->expr()->lt('insert_time', $query->createNamedParameter($minDate, IQueryBuilder::PARAM_DATETIME_MUTABLE)));
+		$query->executeStatement();
+	}
+
+	public function clear(): void {
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache');
+		$query->executeStatement();
+	}
+}

apps/dav/lib/Paginate/PaginatePlugin.php

@@ -0,0 +1,96 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OCA\DAV\Paginate;
+
+use Sabre\DAV\Server;
+use Sabre\DAV\ServerPlugin;
+use Sabre\HTTP\RequestInterface;
+use Sabre\HTTP\ResponseInterface;
+
+class PaginatePlugin extends ServerPlugin {
+	public const PAGINATE_HEADER = 'X-NC-Paginate';
+	public const PAGINATE_TOTAL_HEADER = 'X-NC-Paginate-Total';
+	public const PAGINATE_TOKEN_HEADER = 'X-NC-Paginate-Token';
+	public const PAGINATE_OFFSET_HEADER = 'X-NC-Paginate-Offset';
+	public const PAGINATE_COUNT_HEADER = 'X-NC-Paginate-Count';
+
+	/** @var Server */
+	private $server;
+
+	public function __construct(
+		private PaginateCache $cache,
+		private int $pageSize = 100,
+	) {
+	}
+
+	public function initialize(Server $server): void {
+		$this->server = $server;
+		$server->on('beforeMultiStatus', [$this, 'onMultiStatus']);
+		$server->on('method:SEARCH', [$this, 'onMethod'], 1);
+		$server->on('method:PROPFIND', [$this, 'onMethod'], 1);
+		$server->on('method:REPORT', [$this, 'onMethod'], 1);
+	}
+
+	public function getFeatures(): array {
+		return ['nc-paginate'];
+	}
+
+	public function onMultiStatus(&$fileProperties): void {
+		$request = $this->server->httpRequest;
+		if (is_array($fileProperties)) {
+			$fileProperties = new \ArrayIterator($fileProperties);
+		}
+		if (
+			$request->hasHeader(self::PAGINATE_HEADER) &&
+			(!$request->hasHeader(self::PAGINATE_TOKEN_HEADER) || !$this->cache->exists($request->getHeader(self::PAGINATE_TOKEN_HEADER)))
+		) {
+			$url = $request->getUrl();
+
+			$pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+			$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
+			$copyIterator = new LimitedCopyIterator($fileProperties, $pageSize, $offset);
+			['token' => $token, 'count' => $count] = $this->cache->store($url, $copyIterator);
+
+			$fileProperties = $copyIterator->getRequestedItems();
+			$this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true');
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token);
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, (string)$count);
+			$request->setHeader(self::PAGINATE_TOKEN_HEADER, $token);
+		}
+	}
+
+	public function onMethod(RequestInterface $request, ResponseInterface $response) {
+		if (
+			$request->hasHeader(self::PAGINATE_TOKEN_HEADER) &&
+			$request->hasHeader(self::PAGINATE_OFFSET_HEADER) &&
+			$this->cache->exists($request->getHeader(self::PAGINATE_TOKEN_HEADER))
+		) {
+			$url = $this->server->httpRequest->getUrl();
+			$token = $request->getHeader(self::PAGINATE_TOKEN_HEADER);
+			$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
+			$count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+
+			$items = $this->cache->get($url, $token, $offset, $count);
+
+			$response->setStatus(207);
+			$response->addHeader(self::PAGINATE_HEADER, 'true');
+			$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
+			$response->setHeader('Vary', 'Brief,Prefer');
+
+			$prefer = $this->server->getHTTPPrefer();
+			$minimal = $prefer['return'] === 'minimal';
+
+			$data = $this->server->generateMultiStatus($items, $minimal);
+			$response->setBody($data);
+
+			return false;
+		}
+	}
+}

apps/dav/lib/Server.php

@@ -57,6 +57,7 @@
 use OCA\DAV\Files\BrowserErrorPagePlugin;
 use OCA\DAV\Files\FileSearchBackend;
 use OCA\DAV\Files\LazySearchBackend;
+use OCA\DAV\Paginate\PaginatePlugin;
 use OCA\DAV\Profiler\ProfilerPlugin;
 use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
 use OCA\DAV\SystemTag\SystemTagPlugin;
@@ -228,6 +229,7 @@ public function __construct(
 			$logger,
 			$eventDispatcher,
 		));
+		$this->server->addPlugin(\OCP\Server::get(PaginatePlugin::class));
 
 		// allow setup of additional plugins
 		$eventDispatcher->dispatch('OCA\DAV\Connector\Sabre::addPlugin', $event);