Skip to content

[wip] reduce memory usage for fetching cached mount into #51074

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

[wip] reduce memory usage for fetching cached mount into #51074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3792399
fix: reduce memory usage for fetching cached mount into
icewind1991 Feb 26, 2025
c5d8643
fix: improve getMountsForFileId memory usage and performance
icewind1991 Mar 6, 2025
8380ae1
fix: skip registering mounts if there are no new mount providers
icewind1991 Mar 7, 2025
05ef713
fix: fix oci string length with empty strings
icewind1991 Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,12 @@ public function groupConcat($expr, ?string $separator = ','): IQueryFunction {
public function octetLength($field, $alias = ''): IQueryFunction {
$alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : '';
$quotedName = $this->helper->quoteColumnName($field);
return new QueryFunction('LENGTHB(' . $quotedName . ')' . $alias);
return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias);
}

public function charLength($field, $alias = ''): IQueryFunction {
$alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : '';
$quotedName = $this->helper->quoteColumnName($field);
return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias);
return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias);
}
}
124 changes: 72 additions & 52 deletions lib/private/Files/Config/UserMountCache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
* SPDX-FileCopyrightText: 2016 ownCloud, Inc.
* SPDX-License-Identifier: AGPL-3.0-only
*/

namespace OC\Files\Config;

use OC\User\LazyUser;
use OCP\Cache\CappedMemoryCache;
use OCP\DB\IResult;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Diagnostics\IEventLogger;
use OCP\EventDispatcher\IEventDispatcher;
Expand All @@ -31,11 +33,13 @@ class UserMountCache implements IUserMountCache {

/**
* Cached mount info.
*
* @var CappedMemoryCache<ICachedMountInfo[]>
**/
private CappedMemoryCache $mountsForUsers;
/**
* fileid => internal path mapping for cached mount info.
*
* @var CappedMemoryCache<string>
**/
private CappedMemoryCache $internalPathCache;
Expand Down Expand Up @@ -202,6 +206,19 @@ private function removeFromCache(ICachedMountInfo $mount) {
$query->executeStatement();
}

/**
* @param IResult $result
* @return CachedMountInfo[]
*/
private function fetchMountInfo(IResult $result, ?callable $pathCallback = null): array {
$mounts = [];
while ($row = $result->fetch()) {
$mount = $this->dbRowToMountInfo($row, $pathCallback);
$mounts[] = $mount;
}
return $mounts;
}

/**
* @param array $row
* @param (callable(CachedMountInfo): string)|null $pathCallback
Expand Down Expand Up @@ -251,19 +268,9 @@ public function getMountsForUser(IUser $user) {
->from('mounts', 'm')
->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userUID)));

$result = $query->executeQuery();
$rows = $result->fetchAll();
$result->closeCursor();

/** @var array<string, ICachedMountInfo> $mounts */
$mounts = [];
foreach ($rows as $row) {
$mount = $this->dbRowToMountInfo($row, [$this, 'getInternalPathForMountInfo']);
if ($mount !== null) {
$mounts[$mount->getKey()] = $mount;
}
}
$this->mountsForUsers[$userUID] = $mounts;
$mounts = $this->fetchMountInfo($query->executeQuery(), [$this, 'getInternalPathForMountInfo']);
$keys = array_map(fn (ICachedMountInfo $mount) => $mount->getKey(), $mounts);
$this->mountsForUsers[$userUID] = array_combine($keys, $mounts);
}
return $this->mountsForUsers[$userUID];
}
Expand All @@ -286,8 +293,9 @@ public function getInternalPathForMountInfo(CachedMountInfo $info): string {
* @return CachedMountInfo[]
*/
public function getMountsForStorageId($numericStorageId, $user = null) {
$mounts = [];
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
$query = $builder->select('id', 'storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
->from('mounts', 'm')
->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
Expand All @@ -296,11 +304,7 @@ public function getMountsForStorageId($numericStorageId, $user = null) {
$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
}

$result = $query->executeQuery();
$rows = $result->fetchAll();
$result->closeCursor();

return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows));
return $this->fetchMountInfo($query->executeQuery());
}

/**
Expand All @@ -314,11 +318,7 @@ public function getMountsForRootId($rootFileId) {
->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
->where($builder->expr()->eq('root_id', $builder->createNamedParameter($rootFileId, IQueryBuilder::PARAM_INT)));

$result = $query->executeQuery();
$rows = $result->fetchAll();
$result->closeCursor();

return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows));
return $this->fetchMountInfo($query->executeQuery());
}

/**
Expand All @@ -341,7 +341,7 @@ private function getCacheInfoFromFileId($fileId): array {
$this->cacheInfoCache[$fileId] = [
(int)$row['storage'],
(string)$row['path'],
(int)$row['mimetype']
(int)$row['mimetype'],
];
} else {
throw new NotFoundException('File with id "' . $fileId . '" not found');
Expand All @@ -362,34 +362,54 @@ public function getMountsForFileId($fileId, $user = null) {
} catch (NotFoundException $e) {
return [];
}
$mountsForStorage = $this->getMountsForStorageId($storageId, $user);

// filter mounts that are from the same storage but not a parent of the file we care about
$filteredMounts = array_filter($mountsForStorage, function (ICachedMountInfo $mount) use ($internalPath, $fileId) {
if ($fileId === $mount->getRootId()) {
return true;
}
$internalMountPath = $mount->getRootInternalPath();
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('id', 'storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
->from('mounts', 'm')
->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($builder->expr()->orX(
// filter mounts that are from the same storage but not a parent of the file we care about
$builder->expr()->eq('f.fileid', $builder->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)),
$builder->expr()->eq('f.path', $builder->createNamedParameter('')),
$builder->expr()->isNull('f.path'),
$builder->expr()->eq(
$builder->func()->concat('f.path', $builder->createNamedParameter('/')),
$builder->func()->substring(
$builder->createNamedParameter($internalPath),
$builder->createNamedParameter(1, IQueryBuilder::PARAM_INT),
$builder->func()->add(
$builder->func()->charLength('f.path'),
$builder->createNamedParameter(1, IQueryBuilder::PARAM_INT),
),
),
),
));

return $internalMountPath === '' || str_starts_with($internalPath, $internalMountPath . '/');
});
if ($user) {
$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
}

$filteredMounts = array_values(array_filter($filteredMounts, function (ICachedMountInfo $mount) {
return $this->userManager->userExists($mount->getUser()->getUID());
}));

return array_map(function (ICachedMountInfo $mount) use ($internalPath) {
return new CachedMountFileInfo(
$mount->getUser(),
$mount->getStorageId(),
$mount->getRootId(),
$mount->getMountPoint(),
$mount->getMountId(),
$mount->getMountProvider(),
$mount->getRootInternalPath(),
$internalPath
);
}, $filteredMounts);
$result = $query->executeQuery();
$mounts = [];
while ($row = $result->fetch()) {
$user = $this->userManager->get($row['user_id']);

if ($user) {
$mounts[] = new CachedMountFileInfo(
$user,
(int)$row['storage_id'],
(int)$row['root_id'],
$row['mount_point'],
$row['mount_id'] ? (int)$row['mount_id'] : null,
$row['mount_provider_class'] ?? '',
$row['path'] ?? '',
$internalPath,
);
}
}

return $mounts;
}

/**
Expand Down Expand Up @@ -433,7 +453,7 @@ public function getUsedSpaceForUsers(array $users) {

$mountPoint = $builder->func()->concat(
$builder->func()->concat($slash, 'user_id'),
$slash
$slash,
);

$userIds = array_map(function (IUser $user) {
Expand All @@ -445,7 +465,7 @@ public function getUsedSpaceForUsers(array $users) {
->innerJoin('m', 'filecache', 'f',
$builder->expr()->andX(
$builder->expr()->eq('m.storage_id', 'f.storage'),
$builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files')))
$builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files'))),
))
->where($builder->expr()->eq('m.mount_point', $mountPoint))
->andWhere($builder->expr()->in('m.user_id', $builder->createNamedParameter($userIds, IQueryBuilder::PARAM_STR_ARRAY)));
Expand Down
18 changes: 10 additions & 8 deletions lib/private/Files/SetupManager.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,10 +277,6 @@ private function oneTimeUserSetup(IUser $user) {
private function afterUserFullySetup(IUser $user, array $previouslySetupProviders): void {
$this->eventLogger->start('fs:setup:user:full:post', 'Housekeeping after user is setup');
$userRoot = '/' . $user->getUID() . '/';
$mounts = $this->mountManager->getAll();
$mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot) {
return str_starts_with($mount->getMountPoint(), $userRoot);
});
$allProviders = array_map(function (IMountProvider|IHomeMountProvider|IRootMountProvider $provider) {
return get_class($provider);
}, array_merge(
Expand All @@ -289,10 +285,16 @@ private function afterUserFullySetup(IUser $user, array $previouslySetupProvider
$this->mountProviderCollection->getRootProviders(),
));
$newProviders = array_diff($allProviders, $previouslySetupProviders);
$mounts = array_filter($mounts, function (IMountPoint $mount) use ($previouslySetupProviders) {
return !in_array($mount->getMountProvider(), $previouslySetupProviders);
});
$this->registerMounts($user, $mounts, $newProviders);
if (count($newProviders) > 0) {
$mounts = $this->mountManager->getAll();
$mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot, $previouslySetupProviders) {
if (!str_starts_with($mount->getMountPoint(), $userRoot)) {
return false;
}
return !in_array($mount->getMountProvider(), $previouslySetupProviders);
});
$this->registerMounts($user, $mounts, $newProviders);
}

$cacheDuration = $this->config->getSystemValueInt('fs_mount_cache_duration', 5 * 60);
if ($cacheDuration > 0) {
Expand Down
Loading