Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
matrix:
php-versions: [ '8.1' ]
databases: [ 'pgsql' ]
server-versions: [ 'master' ]
server-versions: [ 'stable30', 'stable31', 'master' ]

name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}

Expand Down Expand Up @@ -280,7 +280,7 @@ jobs:
matrix:
php-versions: [ '8.1' ]
databases: [ 'pgsql' ]
server-versions: [ 'master' ]
server-versions: [ 'stable30', 'stable31', 'master' ]

name: Integration test with cron on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}

Expand Down Expand Up @@ -452,7 +452,7 @@ jobs:
./occ config:app:set --value 30 --type integer context_chat indexing_job_interval # 30 seconds
./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
for i in {1..100}; do
php cron.php -v &
php cron.php & # Starting with stable31 we can use -v here for better visibility
wait
./occ context_chat:stats
done
Expand Down Expand Up @@ -517,7 +517,7 @@ jobs:
matrix:
php-versions: [ '8.1' ]
databases: [ 'pgsql' ]
server-versions: [ 'master' ]
server-versions: [ 'stable30', 'stable31', 'master' ]

name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}

Expand Down Expand Up @@ -655,7 +655,7 @@ jobs:
./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
# Run normal indexing jobs which will only pick up welcome.txt etc
for i in {1..10}; do
php cron.php -v &
php cron.php & # Starting with stable31 we can use -v here for better visibility
wait
./occ context_chat:stats
done
Expand Down Expand Up @@ -696,7 +696,7 @@ jobs:
- name: Run indexer cron
run: |
for i in {1..100}; do
php cron.php -v &
php cron.php & # Starting with stable31 we can use -v here for better visibility
wait
./occ context_chat:stats
done
Expand Down
50 changes: 39 additions & 11 deletions .github/workflows/psalm.yml → .github/workflows/psalm-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,42 @@ permissions:
contents: read

jobs:
static-analysis:
runs-on: ubuntu-latest

name: static-psalm-analysis
matrix:
runs-on: ubuntu-latest-low
outputs:
ocp-matrix: ${{ steps.versions.outputs.ocp-matrix }}
steps:
- name: Checkout
- name: Checkout app
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false

- name: Get php version
- name: Get version matrix
id: versions
uses: icewind1991/nextcloud-version-matrix@58becf3b4bb6dc6cef677b15e2fd8e7d48c0908f # v1.3.1

- name: Check enforcement of minimum PHP version ${{ steps.versions.outputs.php-min }} in psalm.xml
run: grep 'phpVersion="${{ steps.versions.outputs.php-min }}' psalm.xml

- name: Set up php${{ steps.versions.outputs.php-available }}
uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0
static-analysis:
runs-on: ubuntu-latest
needs: matrix
strategy:
# do not stop on another job's failure
fail-fast: false
matrix: ${{ fromJson(needs.matrix.outputs.ocp-matrix) }}

name: static-psalm-analysis ${{ matrix.ocp-version }}
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
php-version: ${{ steps.versions.outputs.php-available }}
persist-credentials: false

- name: Set up php${{ matrix.php-min }}
uses: shivammathur/setup-php@0f7f1d08e3e32076e51cae65eb0b0c871405b16e # v2.34.1
with:
php-version: ${{ matrix.php-min }}
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite
coverage: none
ini-file: development
Expand All @@ -52,8 +67,21 @@ jobs:
composer remove nextcloud/ocp --dev --no-scripts
composer i

- name: Install nextcloud/ocp
run: composer require --dev nextcloud/ocp:dev-${{ steps.versions.outputs.branches-max }} --ignore-platform-reqs --with-dependencies

- name: Install dependencies # zizmor: ignore[template-injection]
run: composer require --dev 'nextcloud/ocp:${{ matrix.ocp-version }}' --ignore-platform-reqs --with-dependencies

- name: Run coding standards check
run: composer run psalm -- --threads=1 --monochrome --no-progress --output-format=github

summary:
runs-on: ubuntu-latest-low
needs: static-analysis

if: always()

name: static-psalm-analysis-summary

steps:
- name: Summary status
run: if ${{ needs.static-analysis.result != 'success' }}; then exit 1; fi
4 changes: 2 additions & 2 deletions lib/BackgroundJobs/SchedulerJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ protected function run($argument): void {
$this->appConfig->setAppValueString('indexed_files_count', (string)0);
$this->appConfig->setAppValueInt('last_indexed_time', 0);
foreach ($this->storageService->getMounts() as $mount) {
$this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['override_root']);
$this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['overridden_root']);
$this->jobList->add(StorageCrawlJob::class, [
'storage_id' => $mount['storage_id'],
'root_id' => $mount['root_id' ],
'override_root' => $mount['override_root'],
'overridden_root' => $mount['overridden_root'],
'last_file_id' => 0,
]);
}
Expand Down
6 changes: 3 additions & 3 deletions lib/BackgroundJobs/StorageCrawlJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@ public function __construct(
}

/**
* @param array{storage_id:int, root_id:int, override_root:int, last_file_id:int} $argument
* @param array{storage_id:int, root_id:int, overridden_root:int|null, override_root:int|null, last_file_id:int} $argument
* @return void
*/
protected function run($argument): void {
$storageId = $argument['storage_id'];
$rootId = $argument['root_id'];
$overrideRoot = $argument['override_root'];
$overrideRoot = $argument['overridden_root'] ?? $argument['override_root'] ?? $rootId;
$lastFileId = $argument['last_file_id'];

// Remove current iteration
Expand All @@ -55,7 +55,7 @@ protected function run($argument): void {
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());

$i = 0;
foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot, $lastFileId, self::BATCH_SIZE) as $fileId) {
foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
$queueFile = new QueueFile();
$queueFile->setStorageId($storageId);
$queueFile->setRootId($rootId);
Expand Down
2 changes: 1 addition & 1 deletion lib/Listener/UserDeletedListener.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@ public function handle(Event $event): void {
return;
}

$this->actionService->deleteUser($event->getUid());
$this->actionService->deleteUser($event->getUser()->getUID());
}
}
54 changes: 51 additions & 3 deletions lib/Service/StorageService.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
use OCA\ContextChat\Logger;
use OCP\DB\Exception;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\IFileAccess;
use OCP\Files\Config\ICachedMountInfo;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\File;
Expand Down Expand Up @@ -45,6 +46,7 @@ public function __construct(
private IUserMountCache $userMountCache,
private IFilesMetadataManager $metadataManager,
private IRootFolder $rootFolder,
private IFileAccess $fileAccess,
) {
}

Expand Down Expand Up @@ -125,11 +127,28 @@ public function countFilesInMount(int $storageId, int $rootId): int {
return $countInMount;
}

private function isFileAccessAvailable(): bool {
$reflection = new \ReflectionClass($this->fileAccess);
return $reflection->hasMethod('getByAncestorInStorage');
}

/**
* @return \Generator<array{root_id: int, override_root: int, storage_id: int}>
* @return \Generator<array{root_id: int, overridden_root: int, storage_id: int}>
* @throws \OCP\DB\Exception
*/
public function getMounts(): \Generator {
if (!$this->isFileAccessAvailable()) {
return $this->getMountsOld();
}

return $this->fileAccess->getDistinctMounts(self::ALLOWED_MOUNT_TYPES, true);
}

/**
* @return \Generator<array{root_id: int, overridden_root: int, storage_id: int}>
* @throws \OCP\DB\Exception
*/
private function getMountsOld(): \Generator {
$qb = $this->db->getQueryBuilder();
$qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) // to avoid scanning each occurrence of a groupfolder
->from('mounts')
Expand All @@ -154,7 +173,7 @@ public function getMounts(): \Generator {
/** @var array|false $root */
$root = $qb
->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('filecache.name', $qb->createNamedParameter('files')))
->andWhere($qb->expr()->eq('filecache.path', $qb->createNamedParameter('files')))
->andWhere($qb->expr()->eq('filecache.parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)))
->executeQuery()->fetch();
if ($root !== false) {
Expand All @@ -168,7 +187,7 @@ public function getMounts(): \Generator {
yield [
'storage_id' => $storageId,
'root_id' => $rootId,
'override_root' => $overrideRoot,
'overridden_root' => $overrideRoot,
];
}
$result->closeCursor();
Expand All @@ -182,6 +201,35 @@ public function getMounts(): \Generator {
* @return \Generator<int,int,mixed,void>
*/
public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
if (!$this->isFileAccessAvailable()) {
return $this->getFilesInMountOld($storageId, $rootId, $lastFileId, $maxResults);
}

return $this->getFilesInMountUsingFileAccess($storageId, $rootId, $lastFileId, $maxResults);
}

/**
* @param int $storageId
* @param int $rootId
* @param int $lastFileId
* @param int $maxResults
* @return \Generator<int,int,mixed,void>
*/
private function getFilesInMountUsingFileAccess(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
$mimeTypeIds = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
foreach ($this->fileAccess->getByAncestorInStorage($storageId, $rootId, $lastFileId, $maxResults, $mimeTypeIds, false, true) as $cacheEntry) {
yield $cacheEntry['fileid'];
}
}

/**
* @param int $storageId
* @param int $rootId
* @param int $lastFileId
* @param int $maxResults
* @return \Generator<int,int,mixed,void>
*/
private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
$qb = $this->getCacheQueryBuilder();
try {
$qb->selectFileCache();
Expand Down
6 changes: 6 additions & 0 deletions tests/psalm-baseline.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,10 @@
<code><![CDATA[\OC]]></code>
</UndefinedClass>
</file>
<file src="lib/Service/StorageService.php">
<UndefinedInterfaceMethod>
<code><![CDATA[getByAncestorInStorage]]></code>
<code><![CDATA[getDistinctMounts]]></code>
</UndefinedInterfaceMethod>
</file>
</files>
Loading