From d7d8822a9a17dd9a3caf0a68806fcdd85f9e68fb Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 13 Jan 2021 18:41:30 +0100 Subject: [PATCH] feat: add command to scan external storages directly the main use case of this over simply scanning through is the ability to provide a username and/or password for cases where login credentials are used Signed-off-by: Robin Appelman --- apps/files_external/appinfo/info.xml | 1 + .../composer/composer/autoload_classmap.php | 2 + .../composer/composer/autoload_static.php | 2 + apps/files_external/lib/Command/Notify.php | 80 +-------- apps/files_external/lib/Command/Scan.php | 155 ++++++++++++++++++ .../lib/Command/StorageAuthBase.php | 130 +++++++++++++++ 6 files changed, 296 insertions(+), 74 deletions(-) create mode 100644 apps/files_external/lib/Command/Scan.php create mode 100644 apps/files_external/lib/Command/StorageAuthBase.php diff --git a/apps/files_external/appinfo/info.xml b/apps/files_external/appinfo/info.xml index 01899ab6411c4..246c9d9d83359 100644 --- a/apps/files_external/appinfo/info.xml +++ b/apps/files_external/appinfo/info.xml @@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se OCA\Files_External\Command\Backends OCA\Files_External\Command\Verify OCA\Files_External\Command\Notify + OCA\Files_External\Command\Scan diff --git a/apps/files_external/composer/composer/autoload_classmap.php b/apps/files_external/composer/composer/autoload_classmap.php index b10fc32e10059..0b168b1170e43 100644 --- a/apps/files_external/composer/composer/autoload_classmap.php +++ b/apps/files_external/composer/composer/autoload_classmap.php @@ -19,6 +19,8 @@ 'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php', 'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php', 'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php', + 'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php', + 'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php', 'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php', 'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php', 'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php', diff --git a/apps/files_external/composer/composer/autoload_static.php b/apps/files_external/composer/composer/autoload_static.php index c5406fe3cf861..29e95ae968a59 100644 --- a/apps/files_external/composer/composer/autoload_static.php +++ b/apps/files_external/composer/composer/autoload_static.php @@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External 'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php', 'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php', 'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php', + 'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php', + 'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php', 'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php', 'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php', 'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php', diff --git a/apps/files_external/lib/Command/Notify.php b/apps/files_external/lib/Command/Notify.php index 2fdd2f3a2ee00..81188960b50a4 100644 --- a/apps/files_external/lib/Command/Notify.php +++ b/apps/files_external/lib/Command/Notify.php @@ -30,9 +30,6 @@ namespace OCA\Files_External\Command; use Doctrine\DBAL\Exception\DriverException; -use OC\Core\Command\Base; -use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException; -use OCA\Files_External\Lib\StorageConfig; use OCA\Files_External\Service\GlobalStoragesService; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Notify\IChange; @@ -40,7 +37,6 @@ use OCP\Files\Notify\IRenameChange; use OCP\Files\Storage\INotifyStorage; use OCP\Files\Storage\IStorage; -use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; use OCP\IUserManager; use Psr\Log\LoggerInterface; @@ -49,14 +45,14 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -class Notify extends Base { +class Notify extends StorageAuthBase { public function __construct( - private GlobalStoragesService $globalService, private IDBConnection $connection, private LoggerInterface $logger, - private IUserManager $userManager + GlobalStoragesService $globalService, + IUserManager $userManager, ) { - parent::__construct(); + parent::__construct($globalService, $userManager); } protected function configure(): void { @@ -97,71 +93,12 @@ protected function configure(): void { parent::configure(); } - private function getUserOption(InputInterface $input): ?string { - if ($input->getOption('user')) { - return (string)$input->getOption('user'); - } - - return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null; - } - - private function getPasswordOption(InputInterface $input): ?string { - if ($input->getOption('password')) { - return (string)$input->getOption('password'); - } - - return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null; - } - protected function execute(InputInterface $input, OutputInterface $output): int { - $mount = $this->globalService->getStorage($input->getArgument('mount_id')); - if (is_null($mount)) { - $output->writeln('Mount not found'); + [$mount, $storage] = $this->createStorage($input, $output); + if ($storage === null) { return self::FAILURE; } - $noAuth = false; - - $userOption = $this->getUserOption($input); - $passwordOption = $this->getPasswordOption($input); - - // if only the user is provided, we get the user object to pass along to the auth backend - // this allows using saved user credentials - $user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null; - - try { - $authBackend = $mount->getAuthMechanism(); - $authBackend->manipulateStorageConfig($mount, $user); - } catch (InsufficientDataForMeaningfulAnswerException $e) { - $noAuth = true; - } catch (StorageNotAvailableException $e) { - $noAuth = true; - } - if ($userOption) { - $mount->setBackendOption('user', $userOption); - } - if ($passwordOption) { - $mount->setBackendOption('password', $passwordOption); - } - - try { - $backend = $mount->getBackend(); - $backend->manipulateStorageConfig($mount, $user); - } catch (InsufficientDataForMeaningfulAnswerException $e) { - $noAuth = true; - } catch (StorageNotAvailableException $e) { - $noAuth = true; - } - - try { - $storage = $this->createStorage($mount); - } catch (\Exception $e) { - $output->writeln('Error while trying to create storage'); - if ($noAuth) { - $output->writeln('Login and/or password required'); - } - return self::FAILURE; - } if (!$storage instanceof INotifyStorage) { $output->writeln('Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications'); return self::FAILURE; @@ -189,11 +126,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int return self::SUCCESS; } - private function createStorage(StorageConfig $mount): IStorage { - $class = $mount->getBackend()->getStorageClass(); - return new $class($mount->getBackendOptions()); - } - private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void { $parent = ltrim(dirname($path), '/'); if ($parent === '.') { diff --git a/apps/files_external/lib/Command/Scan.php b/apps/files_external/lib/Command/Scan.php new file mode 100644 index 0000000000000..5fa8f8108d21e --- /dev/null +++ b/apps/files_external/lib/Command/Scan.php @@ -0,0 +1,155 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Files_External\Command; + +use OC\Files\Cache\Scanner; +use OCA\Files_External\Service\GlobalStoragesService; +use OCP\IUserManager; +use Symfony\Component\Console\Helper\Table; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Scan extends StorageAuthBase { + protected float $execTime = 0; + protected int $foldersCounter = 0; + protected int $filesCounter = 0; + + public function __construct( + GlobalStoragesService $globalService, + IUserManager $userManager + ) { + parent::__construct($globalService, $userManager); + } + + protected function configure(): void { + $this + ->setName('files_external:scan') + ->setDescription('Scan an external storage for changed files') + ->addArgument( + 'mount_id', + InputArgument::REQUIRED, + 'the mount id of the mount to scan' + )->addOption( + 'user', + 'u', + InputOption::VALUE_REQUIRED, + 'The username for the remote mount (required only for some mount configuration that don\'t store credentials)' + )->addOption( + 'password', + 'p', + InputOption::VALUE_REQUIRED, + 'The password for the remote mount (required only for some mount configuration that don\'t store credentials)' + )->addOption( + 'path', + '', + InputOption::VALUE_OPTIONAL, + 'The path in the storage to scan', + '' + ); + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + [, $storage] = $this->createStorage($input, $output); + if ($storage === null) { + return 1; + } + + $path = $input->getOption('path'); + + $this->execTime = -microtime(true); + + /** @var Scanner $scanner */ + $scanner = $storage->getScanner(); + + $scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) { + $output->writeln("\tFile\t$path", OutputInterface::VERBOSITY_VERBOSE); + ++$this->filesCounter; + $this->abortIfInterrupted(); + }); + + $scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) { + $output->writeln("\tFolder\t$path", OutputInterface::VERBOSITY_VERBOSE); + ++$this->foldersCounter; + $this->abortIfInterrupted(); + }); + + $scanner->scan($path); + + $this->presentStats($output); + + return 0; + } + + /** + * @param OutputInterface $output + */ + protected function presentStats(OutputInterface $output): void { + // Stop the timer + $this->execTime += microtime(true); + + $headers = [ + 'Folders', 'Files', 'Elapsed time' + ]; + + $this->showSummary($headers, [], $output); + } + + /** + * Shows a summary of operations + * + * @param string[] $headers + * @param string[] $rows + * @param OutputInterface $output + */ + protected function showSummary(array $headers, array $rows, OutputInterface $output): void { + $niceDate = $this->formatExecTime(); + if (!$rows) { + $rows = [ + $this->foldersCounter, + $this->filesCounter, + $niceDate, + ]; + } + $table = new Table($output); + $table + ->setHeaders($headers) + ->setRows([$rows]); + $table->render(); + } + + + /** + * Formats microtime into a human readable format + * + * @return string + */ + protected function formatExecTime(): string { + $secs = round($this->execTime); + # convert seconds into HH:MM:SS form + return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60); + } +} diff --git a/apps/files_external/lib/Command/StorageAuthBase.php b/apps/files_external/lib/Command/StorageAuthBase.php new file mode 100644 index 0000000000000..ffa9da8551eec --- /dev/null +++ b/apps/files_external/lib/Command/StorageAuthBase.php @@ -0,0 +1,130 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Files_External\Command; + + +use OC\Core\Command\Base; +use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException; +use OCA\Files_External\Lib\StorageConfig; +use OCA\Files_External\NotFoundException; +use OCA\Files_External\Service\GlobalStoragesService; +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\IUserManager; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +abstract class StorageAuthBase extends Base { + public function __construct( + protected GlobalStoragesService $globalService, + protected IUserManager $userManager, + ) { + parent::__construct(); + } + + private function getUserOption(InputInterface $input): ?string { + if ($input->getOption('user')) { + return (string)$input->getOption('user'); + } + + return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null; + } + + private function getPasswordOption(InputInterface $input): ?string { + if ($input->getOption('password')) { + return (string)$input->getOption('password'); + } + + return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null; + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + * @return array + * @psalm-return array{0: StorageConfig, 1: IStorage}|array{0: null, 1: null} + */ + protected function createStorage(InputInterface $input, OutputInterface $output): array { + try { + /** @var StorageConfig|null $mount */ + $mount = $this->globalService->getStorage($input->getArgument('mount_id')); + } catch (NotFoundException $e) { + $output->writeln('Mount not found'); + return [null, null]; + } + if (is_null($mount)) { + $output->writeln('Mount not found'); + return [null, null]; + } + $noAuth = false; + + $userOption = $this->getUserOption($input); + $passwordOption = $this->getPasswordOption($input); + + // if only the user is provided, we get the user object to pass along to the auth backend + // this allows using saved user credentials + $user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null; + + try { + $authBackend = $mount->getAuthMechanism(); + $authBackend->manipulateStorageConfig($mount, $user); + } catch (InsufficientDataForMeaningfulAnswerException $e) { + $noAuth = true; + } catch (StorageNotAvailableException $e) { + $noAuth = true; + } + + if ($userOption) { + $mount->setBackendOption('user', $userOption); + } + if ($passwordOption) { + $mount->setBackendOption('password', $passwordOption); + } + + try { + $backend = $mount->getBackend(); + $backend->manipulateStorageConfig($mount, $user); + } catch (InsufficientDataForMeaningfulAnswerException $e) { + $noAuth = true; + } catch (StorageNotAvailableException $e) { + $noAuth = true; + } + + try { + $class = $mount->getBackend()->getStorageClass(); + /** @var IStorage $storage */ + $storage = new $class($mount->getBackendOptions()); + if (!$storage->test()) { + throw new \Exception(); + } + return [$mount, $storage]; + } catch (\Exception $e) { + $output->writeln('Error while trying to create storage'); + if ($noAuth) { + $output->writeln('Username and/or password required'); + } + return [null, null]; + } + } +}