Skip to content

Commit

Permalink
add command to scan external storages directly
Browse files Browse the repository at this point in the history
the main use case of this over simply scanning through is the ability to provide a username and/or password for cases where login credentials are used

Signed-off-by: Robin Appelman <robin@icewind.nl>
  • Loading branch information
icewind1991 committed Sep 20, 2023
1 parent 09794b6 commit df9bb30
Show file tree
Hide file tree
Showing 6 changed files with 299 additions and 82 deletions.
1 change: 1 addition & 0 deletions apps/files_external/appinfo/info.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
<command>OCA\Files_External\Command\Backends</command>
<command>OCA\Files_External\Command\Verify</command>
<command>OCA\Files_External\Command\Notify</command>
<command>OCA\Files_External\Command\Scan</command>
</commands>

<settings>
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',
Expand Down
85 changes: 3 additions & 82 deletions apps/files_external/lib/Command/Notify.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,13 @@
namespace OCA\Files_External\Command;

use Doctrine\DBAL\Exception\DriverException;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Notify\IChange;
use OCP\Files\Notify\INotifyHandler;
use OCP\Files\Notify\IRenameChange;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
Expand All @@ -49,12 +45,9 @@
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Notify extends Base {
private GlobalStoragesService $globalService;
class Notify extends StorageAuthBase {
private IDBConnection $connection;
private LoggerInterface $logger;
/** @var IUserManager */
private $userManager;

public function __construct(
GlobalStoragesService $globalService,
Expand Down Expand Up @@ -107,79 +100,12 @@ protected function configure(): void {
parent::configure();
}

private function getUserOption(InputInterface $input): ?string {
if ($input->getOption('user')) {
return (string)$input->getOption('user');
} elseif (isset($_ENV['NOTIFY_USER'])) {
return $_ENV['NOTIFY_USER'];
} elseif (isset($_SERVER['NOTIFY_USER'])) {
return $_SERVER['NOTIFY_USER'];
} else {
return null;
}
}

private function getPasswordOption(InputInterface $input): ?string {
if ($input->getOption('password')) {
return (string)$input->getOption('password');
} elseif (isset($_ENV['NOTIFY_PASSWORD'])) {
return $_ENV['NOTIFY_PASSWORD'];
} elseif (isset($_SERVER['NOTIFY_PASSWORD'])) {
return $_SERVER['NOTIFY_PASSWORD'];
} else {
return null;
}
}

protected function execute(InputInterface $input, OutputInterface $output): int {
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
[$mount, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return 1;
}
$noAuth = false;

$userOption = $this->getUserOption($input);
$passwordOption = $this->getPasswordOption($input);

// if only the user is provided, we get the user object to pass along to the auth backend
// this allows using saved user credentials
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;

try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

if ($userOption) {
$mount->setBackendOption('user', $userOption);
}
if ($passwordOption) {
$mount->setBackendOption('password', $passwordOption);
}

try {
$backend = $mount->getBackend();
$backend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

try {
$storage = $this->createStorage($mount);
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Username and/or password required</error>');
}
return 1;
}
if (!$storage instanceof INotifyStorage) {
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
return 1;
Expand Down Expand Up @@ -207,11 +133,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return 0;
}

private function createStorage(StorageConfig $mount): IStorage {
$class = $mount->getBackend()->getStorageClass();
return new $class($mount->getBackendOptions());
}

private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun) {
$parent = ltrim(dirname($path), '/');
if ($parent === '.') {
Expand Down
157 changes: 157 additions & 0 deletions apps/files_external/lib/Command/Scan.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
<?php

declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

namespace OCA\Files_External\Command;

use OC\Files\Cache\Scanner;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\IUserManager;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Scan extends StorageAuthBase {
protected float $execTime = 0;
protected int $foldersCounter = 0;
protected int $filesCounter = 0;

public function __construct(
GlobalStoragesService $globalService,
IUserManager $userManager
) {
parent::__construct();
$this->globalService = $globalService;
$this->userManager = $userManager;
}

protected function configure(): void {
$this
->setName('files_external:scan')
->setDescription('Scan an external storage for changed files')
->addArgument(
'mount_id',
InputArgument::REQUIRED,
'the mount id of the mount to scan'
)->addOption(
'user',
'u',
InputOption::VALUE_REQUIRED,
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'password',
'p',
InputOption::VALUE_REQUIRED,
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'path',
'',
InputOption::VALUE_OPTIONAL,
'The path in the storage to scan',
''
);
parent::configure();
}

protected function execute(InputInterface $input, OutputInterface $output): int {
[, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return 1;
}

$path = $input->getOption('path');

$this->execTime = -microtime(true);

/** @var Scanner $scanner */
$scanner = $storage->getScanner();

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->filesCounter;
$this->abortIfInterrupted();
});

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->foldersCounter;
$this->abortIfInterrupted();
});

$scanner->scan($path);

$this->presentStats($output);

return 0;
}

/**
* @param OutputInterface $output
*/
protected function presentStats(OutputInterface $output): void {
// Stop the timer
$this->execTime += microtime(true);

$headers = [
'Folders', 'Files', 'Elapsed time'
];

$this->showSummary($headers, [], $output);
}

/**
* Shows a summary of operations
*
* @param string[] $headers
* @param string[] $rows
* @param OutputInterface $output
*/
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
$niceDate = $this->formatExecTime();
if (!$rows) {
$rows = [
$this->foldersCounter,
$this->filesCounter,
$niceDate,
];
}
$table = new Table($output);
$table
->setHeaders($headers)
->setRows([$rows]);
$table->render();
}


/**
* Formats microtime into a human readable format
*
* @return string
*/
protected function formatExecTime(): string {
$secs = round($this->execTime);
# convert seconds into HH:MM:SS form
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
}
}
Loading

0 comments on commit df9bb30

Please sign in to comment.