From 191d7c32bf82e5d78fc723f3a08dcb60e4227ae1 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Mon, 9 May 2022 14:19:17 +0200 Subject: [PATCH] Allow scanning for metadata with occ scan:file --generate-metadata Signed-off-by: Carl Schwan Signed-off-by: Louis Chemineau --- apps/files/lib/Command/Scan.php | 49 +++++++++++++------ lib/private/Metadata/FileMetadataMapper.php | 49 +++++-------------- lib/private/Metadata/MetadataManager.php | 10 +--- .../Metadata/Provider/ExifProvider.php | 21 ++++++++ 4 files changed, 68 insertions(+), 61 deletions(-) diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index b40e963efc6df..f1596fa98a5db 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -37,8 +37,11 @@ use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OCP\Files\File; use OC\ForbiddenException; +use OC\Metadata\MetadataManager; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\StorageNotAvailableException; @@ -51,19 +54,22 @@ use Symfony\Component\Console\Output\OutputInterface; class Scan extends Base { + private IUserManager $userManager; + protected float $execTime = 0; + protected int $foldersCounter = 0; + protected int $filesCounter = 0; + private IRootFolder $root; + private MetadataManager $metadataManager; - /** @var IUserManager $userManager */ - private $userManager; - /** @var float */ - protected $execTime = 0; - /** @var int */ - protected $foldersCounter = 0; - /** @var int */ - protected $filesCounter = 0; - - public function __construct(IUserManager $userManager) { + public function __construct( + IUserManager $userManager, + IRootFolder $rootFolder, + MetadataManager $metadataManager + ) { $this->userManager = $userManager; parent::__construct(); + $this->root = $rootFolder; + $this->metadataManager = $metadataManager; } protected function configure() { @@ -83,6 +89,12 @@ protected function configure() { InputArgument::OPTIONAL, 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' ) + ->addOption( + 'generate-metadata', + null, + InputOption::VALUE_NONE, + 'Generate metadata for all scanned files' + ) ->addOption( 'all', null, @@ -106,21 +118,26 @@ protected function configure() { ); } - protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) { + protected function scanFiles(string $user, string $path, bool $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void { $connection = $this->reconnectToDatabase($output); $scanner = new \OC\Files\Utils\Scanner( $user, new ConnectionAdapter($connection), - \OC::$server->query(IEventDispatcher::class), + \OC::$server->get(IEventDispatcher::class), \OC::$server->get(LoggerInterface::class) ); # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception - - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata) { $output->writeln("\tFile\t$path", OutputInterface::VERBOSITY_VERBOSE); ++$this->filesCounter; $this->abortIfInterrupted(); + if ($scanMetadata) { + $node = $this->root->get($path); + if ($node instanceof File) { + $this->metadataManager->generateMetadata($node, false); + } + } }); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { @@ -197,7 +214,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int ++$user_count; if ($this->userManager->userExists($user)) { $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); - $this->scanFiles($user, $path, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only')); + $this->scanFiles($user, $path, $input->getOption('generate-metadata'), $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only')); $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); } else { $output->writeln("Unknown user $user_count $user"); @@ -291,7 +308,7 @@ protected function showSummary($headers, $rows, OutputInterface $output) { protected function formatExecTime() { $secs = round($this->execTime); # convert seconds into HH:MM:SS form - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ( (int)($secs / 60) % 60), $secs % 60); + return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); } protected function reconnectToDatabase(OutputInterface $output): Connection { diff --git a/lib/private/Metadata/FileMetadataMapper.php b/lib/private/Metadata/FileMetadataMapper.php index 4d5591baf1feb..f8f8df4bf3bea 100644 --- a/lib/private/Metadata/FileMetadataMapper.php +++ b/lib/private/Metadata/FileMetadataMapper.php @@ -112,59 +112,36 @@ public function clear(int $fileId): void { * @return Entity the saved entity with the set id * @throws Exception * @throws \InvalidArgumentException if entity has no id - * @since 14.0.0 */ public function update(Entity $entity): Entity { - // if entity wasn't changed it makes no sense to run a db query - $properties = $entity->getUpdatedFields(); - if (\count($properties) === 0) { - return $entity; + if (!($entity instanceof FileMetadata)) { + throw new \Exception("Entity should be a FileMetadata entity"); } // entity needs an id $id = $entity->getId(); if ($id === null) { - throw new \InvalidArgumentException( - 'Entity which should be updated has no id'); - } - - if (!($entity instanceof FileMetadata)) { - throw new \Exception("Entity should be a FileMetadata entity"); + throw new \InvalidArgumentException('Entity which should be updated has no id'); } // entity needs an group_name $groupName = $entity->getGroupName(); - if ($id === null) { - throw new \InvalidArgumentException( - 'Entity which should be updated has no group_name'); - } - - // get updated fields to save, fields have to be set using a setter to - // be saved - // do not update the id and group_name field - unset($properties['id']); - unset($properties['group_name']); - - $qb = $this->db->getQueryBuilder(); - $qb->update($this->tableName); - - // build the fields - foreach ($properties as $property => $updated) { - $column = $entity->propertyToColumn($property); - $getter = 'get' . ucfirst($property); - $value = $entity->$getter(); - - $type = $this->getParameterTypeForProperty($entity, $property); - $qb->set($column, $qb->createNamedParameter($value, $type)); + if ($groupName === null) { + throw new \InvalidArgumentException('Entity which should be updated has no group_name'); } $idType = $this->getParameterTypeForProperty($entity, 'id'); $groupNameType = $this->getParameterTypeForProperty($entity, 'groupName'); + $metadataValue = $entity->getMetadata(); + $metadataType = $this->getParameterTypeForProperty($entity, 'metadata'); - $qb->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType))) - ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType))); + $qb = $this->db->getQueryBuilder(); - $qb->executeStatement(); + $qb->update($this->tableName) + ->set('metadata', $qb->createNamedParameter($metadataValue, $metadataType)) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType))) + ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType))) + ->executeStatement(); return $entity; } diff --git a/lib/private/Metadata/MetadataManager.php b/lib/private/Metadata/MetadataManager.php index d1cb896febfc5..77407a2f5295b 100644 --- a/lib/private/Metadata/MetadataManager.php +++ b/lib/private/Metadata/MetadataManager.php @@ -21,27 +21,19 @@ use OC\Metadata\Provider\ExifProvider; use OCP\Files\File; -use OCP\IConfig; -use Psr\Log\LoggerInterface; class MetadataManager implements IMetadataManager { /** @var array */ private array $providers; private array $providerClasses; private FileMetadataMapper $fileMetadataMapper; - private IConfig $config; - private LoggerInterface $logger; public function __construct( - FileMetadataMapper $fileMetadataMapper, - IConfig $config, - LoggerInterface $logger + FileMetadataMapper $fileMetadataMapper ) { $this->providers = []; $this->providerClasses = []; $this->fileMetadataMapper = $fileMetadataMapper; - $this->config = $config; - $this->logger = $logger; // TODO move to another place, where? $this->registerProvider(ExifProvider::class); diff --git a/lib/private/Metadata/Provider/ExifProvider.php b/lib/private/Metadata/Provider/ExifProvider.php index aa0b546468216..02024bd3877be 100644 --- a/lib/private/Metadata/Provider/ExifProvider.php +++ b/lib/private/Metadata/Provider/ExifProvider.php @@ -1,5 +1,25 @@ + * @copyright Copyright 2022 Louis Chmn + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + namespace OC\Metadata\Provider; use OC\Metadata\FileMetadata; @@ -24,6 +44,7 @@ public static function isAvailable(): bool { return extension_loaded('exif'); } + /** @return array{'gps': FileMetadata, 'size': FileMetadata} */ public function execute(File $file): array { $exifData = []; $fileDescriptor = $file->fopen('rb');