Merge pull request #14913 from nextcloud/bugfix/6954/scan-external-s3

Fix directory detection for s3

Author: rullzer

apps/files_external/lib/Lib/Storage/AmazonS3.php

@@ -61,13 +61,17 @@ public function needsPartFile() {
 	/** @var CappedMemoryCache|Result[] */
 	private $objectCache;
 
+	/** @var CappedMemoryCache|bool[] */
+	private $directoryCache;
+
 	/** @var CappedMemoryCache|array */
 	private $filesCache;
 
 	public function __construct($parameters) {
 		parent::__construct($parameters);
 		$this->parseParams($parameters);
 		$this->objectCache = new CappedMemoryCache();
+		$this->directoryCache = new CappedMemoryCache();
 		$this->filesCache = new CappedMemoryCache();
 	}
 
@@ -98,6 +102,7 @@ private function cleanKey($path) {
 
 	private function clearCache() {
 		$this->objectCache = new CappedMemoryCache();
+		$this->directoryCache = new CappedMemoryCache();
 		$this->filesCache = new CappedMemoryCache();
 	}
 
@@ -110,7 +115,7 @@ private function invalidateCache($key) {
 				unset($this->objectCache[$existingKey]);
 			}
 		}
-		unset($this->filesCache[$key]);
+		unset($this->directoryCache[$key], $this->filesCache[$key]);
 	}
 
 	/**
@@ -135,6 +140,41 @@ private function headObject($key) {
 		return $this->objectCache[$key];
 	}
 
+	/**
+	 * Return true if directory exists
+	 *
+	 * There are no folders in s3. A folder like structure could be archived
+	 * by prefixing files with the folder name.
+	 *
+	 * Implementation from flysystem-aws-s3-v3:
+	 * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
+	 *
+	 * @param $path
+	 * @return bool
+	 * @throws \Exception
+	 */
+	private function doesDirectoryExist($path) {
+		if (!isset($this->directoryCache[$path])) {
+			// Maybe this isn't an actual key, but a prefix.
+			// Do a prefix listing of objects to determine.
+			try {
+				$result = $this->getConnection()->listObjects([
+					'Bucket' => $this->bucket,
+					'Prefix' => rtrim($path, '/') . '/',
+					'MaxKeys' => 1,
+				]);
+				$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
+			} catch (S3Exception $e) {
+				if ($e->getStatusCode() === 403) {
+					$this->directoryCache[$path] = false;
+				}
+				throw $e;
+			}
+		}
+
+		return $this->directoryCache[$path];
+	}
+
 	/**
 	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
 	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
@@ -294,7 +334,9 @@ public function opendir($path) {
 				// sub folders
 				if (is_array($result['CommonPrefixes'])) {
 					foreach ($result['CommonPrefixes'] as $prefix) {
-						$files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));
+						$directoryName = trim($prefix['Prefix'], '/');
+						$files[] = substr($directoryName, strlen($path));
+						$this->directoryCache[$directoryName] = true;
 					}
 				}
 				if (is_array($result['Contents'])) {
@@ -392,8 +434,13 @@ private function getLastModified($path) {
 
 	public function is_dir($path) {
 		$path = $this->normalizePath($path);
+
+		if (isset($this->filesCache[$path])) {
+			return false;
+		}
+
 		try {
-			return $this->isRoot($path) || $this->headObject($path . '/');
+			return $this->isRoot($path) || $this->doesDirectoryExist($path);
 		} catch (S3Exception $e) {
 			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
 			return false;
@@ -411,7 +458,7 @@ public function filetype($path) {
 			if (isset($this->filesCache[$path]) || $this->headObject($path)) {
 				return 'file';
 			}
-			if ($this->headObject($path . '/')) {
+			if ($this->doesDirectoryExist($path)) {
 				return 'dir';
 			}
 		} catch (S3Exception $e) {