Skip to content

Commit f5919d5

Browse files
authored
Merge pull request #20033 from nextcloud/s3-seekable-stream
Enable fseek for files in S3 storage
2 parents 5a82de1 + e22a28e commit f5919d5

File tree

6 files changed

+228
-25
lines changed

6 files changed

+228
-25
lines changed

lib/composer/composer/autoload_classmap.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,6 +1025,7 @@
10251025
'OC\\Files\\Storage\\Wrapper\\Wrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/Wrapper.php',
10261026
'OC\\Files\\Stream\\Encryption' => $baseDir . '/lib/private/Files/Stream/Encryption.php',
10271027
'OC\\Files\\Stream\\Quota' => $baseDir . '/lib/private/Files/Stream/Quota.php',
1028+
'OC\\Files\\Stream\\SeekableHttpStream' => $baseDir . '/lib/private/Files/Stream/SeekableHttpStream.php',
10281029
'OC\\Files\\Type\\Detection' => $baseDir . '/lib/private/Files/Type/Detection.php',
10291030
'OC\\Files\\Type\\Loader' => $baseDir . '/lib/private/Files/Type/Loader.php',
10301031
'OC\\Files\\Type\\TemplateManager' => $baseDir . '/lib/private/Files/Type/TemplateManager.php',

lib/composer/composer/autoload_static.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1054,6 +1054,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
10541054
'OC\\Files\\Storage\\Wrapper\\Wrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Wrapper.php',
10551055
'OC\\Files\\Stream\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Encryption.php',
10561056
'OC\\Files\\Stream\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Quota.php',
1057+
'OC\\Files\\Stream\\SeekableHttpStream' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/SeekableHttpStream.php',
10571058
'OC\\Files\\Type\\Detection' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Detection.php',
10581059
'OC\\Files\\Type\\Loader' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Loader.php',
10591060
'OC\\Files\\Type\\TemplateManager' => __DIR__ . '/../../..' . '/lib/private/Files/Type/TemplateManager.php',

lib/private/Files/ObjectStore/S3ObjectTrait.php

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
use Aws\S3\ObjectUploader;
3131
use Aws\S3\S3Client;
3232
use Icewind\Streams\CallbackWrapper;
33+
use OC\Files\Stream\SeekableHttpStream;
3334

3435
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
3536

@@ -49,27 +50,29 @@ abstract protected function getConnection();
4950
* @since 7.0.0
5051
*/
5152
function readObject($urn) {
52-
$client = $this->getConnection();
53-
$command = $client->getCommand('GetObject', [
54-
'Bucket' => $this->bucket,
55-
'Key' => $urn
56-
]);
57-
$request = \Aws\serialize($command);
58-
$headers = [];
59-
foreach ($request->getHeaders() as $key => $values) {
60-
foreach ($values as $value) {
61-
$headers[] = "$key: $value";
53+
return SeekableHttpStream::open(function ($range) use ($urn) {
54+
$command = $this->getConnection()->getCommand('GetObject', [
55+
'Bucket' => $this->bucket,
56+
'Key' => $urn,
57+
'Range' => 'bytes=' . $range,
58+
]);
59+
$request = \Aws\serialize($command);
60+
$headers = [];
61+
foreach ($request->getHeaders() as $key => $values) {
62+
foreach ($values as $value) {
63+
$headers[] = "$key: $value";
64+
}
6265
}
63-
}
64-
$opts = [
65-
'http' => [
66-
'protocol_version' => 1.1,
67-
'header' => $headers
68-
]
69-
];
66+
$opts = [
67+
'http' => [
68+
'protocol_version' => 1.1,
69+
'header' => $headers,
70+
],
71+
];
7072

71-
$context = stream_context_create($opts);
72-
return fopen($request->getUri(), 'r', false, $context);
73+
$context = stream_context_create($opts);
74+
return fopen($request->getUri(), 'r', false, $context);
75+
});
7376
}
7477

7578
/**
@@ -87,7 +90,7 @@ function writeObject($urn, $stream) {
8790
$uploader = new MultipartUploader($this->getConnection(), $countStream, [
8891
'bucket' => $this->bucket,
8992
'key' => $urn,
90-
'part_size' => S3_UPLOAD_PART_SIZE
93+
'part_size' => S3_UPLOAD_PART_SIZE,
9194
]);
9295

9396
try {
@@ -114,7 +117,7 @@ function writeObject($urn, $stream) {
114117
function deleteObject($urn) {
115118
$this->getConnection()->deleteObject([
116119
'Bucket' => $this->bucket,
117-
'Key' => $urn
120+
'Key' => $urn,
118121
]);
119122
}
120123

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
<?php
2+
/**
3+
*
4+
* @copyright Copyright (c) 2020, Lukas Stabe (lukas@stabe.de)
5+
*
6+
* @license GNU AGPL version 3 or any later version
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the GNU Affero General Public License as
10+
* published by the Free Software Foundation, either version 3 of the
11+
* License, or (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU Affero General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU Affero General Public License
19+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
20+
*
21+
*/
22+
23+
namespace OC\Files\Stream;
24+
25+
use Icewind\Streams\File;
26+
27+
/**
28+
* A stream wrapper that uses http range requests to provide a seekable stream for http reading
29+
*/
30+
class SeekableHttpStream implements File {
31+
private const PROTOCOL = 'httpseek';
32+
33+
private static $registered = false;
34+
35+
/**
36+
* Registers the stream wrapper using the `httpseek://` url scheme
37+
* $return void
38+
*/
39+
private static function registerIfNeeded() {
40+
if (!self::$registered) {
41+
stream_wrapper_register(
42+
self::PROTOCOL,
43+
self::class
44+
);
45+
self::$registered = true;
46+
}
47+
}
48+
49+
/**
50+
* Open a readonly-seekable http stream
51+
*
52+
* The provided callback will be called with byte range and should return an http stream for the requested range
53+
*
54+
* @param callable $callback
55+
* @return false|resource
56+
*/
57+
public static function open(callable $callback) {
58+
$context = stream_context_create([
59+
SeekableHttpStream::PROTOCOL => [
60+
'callback' => $callback
61+
],
62+
]);
63+
64+
SeekableHttpStream::registerIfNeeded();
65+
return fopen(SeekableHttpStream::PROTOCOL . '://', 'r', false, $context);
66+
}
67+
68+
/** @var resource */
69+
public $context;
70+
71+
/** @var callable */
72+
private $openCallback;
73+
74+
/** @var resource */
75+
private $current;
76+
/** @var int */
77+
private $offset = 0;
78+
79+
private function reconnect(int $start) {
80+
$range = $start . '-';
81+
if ($this->current != null) {
82+
fclose($this->current);
83+
}
84+
85+
$this->current = ($this->openCallback)($range);
86+
87+
if ($this->current === false) {
88+
return false;
89+
}
90+
91+
$responseHead = stream_get_meta_data($this->current)['wrapper_data'];
92+
$rangeHeaders = array_values(array_filter($responseHead, function ($v) {
93+
return preg_match('#^content-range:#i', $v) === 1;
94+
}));
95+
if (!$rangeHeaders) {
96+
return false;
97+
}
98+
$contentRange = $rangeHeaders[0];
99+
100+
$content = trim(explode(':', $contentRange)[1]);
101+
$range = trim(explode(' ', $content)[1]);
102+
$begin = intval(explode('-', $range)[0]);
103+
104+
if ($begin !== $start) {
105+
return false;
106+
}
107+
108+
$this->offset = $begin;
109+
110+
return true;
111+
}
112+
113+
public function stream_open($path, $mode, $options, &$opened_path) {
114+
$options = stream_context_get_options($this->context)[self::PROTOCOL];
115+
$this->openCallback = $options['callback'];
116+
117+
return $this->reconnect(0);
118+
}
119+
120+
public function stream_read($count) {
121+
if (!$this->current) {
122+
return false;
123+
}
124+
$ret = fread($this->current, $count);
125+
$this->offset += strlen($ret);
126+
return $ret;
127+
}
128+
129+
public function stream_seek($offset, $whence = SEEK_SET) {
130+
switch ($whence) {
131+
case SEEK_SET:
132+
if ($offset === $this->offset) {
133+
return true;
134+
}
135+
return $this->reconnect($offset);
136+
case SEEK_CUR:
137+
if ($offset === 0) {
138+
return true;
139+
}
140+
return $this->reconnect($this->offset + $offset);
141+
case SEEK_END:
142+
return false;
143+
}
144+
return false;
145+
}
146+
147+
public function stream_tell() {
148+
return $this->offset;
149+
}
150+
151+
public function stream_stat() {
152+
return fstat($this->current);
153+
}
154+
155+
public function stream_eof() {
156+
return feof($this->current);
157+
}
158+
159+
public function stream_close() {
160+
fclose($this->current);
161+
}
162+
163+
public function stream_write($data) {
164+
return false;
165+
}
166+
167+
public function stream_set_option($option, $arg1, $arg2) {
168+
return false;
169+
}
170+
171+
public function stream_truncate($size) {
172+
return false;
173+
}
174+
175+
public function stream_lock($operation) {
176+
return false;
177+
}
178+
179+
public function stream_flush() {
180+
return; //noop because readonly stream
181+
}
182+
}

tests/lib/Files/ObjectStore/ObjectStoreTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ abstract class ObjectStoreTest extends TestCase {
3131
*/
3232
abstract protected function getInstance();
3333

34-
private function stringToStream($data) {
34+
protected function stringToStream($data) {
3535
$stream = fopen('php://temp', 'w+');
3636
fwrite($stream, $data);
3737
rewind($stream);

tests/lib/Files/ObjectStore/S3Test.php

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
class MultiPartUploadS3 extends S3 {
2828
function writeObject($urn, $stream) {
2929
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
30-
'mup_threshold' => 1
30+
'mup_threshold' => 1,
3131
]);
3232
}
3333
}
@@ -36,8 +36,8 @@ class NonSeekableStream extends Wrapper {
3636
public static function wrap($source) {
3737
$context = stream_context_create([
3838
'nonseek' => [
39-
'source' => $source
40-
]
39+
'source' => $source,
40+
],
4141
]);
4242
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
4343
}
@@ -83,4 +83,20 @@ public function testUploadNonSeekable() {
8383

8484
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
8585
}
86+
87+
public function testSeek() {
88+
$data = file_get_contents(__FILE__);
89+
90+
$instance = $this->getInstance();
91+
$instance->writeObject('seek', $this->stringToStream($data));
92+
93+
$read = $instance->readObject('seek');
94+
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
95+
96+
fseek($read, 10);
97+
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
98+
99+
fseek($read, 100, SEEK_CUR);
100+
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
101+
}
86102
}

0 commit comments

Comments
 (0)