Merge pull request #20 from clue-labs/stream

Add streaming API for fetching larger files (fetchFileStream() method)

Author: clue

README.md

@@ -111,6 +111,40 @@ try {
 
 Refer to [clue/block-react](https://github.com/clue/php-block-react#readme) for more details.
 
+#### Streaming
+
+The following API endpoint resolves with the file contents as a string:
+
+```php
+$client->fetchFile($path);
+````
+
+Keep in mind that this means the whole string has to be kept in memory.
+This is easy to get started and works reasonably well for smaller files.
+
+For bigger files it's usually a better idea to use a streaming approach,
+where only small chunks have to be kept in memory.
+This works for (any number of) files of arbitrary sizes.
+
+The following API endpoint complements the default Promise-based API and returns
+an instance implementing `ReadableStreamInterface` instead:
+
+```php
+$stream = $client->fetchFileStream($path);
+
+$stream->on('data', function ($chunk) {
+    echo $chunk;
+});
+
+$stream->on('error', function (Exception $error) {
+    echo 'Error: ' . $error->getMessage() . PHP_EOL;
+});
+
+$stream->on('close', function () {
+    echo '[DONE]' . PHP_EOL;
+});
+```
+
 ## Install
 
 The recommended way to install this library is [through composer](http://getcomposer.org). [New to composer?](http://getcomposer.org/doc/00-intro.md)

composer.json

@@ -20,7 +20,8 @@
         "clue/buzz-react": "^0.5",
         "ext-simplexml": "*",
         "neitanod/forceutf8": "~1.4",
-        "rize/uri-template": "^0.3"
+        "rize/uri-template": "^0.3",
+        "clue/promise-stream-react": "^0.1"
     },
     "require-dev": {
         "clue/block-react": "~0.3.0"

examples/directory.php

@@ -3,6 +3,7 @@
 use Clue\React\ViewVcApi\Client;
 use React\EventLoop\Factory as LoopFactory;
 use Clue\React\Buzz\Browser;
+use React\Stream\Stream;
 
 require __DIR__ . '/../vendor/autoload.php';
 
@@ -31,7 +32,17 @@
 if (substr($path, -1) === '/') {
     $client->fetchDirectory($path, $revision)->then('print_r', 'printf');
 } else {
-    $client->fetchFile($path, $revision)->then('print_r', 'printf');
+    //$client->fetchFile($path, $revision)->then('print_r', 'printf');
+
+    $stream = $client->fetchFileStream($path, $revision);
+
+    // any errors
+    $stream->on('error', 'printf');
+
+    // pipe stream into STDOUT
+    $out = new Stream(STDOUT, $loop);
+    $out->pause();
+    $stream->pipe($out);
 }
 
 $loop->run();

src/Client.php

@@ -12,6 +12,7 @@
 use Clue\React\ViewVcApi\Io\Parser;
 use Clue\React\ViewVcApi\Io\Loader;
 use Rize\UriTemplate;
+use Clue\React\Promise\Stream;
 
 class Client
 {
@@ -65,6 +66,46 @@ public function fetchFile($path, $revision = null)
         );
     }
 
+    /**
+     * Reads the file contents of the given file path as a readable stream
+     *
+     * This works for files of arbitrary sizes as only small chunks have to
+     * be kept in memory. The resulting stream is a well-behaving readable stream
+     * that will emit the normal stream events.
+     *
+     * @param string      $path
+     * @param string|null $revision
+     * @return ReadableStreamInterface
+     * @throws InvalidArgumentException
+     * @see self::fetchFile()
+     */
+    public function fetchFileStream($path, $revision = null)
+    {
+        if (substr($path, -1) === '/') {
+            throw new InvalidArgumentException('File path MUST NOT end with trailing slash');
+        }
+
+        // TODO: fetching a directory redirects to path with trailing slash
+        // TODO: status returns 200 OK, but displays an error message anyways..
+        // TODO: see not-a-file.html
+        // TODO: reject all paths with trailing slashes
+
+        return Stream\unwrapReadable(
+            $this->browser->withOptions(array('streaming' => true))->get(
+                $this->uri->expand(
+                    '{+path}?view=co{&pathrev}',
+                    array(
+                        'path' => $path,
+                        'pathrev' => $revision
+                    )
+                )
+            )->then(function (ResponseInterface $response) {
+                // the body implements ReadableStreamInterface, so let's just return this to the unwrapper
+                return $response->getBody();
+            })
+        );
+    }
+
     public function fetchDirectory($path, $revision = null, $showAttic = false)
     {
         if (substr($path, -1) !== '/') {

tests/ClientTest.php

@@ -44,6 +44,25 @@ public function testFetchFile()
         $this->expectPromiseResolveWith('# hello', $promise);
     }
 
+    public function testFetchFileStream()
+    {
+        $response = new Response(200, array(), '# hello', '1.0', 'OK');
+
+        $this->expectRequest($this->uri . 'README.md?view=co')->will($this->returnValue(Promise\reject()));
+
+        $stream = $this->client->fetchFileStream('README.md');
+
+        $this->assertInstanceOf('React\Stream\ReadableStreamInterface', $stream);
+    }
+
+    /**
+     * @expectedException InvalidArgumentException
+     */
+    public function testInvalidFileStream()
+    {
+        $this->client->fetchFileStream('invalid/');
+    }
+
     public function testFetchFileExcessiveSlashesAreIgnored()
     {
         $this->expectRequest($this->uri . 'README.md?view=co')->will($this->returnValue(Promise\reject()));

tests/FunctionalApacheClientTest.php

@@ -4,6 +4,7 @@
 use React\EventLoop\Factory as LoopFactory;
 use Clue\React\Buzz\Browser;
 use Clue\React\Block;
+use Clue\React\Promise\Stream;
 
 class FunctionalApacheClientTest extends TestCase
 {
@@ -41,6 +42,17 @@ public function testFetchFile()
         $this->assertStringStartsWith('/*', $recipe);
     }
 
+    public function testFetchFileStream()
+    {
+        $file = 'jakarta/ecs/tags/V1_0/src/java/org/apache/ecs/AlignType.java';
+        $revision = '168703';
+
+        $promise = $this->viewvc->fetchFileStream($file, $revision);
+        $recipe = Block\await(Stream\buffer($promise), $this->loop);
+
+        $this->assertStringStartsWith('/*', $recipe);
+    }
+
     public function testFetchFileOldFileNowDeletedButRevisionAvailable()
     {
         $file = 'commons/STATUS';