Improve and add tests for Curl::fastDownload()

zachborboa · zachborboa · commit 76e74fdc4408 · 2023-06-13T02:52:59.000-07:00
diff --git a/src/Curl/Curl.php b/src/Curl/Curl.php
@@ -373,19 +373,23 @@ public function download($url, $mixed_filename)
      */
     public function fastDownload($url, $filename, $connections = 4)
     {
-        // Retrieve content length from the "Content-Length" header and use an
-        // HTTP GET request because not all hosts support HEAD requests.
-        $this->setOpts([
-            CURLOPT_CUSTOMREQUEST => 'GET',
-            CURLOPT_NOBODY        => true,
-            CURLOPT_HEADER        => true,
-            CURLOPT_ENCODING      => '',
-        ]);
-        $this->setUrl($url);
-        $this->exec();
+        // Retrieve content length from the "Content-Length" header from the url
+        // to download. Use an HTTP GET request without a body instead of a HEAD
+        // request because not all hosts support HEAD requests.
+        $curl = new Curl();
+        $curl->setOptInternal(CURLOPT_NOBODY, true);
+
+        // Pass user-specified options to the instance checking for content-length.
+        $curl->setOpts($this->userSetOptions);
+        $curl->get($url);
+
+        // Exit early when an error occurred.
+        if ($curl->error) {
+            return false;
+        }
 
-        $content_length = isset($this->responseHeaders['Content-Length']) ?
-            $this->responseHeaders['Content-Length'] : null;
+        $content_length = isset($curl->responseHeaders['Content-Length']) ?
+            $curl->responseHeaders['Content-Length'] : null;
 
         // Use a regular download when content length could not be determined.
         if (!$content_length) {
@@ -395,25 +399,21 @@ public function fastDownload($url, $filename, $connections = 4)
         // Divide chunk_size across the number of connections.
         $chunk_size = ceil($content_length / $connections);
 
-        // First bytes.
-        $offset = 0;
-        $next_chunk = $chunk_size;
-
         // Keep track of file name parts.
         $part_file_names = [];
 
         $multi_curl = new MultiCurl();
         $multi_curl->setConcurrency($connections);
-        $multi_curl->error(function ($instance) {
-            return false;
-        });
 
-        for ($i = 1; $i <= $connections; $i++) {
-            // If last chunk then no need to supply it.
-            // Range starts with 0, so subtract 1.
-            $next_chunk = $i === $connections ? '' : $next_chunk - 1;
+        for ($part_number = 1; $part_number <= $connections; $part_number++) {
+            $range_start = ($part_number - 1) * $chunk_size;
+            $range_end = $range_start + $chunk_size - 1;
+            if ($part_number === $connections) {
+                $range_end = '';
+            }
+            $range = $range_start . '-' . $range_end;
 
-            $part_file_name = $filename . '.part' . $i;
+            $part_file_name = $filename . '.part' . $part_number;
 
             // Save the file name of this part.
             $part_file_names[] = $part_file_name;
@@ -424,25 +424,28 @@ public function fastDownload($url, $filename, $connections = 4)
             }
 
             // Create file part.
-            $file_handle = fopen($part_file_name, 'w');
+            $file_handle = tmpfile();
 
+            // Setup the instance downloading a part.
             $curl = new Curl();
-            $curl->setOpt(CURLOPT_ENCODING, '');
-            $curl->setRange($offset . '-' . $next_chunk);
-            $curl->setFile($file_handle);
-            $curl->disableTimeout(); // otherwise download may fail.
             $curl->setUrl($url);
 
-            $curl->complete(function () use ($file_handle) {
-                fclose($file_handle);
-            });
+            // Pass user-specified options to the instance downloading a part.
+            $curl->setOpts($this->userSetOptions);
 
-            $multi_curl->addCurl($curl);
+            $curl->setOptInternal(CURLOPT_CUSTOMREQUEST, 'GET');
+            $curl->setOptInternal(CURLOPT_HTTPGET, true);
+            $curl->setRangeInternal($range);
+            $curl->setFileInternal($file_handle);
+            $curl->fileHandle = $file_handle;
 
-            if ($i !== $connections) {
-                $offset = $next_chunk + 1; // Add 1 to match offset.
-                $next_chunk = $next_chunk + $chunk_size;
-            }
+            $curl->downloadCompleteCallback = function ($instance, $tmpfile) use ($part_file_name) {
+                $fh = fopen($part_file_name, 'wb');
+                stream_copy_to_stream($tmpfile, $fh);
+                fclose($fh);
+            };
+
+            $multi_curl->addCurl($curl);
         }
 
         // Start the simultaneous downloads for each of the ranges in parallel.
@@ -457,7 +460,15 @@ public function fastDownload($url, $filename, $connections = 4)
         $main_file_handle = fopen($filename, 'w');
 
         foreach ($part_file_names as $part_file_name) {
+            if (!is_file($part_file_name)) {
+                return false;
+            }
+
             $file_handle = fopen($part_file_name, 'r');
+            if ($file_handle === false) {
+                return false;
+            }
+
             stream_copy_to_stream($file_handle, $main_file_handle);
             fclose($file_handle);
             unlink($part_file_name);
@@ -1156,6 +1167,9 @@ protected function setOptInternal($option, $value)
      */
     public function setOpts($options)
     {
+        if (!count($options)) {
+            return true;
+        }
         foreach ($options as $option => $value) {
             if (!$this->setOpt($option, $value)) {
                 return false;
@@ -1799,6 +1813,9 @@ function ($key) use ($curl_const_prefix) {
             echo "\n";
         } elseif (is_bool($value)) {
             echo ' ' . ($value ? 'true' : 'false') . "\n";
+        } elseif (is_array($value)) {
+            echo ' ';
+            var_dump($value);
         } elseif (is_callable($value)) {
             echo ' (callable)' . "\n";
         } else {
diff --git a/tests/PHPCurlClass/PHPCurlClassTest.php b/tests/PHPCurlClass/PHPCurlClassTest.php
@@ -884,6 +884,81 @@ public function testDownloadCallbackError()
         $this->assertFalse($download_callback_called);
     }
 
+    public function testFastDownloadSuccessOnly()
+    {
+        // Create a local file.
+        $local_file_path = \Helper\get_png();
+
+        // Upload file to server.
+        $uploaded_server_file_path = \Helper\upload_file_to_server($local_file_path);
+
+        // Download server file and save locally.
+        $url = Test::TEST_URL . '?' . http_build_query([
+            'file_path' => $uploaded_server_file_path,
+        ]);
+        $downloaded_local_file_path = \Helper\get_tmp_file_path();
+        $curl = new Curl();
+        $curl->setHeader('X-DEBUG-TEST', 'download_response');
+        $curl->fastDownload($url, $downloaded_local_file_path);
+
+        $this->assertEquals(md5_file($local_file_path), md5_file($downloaded_local_file_path));
+
+        // Remove server file.
+        \Helper\remove_file_from_server($uploaded_server_file_path);
+
+        unlink($local_file_path);
+        $this->assertFalse(file_exists($local_file_path));
+
+        unlink($downloaded_local_file_path);
+        $this->assertFalse(file_exists($downloaded_local_file_path));
+    }
+
+    public function testFastDownloadFailOnly()
+    {
+        $url = Test::TEST_URL . '?failures=1';
+        $downloaded_local_file_path = \Helper\get_tmp_file_path();
+
+        $curl = new Curl();
+        $curl->setHeader('X-DEBUG-TEST', 'retry');
+        $response = $curl->fastDownload($url, $downloaded_local_file_path);
+
+        $this->assertFalse($response);
+    }
+
+    public function testFastDownloadSuccessFail()
+    {
+        $url = Test::TEST_URL . '?failures=0,1';
+        $downloaded_local_file_path = \Helper\get_tmp_file_path();
+        $cookie_jar = __DIR__ . '/cookiejar.txt';
+        $connections = 1;
+
+        $curl = new Curl();
+        $curl->setHeader('X-DEBUG-TEST', 'retry');
+        $curl->setCookieFile($cookie_jar);
+        $curl->setCookieJar($cookie_jar);
+        $response = $curl->fastDownload($url, $downloaded_local_file_path, $connections);
+
+        $this->assertFalse($response);
+        $this->assertTrue(unlink($cookie_jar));
+    }
+
+    public function testFastDownloadSuccessSuccessFail()
+    {
+        $url = Test::TEST_URL . '?failures=0,0,1';
+        $downloaded_local_file_path = \Helper\get_tmp_file_path();
+        $cookie_jar = __DIR__ . '/cookiejar.txt';
+        $connections = 2;
+
+        $curl = new Curl();
+        $curl->setHeader('X-DEBUG-TEST', 'retry');
+        $curl->setCookieFile($cookie_jar);
+        $curl->setCookieJar($cookie_jar);
+        $response = $curl->fastDownload($url, $downloaded_local_file_path, $connections);
+
+        $this->assertFalse($response);
+        $this->assertTrue(unlink($cookie_jar));
+    }
+
     public function testMaxFilesize()
     {
         $tests = [
diff --git a/tests/server.php b/tests/server.php
@@ -283,9 +283,13 @@
     $unsafe_file_path = $_GET['file_path'];
     header('Content-Type: image/png');
     header('Content-Disposition: attachment; filename="image.png"');
-    header('Content-Length: ' . filesize($unsafe_file_path));
-    header('ETag: ' . md5_file($unsafe_file_path));
-    readfile($unsafe_file_path);
+
+    if (!isset($_SERVER['HTTP_RANGE'])) {
+        header('ETag: ' . md5_file($unsafe_file_path));
+    }
+
+    $server = new ContentRangeServer\ContentRangeServer();
+    $server->serve($unsafe_file_path);
     exit;
 } elseif ($test === 'download_file_size') {
     if (isset($_GET['http_response_code'])) {
