Merge pull request php-curl-class#387 from zachborboa/updates

Implement a more efficient handling of requests in parallel

Author: zachborboa

examples/flickr_photo_search.php

@@ -3,7 +3,7 @@
 
 use \Curl\Curl;
 
-const FLICKR_API_KEY = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX');
+const FLICKR_API_KEY = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX';
 
 $data = array(
     'method' => 'flickr.photos.search',

src/Curl/Curl.php

@@ -100,7 +100,7 @@ public function __construct($base_url = null)
         }
 
         $this->curl = curl_init();
-        $this->id = 1;
+        $this->id = uniqid('', true);
         $this->setDefaultUserAgent();
         $this->setDefaultJsonDecoder();
         $this->setDefaultXmlDecoder();
@@ -385,6 +385,11 @@ public function exec($ch = null)
 
         $this->call($this->completeFunction);
 
+        // Close open file handles and reset the curl instance.
+        if (!($this->fileHandle === null)) {
+            $this->downloadComplete($this->fileHandle);
+        }
+
         return $this->response;
     }
 

src/Curl/MultiCurl.php

@@ -6,8 +6,10 @@ class MultiCurl
 {
     public $baseUrl = null;
     public $multiCurl;
-    public $curls = array();
-    private $nextCurlId = 1;
+    public $windowSize = 25;
+
+    private $curls = array();
+    private $activeCurls = array();
     private $isStarted = false;
 
     private $beforeSendFunction = null;
@@ -56,7 +58,7 @@ public function addDelete($url, $query_parameters = array(), $data = array())
         $curl->setURL($url, $query_parameters);
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'DELETE');
         $curl->setOpt(CURLOPT_POSTFIELDS, $curl->buildPostData($data));
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -90,7 +92,7 @@ public function addDownload($url, $mixed_filename)
         $curl->setOpt(CURLOPT_FILE, $curl->fileHandle);
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'GET');
         $curl->setOpt(CURLOPT_HTTPGET, true);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -113,7 +115,7 @@ public function addGet($url, $data = array())
         $curl->setURL($url, $data);
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'GET');
         $curl->setOpt(CURLOPT_HTTPGET, true);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -136,7 +138,7 @@ public function addHead($url, $data = array())
         $curl->setURL($url, $data);
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'HEAD');
         $curl->setOpt(CURLOPT_NOBODY, true);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -159,7 +161,7 @@ public function addOptions($url, $data = array())
         $curl->setURL($url, $data);
         $curl->unsetHeader('Content-Length');
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'OPTIONS');
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -183,7 +185,7 @@ public function addPatch($url, $data = array())
         $curl->unsetHeader('Content-Length');
         $curl->setOpt(CURLOPT_CUSTOMREQUEST, 'PATCH');
         $curl->setOpt(CURLOPT_POSTFIELDS, $data);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -225,7 +227,7 @@ public function addPost($url, $data = array(), $follow_303_with_post = false)
 
         $curl->setOpt(CURLOPT_POST, true);
         $curl->setOpt(CURLOPT_POSTFIELDS, $curl->buildPostData($data));
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -252,7 +254,7 @@ public function addPut($url, $data = array())
             $curl->setHeader('Content-Length', strlen($put_data));
         }
         $curl->setOpt(CURLOPT_POSTFIELDS, $put_data);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -279,7 +281,7 @@ public function addSearch($url, $data = array())
             $curl->setHeader('Content-Length', strlen($put_data));
         }
         $curl->setOpt(CURLOPT_POSTFIELDS, $put_data);
-        $this->addHandle($curl);
+        $this->queueHandle($curl);
         return $curl;
     }
 
@@ -522,40 +524,47 @@ public function start()
             return;
         }
 
-        foreach ($this->curls as $ch) {
-            $this->initHandle($ch);
+        $this->isStarted = true;
+
+        $window_size = $this->windowSize;
+        if ($window_size > count($this->curls)) {
+            $window_size = count($this->curls);
         }
 
-        $this->isStarted = true;
+        for ($i = 0; $i < $window_size; $i++) {
+            $this->initHandle(array_pop($this->curls));
+        }
 
         do {
             curl_multi_select($this->multiCurl);
             curl_multi_exec($this->multiCurl, $active);
 
             while (!($info_array = curl_multi_info_read($this->multiCurl)) === false) {
                 if ($info_array['msg'] === CURLMSG_DONE) {
-                    foreach ($this->curls as $key => $ch) {
+                    foreach ($this->activeCurls as $key => $ch) {
                         if ($ch->curl === $info_array['handle']) {
                             // Set the error code for multi handles using the "result" key in the array returned by
                             // curl_multi_info_read(). Using curl_errno() on a multi handle will incorrectly return 0
                             // for errors.
                             $ch->curlErrorCode = $info_array['result'];
                             $ch->exec($ch->curl);
-                            curl_multi_remove_handle($this->multiCurl, $ch->curl);
-                            unset($this->curls[$key]);
 
-                            // Close open file handles and reset the curl instance.
-                            if (!($ch->fileHandle === null)) {
-                                $ch->downloadComplete($ch->fileHandle);
+                            unset($this->activeCurls[$key]);
+
+                            // Start a new request before removing the handle of the completed one.
+                            if (count($this->curls) >= 1) {
+                                $this->initHandle(array_pop($this->curls));
                             }
+                            curl_multi_remove_handle($this->multiCurl, $ch->curl);
+
                             break;
                         }
                     }
                 }
             }
 
             if (!$active) {
-                $active = count($this->curls);
+                $active = count($this->activeCurls);
             }
         } while ($active > 0);
 
@@ -614,31 +623,22 @@ public function __destruct()
     }
 
     /**
-     * Add Handle
+     * Queue Handle
      *
      * @access private
      * @param  $curl
-     * @throws \ErrorException
      */
-    private function addHandle($curl)
+    private function queueHandle($curl)
     {
-        $curlm_error_code = curl_multi_add_handle($this->multiCurl, $curl->curl);
-        if (!($curlm_error_code === CURLM_OK)) {
-            throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code));
-        }
-        $this->curls[] = $curl;
-        $curl->id = $this->nextCurlId++;
-
-        if ($this->isStarted) {
-            $this->initHandle($curl);
-        }
+        $this->curls[$curl->id] = $curl;
     }
 
     /**
      * Init Handle
      *
      * @access private
      * @param  $curl
+     * @throws \ErrorException
      */
     private function initHandle($curl)
     {
@@ -664,6 +664,13 @@ private function initHandle($curl)
         }
         $curl->setJsonDecoder($this->jsonDecoder);
         $curl->setXmlDecoder($this->xmlDecoder);
+
+        $curlm_error_code = curl_multi_add_handle($this->multiCurl, $curl->curl);
+        if (!($curlm_error_code === CURLM_OK)) {
+            throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code));
+        }
+
+        $this->activeCurls[$curl->id] = $curl;
         $curl->call($curl->beforeSendFunction);
     }
 }

tests/script.sh

@@ -1,8 +1,11 @@
-# Check syntax in php files.
-find . -type "f" -iname "*.php" ! -path "*/vendor/*" -exec php -l {} \;
-
 errors=0
 
+# Check syntax in php files. Use `xargs' over `find -exec' as xargs exits with a value of 1 when any command errors.
+find . -type "f" -iname "*.php" ! -path "*/vendor/*" | xargs -L "1" php -l
+if [[ "${?}" -ne 0 ]]; then
+    ((errors++))
+fi
+
 # Run tests.
 phpunit --configuration tests/phpunit.xml
 if [[ "${?}" -ne 0 ]]; then