Skip to content

Commit 976008d

Browse files
zachborboazachborboa
committed
Implement rate limiting requests via MultiCurl::setRateLimit()
1 parent 9c00119 commit 976008d

File tree

8 files changed

+1332
-55
lines changed

8 files changed

+1332
-55
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ MultiCurl::setProxyAuth($auth)
333333
MultiCurl::setProxyTunnel($tunnel = true)
334334
MultiCurl::setProxyType($type)
335335
MultiCurl::setRange($range)
336+
MultiCurl::setRateLimit($rate_limit)
336337
MultiCurl::setReferer($referer)
337338
MultiCurl::setReferrer($referrer)
338339
MultiCurl::setRetry($mixed)

‎src/Curl/MultiCurl.php

Lines changed: 137 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,16 @@ class MultiCurl
1515
private $concurrency = 25;
1616
private $nextCurlId = 0;
1717

18+
private $rateLimit = null;
19+
private $rateLimitEnabled = false;
20+
private $rateLimitReached = false;
21+
private $maxRequests = null;
22+
private $interval = null;
23+
private $intervalSeconds = null;
24+
private $unit = null;
25+
private $currentStartTime = null;
26+
private $currentRequestCount = 0;
27+
1828
private $beforeSendCallback = null;
1929
private $successCallback = null;
2030
private $errorCallback = null;
@@ -713,6 +723,57 @@ public function setRange($range)
713723
$this->setOpt(CURLOPT_RANGE, $range);
714724
}
715725

726+
/**
727+
* Set Rate Limit
728+
*
729+
* @access public
730+
* @param $rate_limit string (e.g. "60/1m").
731+
*/
732+
public function setRateLimit($rate_limit)
733+
{
734+
$rate_limit_pattern =
735+
'/' . // delimiter
736+
'^' . // assert start
737+
'(\d+)' . // digit(s)
738+
'\/' . // slash
739+
'(\d+)?' . // digit(s), optional
740+
'(s|m|h)' . // unit, s for seconds, m for minutes, h for hours
741+
'$' . // assert end
742+
'/' . // delimiter
743+
'i' . // case-insensitive matches
744+
'';
745+
if (!preg_match($rate_limit_pattern, $rate_limit, $matches)) {
746+
throw new \UnexpectedValueException(
747+
'rate limit must be formatted as $max_requests/$interval(s|m|h) ' .
748+
'(e.g. "60/1m" for a maximum of 60 requests per 1 minute)'
749+
);
750+
}
751+
752+
$max_requests = (int)$matches['1'];
753+
if ($matches['2'] === '') {
754+
$interval = 1;
755+
} else {
756+
$interval = (int)$matches['2'];
757+
}
758+
$unit = strtolower($matches['3']);
759+
760+
// Convert interval to seconds based on unit.
761+
if ($unit === 's') {
762+
$interval_seconds = $interval * 1;
763+
} elseif ($unit === 'm') {
764+
$interval_seconds = $interval * 60;
765+
} elseif ($unit === 'h') {
766+
$interval_seconds = $interval * 3600;
767+
}
768+
769+
$this->rateLimit = $max_requests . '/' . $interval . $unit;
770+
$this->rateLimitEnabled = true;
771+
$this->maxRequests = $max_requests;
772+
$this->interval = $interval;
773+
$this->intervalSeconds = $interval_seconds;
774+
$this->unit = $unit;
775+
}
776+
716777
/**
717778
* Set Referer
718779
*
@@ -823,17 +884,21 @@ public function start()
823884
}
824885

825886
$this->isStarted = true;
887+
$this->currentStartTime = microtime(true);
888+
$this->currentRequestCount = 0;
826889

827-
$concurrency = $this->concurrency;
828-
if ($concurrency > count($this->curls)) {
829-
$concurrency = count($this->curls);
830-
}
890+
do {
891+
while (count($this->curls) &&
892+
count($this->activeCurls) < $this->concurrency &&
893+
(!$this->rateLimitEnabled || $this->hasRequestQuota())
894+
) {
895+
$this->initHandle();
896+
}
831897

832-
for ($i = 0; $i < $concurrency; $i++) {
833-
$this->initHandle(array_shift($this->curls));
834-
}
898+
if ($this->rateLimitEnabled && !count($this->activeCurls) && !$this->hasRequestQuota()) {
899+
$this->waitUntilRequestQuotaAvailable();
900+
}
835901

836-
do {
837902
// Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.
838903
// https://bugs.php.net/bug.php?id=63411
839904
if (curl_multi_select($this->multiCurl) === -1) {
@@ -842,7 +907,7 @@ public function start()
842907

843908
curl_multi_exec($this->multiCurl, $active);
844909

845-
while (!($info_array = curl_multi_info_read($this->multiCurl)) === false) {
910+
while (($info_array = curl_multi_info_read($this->multiCurl)) !== false) {
846911
if ($info_array['msg'] === CURLMSG_DONE) {
847912
foreach ($this->activeCurls as $key => $curl) {
848913
if ($curl->curl === $info_array['handle']) {
@@ -868,10 +933,7 @@ public function start()
868933
// Remove completed instance from active curls.
869934
unset($this->activeCurls[$key]);
870935

871-
// Start new requests before removing the handle of the completed one.
872-
while (count($this->curls) >= 1 && count($this->activeCurls) < $this->concurrency) {
873-
$this->initHandle(array_shift($this->curls));
874-
}
936+
// Remove handle of the completed instance.
875937
curl_multi_remove_handle($this->multiCurl, $curl->curl);
876938

877939
// Clean up completed instance.
@@ -883,11 +945,7 @@ public function start()
883945
}
884946
}
885947
}
886-
887-
if (!$active) {
888-
$active = count($this->activeCurls);
889-
}
890-
} while ($active > 0);
948+
} while ($active || count($this->activeCurls) || count($this->curls));
891949

892950
$this->isStarted = false;
893951
}
@@ -993,8 +1051,17 @@ private function queueHandle($curl)
9931051
* @param $curl
9941052
* @throws \ErrorException
9951053
*/
996-
private function initHandle($curl)
1054+
private function initHandle()
9971055
{
1056+
$curl = array_shift($this->curls);
1057+
if ($curl === null) {
1058+
return;
1059+
}
1060+
1061+
// Add instance to list of active curls.
1062+
$this->currentRequestCount += 1;
1063+
$this->activeCurls[$curl->id] = $curl;
1064+
9981065
// Set callbacks if not already individually set.
9991066
if ($curl->beforeSendCallback === null) {
10001067
$curl->beforeSend($this->beforeSendCallback);
@@ -1033,7 +1100,57 @@ private function initHandle($curl)
10331100
throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code));
10341101
}
10351102

1036-
$this->activeCurls[$curl->id] = $curl;
10371103
$curl->call($curl->beforeSendCallback);
10381104
}
1105+
1106+
/**
1107+
* Has Request Quota
1108+
*
1109+
* Checks if there is any available quota to make additional requests while
1110+
* rate limiting is enabled.
1111+
*
1112+
* @access private
1113+
*/
1114+
private function hasRequestQuota()
1115+
{
1116+
// Calculate if there's request quota since ratelimiting is enabled.
1117+
if ($this->rateLimitEnabled) {
1118+
// Determine if the limit of requests per interval has been reached.
1119+
if ($this->currentRequestCount >= $this->maxRequests) {
1120+
$elapsed_seconds = microtime(true) - $this->currentStartTime;
1121+
if ($elapsed_seconds <= $this->intervalSeconds) {
1122+
$this->rateLimitReached = true;
1123+
return false;
1124+
} elseif ($this->rateLimitReached) {
1125+
$this->rateLimitReached = false;
1126+
$this->currentStartTime = microtime(true);
1127+
$this->currentRequestCount = 0;
1128+
}
1129+
}
1130+
1131+
return true;
1132+
} else {
1133+
return true;
1134+
}
1135+
}
1136+
1137+
/**
1138+
* Wait Until Request Quota Available
1139+
*
1140+
* Waits until there is available request quota available based on the rate limit.
1141+
*
1142+
* @access private
1143+
*/
1144+
private function waitUntilRequestQuotaAvailable()
1145+
{
1146+
$sleep_until = $this->currentStartTime + $this->intervalSeconds;
1147+
$sleep_until_relative = $sleep_until - $this->currentStartTime;
1148+
$sleep_seconds = $sleep_until - microtime(true);
1149+
1150+
// Avoid using time_sleep_until() as it appears to be less precise and not sleep long enough.
1151+
usleep($sleep_seconds * 1000000);
1152+
1153+
$this->currentStartTime = microtime(true);
1154+
$this->currentRequestCount = 0;
1155+
}
10391156
}

‎tests/PHPCurlClass/Helper.php

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,16 @@ public function chainRequests($first, $second, $data = array())
5050
$this->chainedRequest($first, $data);
5151
$this->chainedRequest($second, $data);
5252
}
53+
54+
public static function getTestUrl($port)
55+
{
56+
if (getenv('PHP_CURL_CLASS_LOCAL_TEST') === 'yes' ||
57+
in_array(getenv('TRAVIS_PHP_VERSION'), array('7.0', '7.1', '7.2', '7.3', '7.4', 'nightly'))) {
58+
return 'http://127.0.0.1:' . $port . '/';
59+
} else {
60+
return self::TEST_URL;
61+
}
62+
}
5363
}
5464

5565
function create_png()
@@ -127,3 +137,11 @@ function remove_file_from_server($uploaded_file_path) {
127137
)));
128138
assert(file_exists($uploaded_file_path) === false);
129139
}
140+
141+
function get_multi_curl_property_value($instance, $property_name)
142+
{
143+
$reflector = new \ReflectionClass('\Curl\MultiCurl');
144+
$property = $reflector->getProperty($property_name);
145+
$property->setAccessible(true);
146+
return $property->getValue($instance);
147+
}

0 commit comments

Comments
 (0)