php curlm ok,php限制curl并发数

class MultiCurl

{

private $allToDo;

private $multiHandle;

private $maxConcurrent = 2;

private $currentIndex = 0;

private $info = array();

private $options = array(CURLOPT_RETURNTRANSFER => true,

CURLOPT_FOLLOWLOCATION => true,

CURLOPT_MAXREDIRS => 3,

CURLOPT_TIMEOUT => 3);

public function __construct($todo, $concurrent)

{

$this->allToDo = $todo;

$this->maxConcurrent = $concurrent;

$this->multiHandle = curl_multi_init();

}

private function _addHandles($num)

{

while ($num-- > 0) {

$handle = curl_init($this->allToDo[$this->currentIndex]);

curl_setopt_array($handle, $this->options);

curl_multi_add_handle($this->multiHandle, $handle);

$this->info[$this->currentIndex] = $handle;

$this->currentIndex++;

}

}

private function _moreToDo()

{

return count($this->allToDo) - $this->currentIndex;

}

public function process()

{

do{

$running = null;

$concurrent=min($this->maxConcurrent, $this->_moreToDo());

$this->_addHandles($concurrent);

// Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.

// https://bugs.php.net/bug.php?id=63411

do {

$mrc = curl_multi_exec($this->multiHandle, $running);

} while ($mrc == CURLM_CALL_MULTI_PERFORM);

while ($running && $mrc == CURLM_OK) {

if (curl_multi_select($this->multiHandle) == -1) {

usleep(100);

}

do {

$mrc = curl_multi_exec($this->multiHandle, $running);

} while ($mrc == CURLM_CALL_MULTI_PERFORM);

}

}while($this->_moreTodo());

foreach($this->info as $info){

$cur_info=curl_getinfo($info);

$content = curl_multi_getcontent($info);

echo $cur_info['url'] . ' - ' . strlen($content) . ' bytes
';

curl_multi_remove_handle($this->multiHandle, $info);

curl_close($info);

}

curl_multi_close($this->multiHandle);

//$this->clear();

return $this;

}

public function clear(){

$m=memory_get_usage(); //获取当前占用内存

echo "memory_usage:",$m,"
";

$this->multiHandle=null;

$this->info=null;

$m=memory_get_usage(); //获取当前占用内存

echo "after memory_usage:",$m,"
";

}

}

function microtime_float()

{

list($usec, $sec) = explode(" ", microtime());

return ((float)$usec + (float)$sec);

}

$sites = [

'http://localhost:88/2018/',

'http://localhost:88/2018/1.php?v=1',

'http://localhost:88/2018/1.php?v=2',

'http://localhost:88/2018/1.php?v=3',

'http://localhost:88/2018/1.php?v=4',

'http://localhost:88/2018/1.php?v=5',

'http://localhost:88/2018/1.php?v=6',

'http://localhost:88/2018/1.php?v=7',

'http://localhost:88/2018/1.php?v=8',

'http://localhost:88/2018/1.php?v=9',

'http://localhost:88/2018/1.php?v=10',

];

$concurrent = 3; // Any number.

$time_start=microtime_float();

$mc = new MultiCurl($sites, $concurrent);

$mc->process();

$time_end=microtime_float();

echo $time_end - $time_start;

1.php

sleep(2);

以下为使用guzzle：

use GuzzleHttp\Client;

use GuzzleHttp\Pool;

use GuzzleHttp\Promise;

require_once 'vendor/autoload.php';

$client=new Client([

'timeout'=>4

]);

function microtime_float()

{

list($usec, $sec) = explode(" ", microtime());

return ((float)$usec + (float)$sec);

}

$urls = [

'http://localhost:88/2018/',

'http://localhost:88/2018/1.php?v=1',

'http://localhost:88/2018/1.php?v=2',

'http://localhost:88/2018/1.php?v=3',

'http://localhost:88/2018/1.php?v=4',

'http://localhost:88/2018/1.php?v=5',

'http://localhost:88/2018/1.php?v=6',

'http://localhost:88/2018/1.php?v=7',

'http://localhost:88/2018/1.php?v=8',

'http://localhost:88/2018/1.php?v=9',

'http://localhost:88/2018/1.php?v=10',

];

$time_start=microtime_float();

$requests = function () use ($client,$urls) {

foreach ($urls as $uri) {

yield function() use ($client, $uri) {

return $client->getAsync($uri);

};

}

};

$pool = new Pool($client, $requests(),[

'concurrency' => 3,//default 25

'fulfilled' => function ($response, $index) {

// this is delivered each successful response

echo $response->getStatusCode().'
';

},

'rejected' => function ($reason, $index) {

// this is delivered each failed request

echo $reason."
";

},

]);

$promise = $pool->promise();

$promise->wait();

$time_end=microtime_float();

echo $time_end - $time_start;

相关：

python2:grequests

# coding=utf-8

import grequests

import time

start =time.clock()

urls = [

'http://localhost:88/2018/',

'http://localhost:88/2018/1.php?v=1',

'http://localhost:88/2018/1.php?v=2',

'http://localhost:88/2018/1.php?v=3',

'http://localhost:88/2018/1.php?v=4',

'http://localhost:88/2018/1.php?v=5',

'http://localhost:88/2018/1.php?v=6',

'http://localhost:88/2018/1.php?v=7',

'http://localhost:88/2018/1.php?v=8',

'http://localhost:88/2018/1.php?v=9',

'http://localhost:88/2018/1.php?v=10',

]

rs = (grequests.get(u,timeout=10) for u in urls)

response=grequests.map(rs)

end = time.clock()

print('Running time: %s Seconds'%(end-start))

print(u'请求完成..')

response0 = response[0]

#print response0.text

python2:requests+gevent

# coding=utf-8

import requests

import gevent

from gevent import monkey

monkey.patch_socket()

urls = ["http://www.baidu.com/"] *10

def get_content(url):

data = requests.get(url,timeout=10)

return data

#response = requests.get('http://baidu.com',timeout=10)

#print('请求完成..')

jobs = [gevent.spawn(get_content, url) for url in urls]

gevent.joinall(jobs)

print('请求完成..')

python3:requests-futures

import requests

import concurrent.futures

def get_urls():

return ["url1","url2"]

def load_url(url, timeout):

return requests.get(url, timeout = timeout)

with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:

future_to_url = {executor.submit(load_url, url, 10): url for url in get_urls()}

for future in concurrent.futures.as_completed(future_to_url):

url = future_to_url[future]

try:

data = future.result()

except Exception as exc:

resp_err = resp_err + 1

else:

resp_ok = resp_ok + 1