python批量请求url_python + pycurl + 高效批量get请求

pycurl包是用C编写的libcurl  python接口，速度高于urllib。Libcurl 是一个支持FTP, FTPS, HTTP, HTTPS, GOPHER, TELNET, DICT, FILE 和 LDAP的客户端URL传输库.libcurl也支持HTTPS认证,HTTP POST,HTTP PUT,FTP上传,代理,Cookies,基本身份验证,FTP文件断点继传,HTTP代理通道等等。

def doBatchGet(url_list):

"""批量get请求,

返回成功的请求list和失败的请求list

"""

mgr = pycurl.CurlMulti()

handles = []

for url in url_list:

c = pycurl.Curl()

c.url = url

c.body = StringIO.StringIO()

c.http_code = 1

c.setopt(c.URL, c.url)

c.setopt(c.WRITEFUNCTION, c.body.write)

mgr.add_handle(c)

handles.append(c)

handle_num = len(handles)

while handle_num:

while 1:

ret, handle_num = mgr.socket_all()

if ret != pycurl.E_CALL_MULTI_PERFORM:

break

mgr.select(0.1)

err_list, succ_list = [], []

for c in handles:

c.http_code = c.getinfo(c.HTTP_CODE)

if c.http_code != 200:

print 'get error: %s, %s' % (c.url, c.http_code)

err_list.append(c.url)

else:

res = c.body.getvalue()

if res: #添加成功条件

succ_list.append((c.url, res))

else:

err_list.append(c.url)

for c in handles:

mgr.remove_handle(c)

c.body.close()

c.close()

mgr.close()

return err_list, succ_list