from requests.models import Response
import requests
import random
import time
class WebRequest(object):
def __init__(self, *args, **kwargs):
pass
@property
def user_agent(self):
"""
return an User-Agent at random
:return:
"""
ua_list = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
]
return random.choice(ua_list)
@property
def header(self):
"""
basic header
:return:
"""
return {'User-Agent': self.user_agent,
'Accept': '*/*',
'Connection': 'keep-alive',
'Accept-Language': 'zh-CN,zh;q=0.8'}
def get(self, url, header=None, retry_time=5, timeout=30,
retry_interval=5, *args, **kwargs):
"""
get method
:param url: target url
:param header: headers
:param retry_time: retry time when network error
:param timeout: network timeout
:param retry_flag: if retry_flag in content. do retry
:param retry_interval: retry interval(second)
:param args:
:param kwargs:
:return:
"""
headers = self.header
if header and isinstance(header, dict):
headers.update(header)
while True:
try:
html = requests.get(url, headers=headers, timeout=timeout, **kwargs)
return html
except Exception as e:
print(e)
retry_time -= 1
if retry_time <= 0:
# 多次请求失败
resp = Response()
resp.status_code = 200
return resp
time.sleep(retry_interval)
HTTPConnectionPool(host:XX)Max retries exceeded with url:
爬虫多次访问同一个网站一段时间后会出现错误 HTTPConnectionPool(host:XX)Max retries exceeded with url '<requests.packages.urllib3.connection.HTTPConnection object at XXXX>: Failed to establish a new connection: [Errno 99] Cannot assign requested address'
是因为在每次数据传输前客户端要和服务器建立TCP连接,为节省传输消耗,默认为keep-alive,即连接一次,传输多次,然而在多次访问后不能结束并回到连接池中,导致不能产生新的连接
headers中的Connection默认为keep-alive,
将header中的Connection一项置为close
是因为在每次数据传输前客户端要和服务器建立TCP连接,为节省传输消耗,默认为keep-alive,即连接一次,传输多次,然而在多次访问后不能结束并回到连接池中,导致不能产生新的连接
headers中的Connection默认为keep-alive,
将header中的Connection一项置为close
headers = {
'Connection': 'close',
}
ok
ok