1、检查 robots.txt
import urllib.robotparser as rparser
def isCheckRobots(user_agent,robots_url, url):
'''检查robots.txt'''
rp = rparser.RobotFileParser()
rp.set_url(robots_url)
rp.read()
return rp.can_fetch("*", url)
2、设置用户代理
其实就是重新设置一下 user_agent
3、重试下载
服务器返回码为 4xx 时说明 请求 存在错误,
而返回码为 5xx 时说明服务端存在问题,我们只要确保返回码在 5xx 时重试下载
def getHtmlText(url, encoding = 'utf-8', user_agent = 'Mozilla/5.0', new_time_out = 3, num_retries = 2, isNeedCheckRobots = False, robots_url = None):
'''获取信息'''
if isNeedCheckRobots:
if isCheckRobots(user_agent, robots_url, url) is False:
return 'robots.txt 不允许'
try:
headers = {'user_agent' : user_agent}
r = requests.get(url, timeout = new_time_out, headers = headers)
r.encoding = encoding
r.raise_for_status()
return r.text
except:
# 服务器返回码为 5xx 时,则说明错误发生在服务器端,以下代码支持重试下载
if r.status_code >= 500 and r.status_code <= 600 and num_retries > 0:
return getHtmlText(url, encoding, new_time_out, num_retries - 1)
return '下载网页失败'
4、下载限速
在两次相同域名的 url 请求下加入延迟,以免爬取网站的速度过快被服务器封禁
import time
import datetime
from urllib import parse
class Throttle():
'''在同个域名请求加入延时'''
def __init__(self, delay):
# 延时时间
self.delay = delay
# 最后一次访问域名的时间戳
self.domain = {}
def wait(self, url):
domain = parse.urlparse(url).netloc
last_visit = self.domain.get(domain)
if self.delay > 0 and last_visit is not None:
sleep_secs = self.delay - (datetime.datetime.now() - last_visit).seconds
if sleep_secs > 0:
time.sleep(sleep_secs)
self.domain[domain] = datetime.datetime.now()