如果爬虫爬取速度过快,很容易被反爬虫给禁掉IP,公司的网络,大部分的网络是一种动态分配的,对待这种情况 变化方案和设置IP代理,除了这点也要考虑网站门户的访问压力。
主要有效方案:
1、设置IP代理池。
2、adsl定时拨号()。
设置爬虫的 ip代理:
middlewares.py
from fake_useragent import UserAgent class ArticlespiderMiddleware(object): def __init__(self,crawler): super(ArticlespiderMiddleware, self).__init__() self.ua = UserAgent() self.ua_type = crawler.settings.get('RANDOM_UA_TPYE','random') @classmethod def from_crawler(cls,crawler): '''在初始化是调用''' return cls(crawler) def process_requests(self,request,spider): def get_ua(): return getattr(self.ua,self.ua_type) request.headers.setdefault('User-Agent',get_ua()) request.meta["proxy"] = "http://ip:Port"
上边只是简单的在中间件中设置了一个IP代理,但是我们需要的不是一个,所以,可以通过免费的IP代理来设置创建我们的Ip代理池:
#!/usr/bin/env python # -*- coding: utf-8 -*- __author__ = 'Fade Zhao' import requests from scrapy.selector import Selector from fake_useragent import UserAgent import MySQLdb class GetHost(object): def __init__(self): self.conn = MySQLdb.connect(host='localhost', user='root', password='zhaoyinghan', db='fiction', charset='utf8') self.cursor = self.conn.cursor() self.headers = {} def insert_IP(self): for i in range(1, 11): url = 'http://www.xicidaili.com/nn/{0}'.format(i) agent = UserAgent().random self.headers = { 'User-Agent': agent } session = requests.session() response = session.get(url=url, headers=self.headers) selector = Selector(text=response.text) all_trs = selector.css('#ip_list tr') ip_list = [] for tr in all_trs[1:]: speed_str = tr.css(".bar::attr(title)").extract()[0] speed = 1 if speed_str: speed = float(speed_str.split('秒')[0]) if speed > 0.3: continue td_text = tr.css("td::text").extract() ip = td_text[0] port = td_text[1] proxy_type = td_text[5] ip_list.append((ip, port, proxy_type, speed)) print(ip_list) sql_str = '''INSERT INTO proxy_ip(ip,port,proxy_type,speed) VALUES(%s,%s,%s,%s) on duplicate key update port=values(port)''' try: self.cursor.executemany(sql_str, ip_list) self.conn.commit() print(i, '插完了') except Exception as e: print('插入错误', e) self.conn.rollback() def judge_ip(self,ip,port): '''判断ip是否可用''' http_url = 'https://www.baidu.com' proxy_url = 'http://{0}:{1}'.format(ip,port) try: proxy_dict = { "http":proxy_url } response = requests.get(http_url,proxies=proxy_dict,headers=self.headers) except Exception as e: print('invalid ip and port =',e) self.delete_ip(ip) return False else: code= response.status_code if code >= 200 and code < 300: print('成功代理') return True else: return False def delete_ip(self,ip): '''删除废弃host''' sql_str ='''delete from proxy_ip WHERE ip="{0}"'''.format(ip) self.cursor.execute(sql_str) self.conn.commit() def get_IP(self): sql_str = '''select ip,port from proxy_ip ORDER BY RAND() limit 1''' self.cursor.execute(sql_str) host = self.cursor.fetchone() ip = host[0] port = host[1] print(host) judge = self.judge_ip(ip,port) if judge: return "http://{0}:{0}".format(ip,port) else: self.get_IP()
middlewares.py
from ArticleSpider.utls.crawl_xiciIP import GetHost class RandomProxyMiddleware(object): '''随机获取代理''' def process_requests(self,request,spider): host = GetHost() request.meta['proxy'] = host.get_IP()
以上就是IP代理池的设置和添加,但是我要说的是,GitHub上也已经有人给出了更多功能的IP代理吃的插件,正所谓是前任摘树,后人纳凉啊~
https://github.com/aivarsk/scrapy-proxies。