在众多的网站防爬措施中,有一种是根据ip的访问频率进行限制,即在某一时间段内,当某个ip的访问次数达到一定的阀值时,该ip就会被拉黑、在一段时间内禁止访问。 搭建一个IP代理池,使用不同的IP轮流进行爬取。
获取模块
import requests
import chardetimport tracebackfrom lxml import etree
class Downloader(object): def __init__(self):
self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
} def download(self, url):
print('正在下载页面:{}'.format(url))
try:
resp = requests.get(url, headers=self.headers) resp.encoding = chardet.detect(resp.content)['encoding']
if resp.status_code == 200:
return self.xpath_parse(resp.text)
else:
raise ConnectionError
except Exception:
print('下载页面出错:{}'.format(url))
traceback.print_exc() def xpath_parse(self, resp):
try:
page = etree.HTML(resp) trs = page.xpath('//div[@id="list"]/table/tbody/tr')
proxy_list = [] for tr in trs:
ip = tr.xpath('./td[1]/text()')[0]
port = tr.xpath('./td[2]/text()')[0]
proxy = { 'proxy': ip + ':' + port
} proxy_list.append(proxy) return proxy_list
except Exception:
print('解析IP地址出错')
traceback.print_exc()if __name__ == '__main__':
print(Downloader().download('https://www.kuaidaili.com/free/inha/1/'))
存储模块
import pymongo
from pymongo.errors import DuplicateKeyErrorclass MongoDB(object): def __init__(self):
self.client = pymongo.MongoClient()
self.db = self.client['proxypool3']
self.proxies = self.db['proxies']
self.proxies.ensure_index('proxy', unique=True)
self.proxies.create_index()
# createIndex()
def insert(self, proxy):
try:
self.proxies.insert(proxy)
print('插入成功:{}'.format(proxy))
except DuplicateKeyError:
pass def delete(self, conditions):
self.proxies.remove(conditions) print('删除成功:{}'.format(conditions))
def update(self, conditions, values):
self.proxies.update(conditions, {"$set": values})
print('更新成功:{},{}'.format(conditions,values))
def get(self, count, conditions=None):
conditions = conditions if conditions else {}
count = int(count)
items = self.proxies.find(conditions, limit=count).sort('delay', pymongo.ASCENDING)
items = list(items)
return items
def get_count(self):
return self.proxies.count({})
if __name__ == '__main__':
m = MongoDB() print(m.get(3))
检测模块
import requests
import timeimport tracebackfrom requests.exceptions import ProxyError, ConnectionError
from db.mongo_db import MongoDB
from multiprocessing.pool import ThreadPool
def valid_many(proxy_list, method): pool = ThreadPool(16)
for proxy in proxy_list:
pool.apply_async(valid_one, args=(proxy, method)) pool.close()
pool.join()def valid_one(proxy, method, url='https://www.baidu.com'):
headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
} proxies = { 'http': 'http://' + proxy['proxy'],
'https': 'http://' + proxy['proxy']
} try: start_time = time.time() # requests.packages.urllib3.disable_warnings() resp = requests.get(url, headers=headers, proxies=proxies, timeout=5, verify=False)
delay = round(time.time() - start_time, 2)
if resp.status_code == 200:
proxy['delay'] = delay
if method == 'insert':
MongoDB().insert(proxy) elif method == 'check':
MongoDB().update({'proxy': proxy['proxy']}, {'delay': proxy['delay']})
else:
if method == 'check':
MongoDB().delete({'proxy': proxy['proxy']})
except (ProxyError, ConnectionError): if method == 'check':
MongoDB().delete({'proxy': proxy['proxy']})
except Exception: traceback.print_exc()
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
} proxies = { 'http': 'http://' + proxy['proxy'],
'https': 'http://' + proxy['proxy']
} try: start_time = time.time()
# requests.packages.urllib3.disable_warnings() resp = requests.get(url, headers=headers, proxies=proxies, timeout=5, verify=False)
delay = round(time.time() - start_time, 2)
if resp.status_code == 200:
proxy['delay'] = delay
if method == 'insert':
MongoDB().insert(proxy)
elif method == 'check':
MongoDB().update({'proxy': proxy['proxy']}, {'delay': proxy['delay']})
else:
if method == 'check':
MongoDB().delete({'proxy': proxy['proxy']})
except (ProxyError, ConnectionError): if method == 'check':
MongoDB().delete({'proxy': proxy['proxy']})
except Exception: traceback.print_exc()