python爬取爬虫技巧_Python爬虫基础 | 爬虫反ban的技巧

动态设置user agent

禁用cookies/启用cookie

设置延迟下载

使用Google cache (未记录)

使用IP地址池(Tor project、VPN和代理IP)

利用第三方平台crawlera做scrapy爬虫防屏蔽 (未记录)

动态设置user agent

# -*- coding:utf-8 -*-

import random

def get_headers():

useragent_list = [

'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.9.2.1000 Chrome/39.0.2146.0 Safari/537.36',

'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/532.3',

'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5',

'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36'

]

useragent = random.choice(useragent_list)

header = {'User-Agent': useragent}

return header

禁用cookies/启用cookie

Cookies

cookies(来自维基百科)因为HTTP协议是无状态的，即服务器不知道用户上一次做了什么，这严重阻碍了交互式Web应用程序的实现。在典型的网上购物场景中，用户浏览了几个页面，买了一盒饼干和两饮料。最后结帐时，由于HTTP的无状态性，不通过额外的手段，服务器并不知道用户到底买了什么。 所以Cookie就是用来绕开HTTP的无状态性的“额外手段”之一。服务器可以设置或读取Cookies中包含信息，借此维护用户跟服务器会话中的状态。

Cookie另一个典型的应用是当登录一个网站时，网站往往会请求用户输入用户名和密码，并且用户可以勾选“下次自动登录”。如果勾选了，那么下次访问同一网站时，用户会发现没输入用户名和密码就已经登录了。这正是因为前一次登录时，服务器发送了包含登录凭据(用户名加密码的某种加密形式)的Cookie到用户的硬盘上。第二次登录时，(如果该Cookie尚未到期)浏览器会发送该Cookie，服务器验证凭据，于是不必输入用户名和密码就让用户登录了。

采用selenium + PhantomJS 模拟浏览器登录Lagou，获取cookie

# -*- coding:utf-8 -*-

import sys

import time

import random

from selenium import webdriver

reload(sys)

sys.setdefaultencoding('utf-8')

def random_sleep_time():

sleeptime = random.randint(0, 10)

return time.sleep(sleeptime)

def get_headers_with_cookie():

driver = webdriver.PhantomJS(executable_path="D:\phantomjs-2.1.1-windows\\bin\phantomjs.exe") #需下载PhantomJS并解压到某一路径

url_login = 'https://passport.lagou.com/login/login.html'

driver.get(url_login)

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[1]/input').clear()

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[1]/input').send_keys('username') #需替换可用账户

random_sleep_time()

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[2]/input').clear()

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[2]/input').send_keys('password') #需替换可用账户

random_sleep_time()

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[5]/input').click()

random_sleep_time()

cookies = "; ".join([item["name"] + "=" + item["value"] for item in driver.get_cookies()])

headers = get_headers()

headers['cookie'] = cookies.encode('utf-8')

return headers

XPath解析 Copy XPath技巧 参考向右奔跑-009 - 使用XPath解析网页

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[1]/input')

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[2]/input')

driver.find_element_by_xpath('/html/body/section/div[1]/form/div/div[5]/input')

Copy XPath

Scrapy 禁用Cookie

在setting.py中设置

COOKIES_ENABLED=False

代理设置 PROXIES

在setting.py中设置

PROXIES = [

{'ip_port': '111.11.228.75:80', 'user_pass': ''},

{'ip_port': '120.198.243.22:80', 'user_pass': ''},

{'ip_port': '111.8.60.9:8123', 'user_pass': ''},

{'ip_port': '101.71.27.120:80', 'user_pass': ''},

{'ip_port': '122.96.59.104:80', 'user_pass': ''},

{'ip_port': '122.224.249.122:8088', 'user_pass': ''},

]

设置下载延迟

在setting.py中设置

DOWNLOAD_DELAY=3

创建中间件(middlewares.py)

import random

import base64

from settings import PROXIES

class RandomUserAgent(object):

"""Randomly rotate user agents based on a list of predefined ones"""

def __init__(self, agents):

self.agents = agents

@classmethod

def from_crawler(cls, crawler):

return cls(crawler.settings.getlist('USER_AGENTS'))

def process_request(self, request, spider):

# print "**************************" + random.choice(self.agents)

request.headers.setdefault('User-Agent', random.choice(self.agents))

class ProxyMiddleware(object):

def process_request(self, request, spider):

proxy = random.choice(PROXIES)

if proxy['user_pass'] is not None:

request.meta['proxy'] = "http://%s" % proxy['ip_port']

encoded_user_pass = base64.encodestring(proxy['user_pass'])

request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass

print "**************ProxyMiddleware have pass************" + proxy['ip_port']

else:

print "**************ProxyMiddleware no pass************" + proxy['ip_port']

request.meta['proxy'] = "http://%s" % proxy['ip_port']

设置下载中间件

DOWNLOADER_MIDDLEWARES = {

# 'myproject.middlewares.MyCustomDownloaderMiddleware': 543,

'myproject.middlewares.RandomUserAgent': 1,

'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 110,

# 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,

'myproject.middlewares.ProxyMiddleware': 100,

}

参考资料