反爬
- python访问被拒绝,要使用浏览器访问。
解决方法:修改表头(加入User-Agent,可以是多个随机) :
url = '....'
data = {...} # 表头的最后一项
head = {}
data = urllib.parse.urlencode(data).encode('utf-8')
# 浏览器开发者模式中的network中表头(header)的post请求中User-Agent添加到python请求
# 方法一
head['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58'
res = urllib.request.Request(url, data, head)
response = urllib.request.urlopen(res)
# 方法二(在请求之后)
req = urllib.request.Request(url, data)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58')
response = urllib.request.urlopen(req)
# 有时还要加入:
# head['Cookie'](如:='OUTFOX_SEARCH_USER_ID=-1764369496@10.108.160.18;')
# head['Referer'](如:='http://fanyi.youdao.com/')
- ip访问太频繁(服务器发送验证码拒绝)
解决方法:
方法一:使用time模块,time.sleep(n)休眠一段时间 放在循环中模拟真人(不建议,太慢了)
方法二:使用代理(服务器看到的是代理的IP)(网上搜代理IP)
# 1.proxy_support = urllib.request.ProxyHandler({'类型':'ip:端口号'})
# 2.opener = urllib.request.build_opener(proxy_support)
# 3a.urllib.request.install_opener(opener) 安装opener,一劳永逸
# 3b.opener.open(url)使用特殊opener打开网页
import urllib.request
import random # 可以多弄几个随机访问
url = 'https://www.csdn.net/' # 查询IP的网站,用来测试
iplist =['163.125.221.229:8118', '183.166.20.6:9999', '125.108.80.240:9000', '125.108.123.95:9000']
proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)})
opener = urllib.request.build_opener(proxy_support)
opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58')]
urllib.request.install_opener(opener)
response = urllib.request.urlopen(url)
# 若出现utf-8无法解码的,在后面加上ignore即可(会出现乱码)
html = response.read().decode('utf-8') # 开发者模式看charset
print(html)
爬取有道翻译(老版url = ‘http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule’)
import urllib.request
import urllib.parse # 解析
import json
# translate_o?删除了_o 新版本有防爬,具体解决方法参考:https://tendcode.com/article/youdao-spider/
url = 'http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'
content = input('输入翻译内容:')
# 防止反爬 修改headers(表头)方法一
head = {}
head['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58'
data = {'i': content,
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'salt': '15972989870051',
'sign': '26393ce3bf4fc21109f3d258493f0ac1',
'lts': '1597298987005',
'bv': 'a612219d8ae465584a02998c7f4cede1',
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_REALTlME'}
# 解码,然后将python默认的Unicode编码成浏览器支持的utf-8
data = urllib.parse.urlencode(data).encode('utf-8')
res = urllib.request.Request(url, data, head)
response = urllib.request.urlopen(res)
# 防止反爬 修改headers(表头)方法二(在请求之后)
'''
req = urllib.request.Request(url, data)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58')
response = urllib.request.urlopen(req)
'''
html = response.read().decode('utf-8') # 反解码
# print(html):'{"type":"EN2ZH_CN","errorCode":0,"elapsedTime":1,
# "translateResult":[[{"src":"i love you","tgt":"我爱你"}]]}'
target = json.loads(html) # 将json字符串转换位python的字典(去掉引号)
print('翻译结果:%s'%target['translateResult'][0][0]['tgt'])
参考后(https://tendcode.com/article/youdao-spider/):
爬取新版(url = ‘http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule’)
import urllib.request
import hashlib
import time
import json
import random
class Youdao(object):
def __init__(self, msg):
self.msg = msg
self.url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'
self.D = "]BjuETDhU)zqSxf-=B#7m"
self.salt = self.get_salt()
self.sign = self.get_sign()
def get_md(self, value):
'''md5加密'''
m = hashlib.md5()
# m.update(value)
m.update(value.encode('utf-8'))
return m.hexdigest()
def get_salt(self):
'''根据当前时间戳获取salt参数'''
s = int(time.time() * 1000) + random.randint(0, 10)
return str(s)
def get_sign(self):
'''使用md5函数和其他参数,得到sign参数'''
s = "fanyideskweb" + self.msg + self.salt + self.D
return self.get_md(s)
def get_result(self):
headers={}
headers['Cookie']='OUTFOX_SEARCH_USER_ID=-1764369496@10.108.160.18;'
headers['Referer']='http://fanyi.youdao.com/'
headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58'
data = {
'i': self.msg,
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'salt': self.salt,
'sign': self.sign,
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_CL1CKBUTTON',
'typoResult': 'true'
}
# 解码,然后将python默认的Unicode编码成浏览器支持的utf-8
data = urllib.parse.urlencode(data).encode('utf-8')
res = urllib.request.Request(self.url, data, headers)
response = urllib.request.urlopen(res)
# 防止反爬 修改headers(表头)方法二(在请求之后)
'''
req = urllib.request.Request(url, data)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.58')
response = urllib.request.urlopen(req)
'''
html = response.read().decode('utf-8') # 反解码
# print(html):'{"type":"EN2ZH_CN","errorCode":0,"elapsedTime":1,
# "translateResult":[[{"src":"i love you","tgt":"我爱你"}]]}'
target = json.loads(html) # 将json字符串转换位python的字典(去掉引号)
print('翻译结果:%s'%target['translateResult'][0][0]['tgt'])
if __name__ == '__main__':
content = input('输入翻译内容:')
y = Youdao(content)
y.get_result()