基于python爬取有道翻译,并在线翻译
由于我也是爬虫新学者,有什么做的不对的请多加包涵
我们需要使用的库如下
from urllib import request
import urllib
import re
import random
爬虫新学者建议先使用urllib库学习,然后使用更简洁的resquests库
我的请求头有点不一样,你们可以改变一下
# 构造请求头信息,随机抽取信息
agent1 = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0'
agent2 = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1'
agent3 = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'
agent4 = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR ' \
'3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0) '
agent5 = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR ' \
'3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) '
list1 = [agent1, agent2, agent3, agent4, agent5]
agent = random.choice(list1)
# 打印请求头信息
print(agent)
header = {
'User-Agent': agent
}
好了,直接上代码
from urllib import request
import urllib
import re
import random
# 有道翻译
# 构造请求头信息,随机抽取信息
agent1 = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0'
agent2 = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1'
agent3 = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'
agent4 = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR ' \
'3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0) '
agent5 = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR ' \
'3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) '
list1 = [agent1, agent2, agent3, agent4, agent5]
agent = random.choice(list1)
# 打印请求头信息
print(agent)
header = {
'User-Agent': agent
}
url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule"
key = input("请输入要翻译的内容:")
# post请求需要提交的参数
# 请在网页HTML中查找
formdata = {
"i": key,
"from": "AUTO",
"to": "AUTO",
"smartresult": "dict",
"client": "fanyideskweb",
"salt": "15503049709404",
"sign": "3da914b136a37f75501f7f31b11e75fb",
"ts": "1550304970940",
"bv": "ab57a166e6a56368c9f95952de6192b5",
"doctype": "json",
"version": "2.1",
"keyfrom": "fanyi.web",
"action": "FY_BY_REALTIME",
"typoResult": "false"
}
data = urllib.parse.urlencode(formdata).encode(encoding='utf-8')
req = request.Request(url, data=data, headers=header)
resp = request.urlopen(req).read().decode()
# 正则表达式 提取"tgt":"和"}]]中间的任意内容
# 或者使用bs4库直接提取,本人更喜欢bs4,但是正则表达式也是要学的
pat = r'"tgt":"(.*?)"}]]'
result = re.findall(pat, resp)
print(result[0])
使用xpath可以更方便的找到正则信息
有什么不懂得或者您对这篇文章有什么指教的欢迎留言!
谢谢