使用python爬取猫眼电影、房王、股吧论坛、百度翻译、有道翻译、高德天气、华夏基金、扇贝单词、糗事百科（有道翻译）

最新推荐文章于 2024-10-01 20:02:09 发布

物喜己悲

最新推荐文章于 2024-10-01 20:02:09 发布

阅读量226

点赞数

分类专栏：爬虫文章标签：爬虫有道翻译

本文链接：https://blog.csdn.net/yu1860110/article/details/90581695

版权

爬虫专栏收录该内容

8 篇文章 0 订阅

订阅专栏

本文介绍了如何利用Python进行网络数据抓取，包括猫眼电影数据、房王信息、股吧论坛帖子、百度翻译、有道翻译、高德天气、华夏基金数据、扇贝单词以及糗事百科的内容。通过实例详细阐述了爬虫的实现过程，涉及到的主要库有requests、BeautifulSoup等，同时提到了有道翻译API的使用。

摘要由CSDN通过智能技术生成

import requests
import time
import random
import hashlib

def md5(value):
    # 创建MD5对象
    md5_obj = hashlib.md5()
    # 加密字符串
    md5_obj.update(bytes(value, encoding="utf-8"))
    # 进行16位的加密
    sign = md5_obj.hexdigest()
    return sign

def youdao(i):

    # 获取salt
    salt = str(int(time.time() * 1000)) + str(random.randint(0, 9))
    # print(salt)

    # 获取sign
    sign1 = "fanyideskweb" + i + salt + "@6f#X3=cCuncYssPsuRUE"
    sign = md5(sign1)

    # 定义data参数
    data = {
        "i": i,
        # "from": "AUTO",
        # "to": "AUTO",
        # "smartresult": "dict",
        "client": "fanyideskweb",
        "salt": salt,
        "sign": sign,
        # "ts": "1558514897639",
        # "bv": "cf156b581152bd0b259b90070b1120e6",
        # "doctype": "json",
        # "version": "2.1",
        "keyfrom": "fanyi.web",
        # "action": "FY_BY_REALTlME"
    }

    # 加上请求头 浏览器信息
    headers = {
        # "Accept": "application/json, text/javascript, */*; q=0.01",
        # "Accept-Encoding": "gzip, deflate",
        # "Accept-Language": "zh-CN,zh;q=0.9",
        # "Connection": "keep-alive",
        # "Content-Length": "238",
        # "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
        "Cookie": "OUTFOX_SEARCH_USER_ID=1007772075@10.168.8.76; OUTFOX_SEARCH_USER_ID_NCOO=1844201936.6123636; _ga=GA1.2.1939912746.1552966532; JSESSIONID=aaaB9UfpkFL02gnEynoRw; ___rl__test__cookies=1558514897636",
        # "Host": "fanyi.youdao.com",
        # "Origin": "http://fanyi.youdao.com",
        "Referer": "http://fanyi.youdao.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
        # "X-Requested-With": "XMLHttpRequest"
    }

    # 定义起始url
    base_url = "http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule"

    # 发送请求
    response = requests.post(base_url, data=data, headers=headers)
    # 获取response里面的json数据
    json_data = response.json()
    print(json_data)
    print(type(json_data))


if __name__ == '__main__':
    i = input("请输入需要翻译的内容：")
    # i = "banana"
    youdao(i)

"""
遇到的问题1：
""只携带参数data 发起请求的时候，请求不到数据，出现{'errorCode'"": ""50}，",",
此时的解决方案是：加上请求头浏览器信息 再次发起请求 
""问题2：还是获取不到信息 {'errorCode'"": ""50}",",
解决的方案是：把所有的请求头信息添加到headers中
"""

# i: banana
# client: fanyideskweb
# salt: 15585168560444
# sign: da50e3193cda496e1455ff28c1bb21b1
# keyfrom: fanyi.web
#
# i: apple
# "client": "fanyideskweb",
# "salt": "15585148976393",
# "sign": "147950af9758d1e79aeaacd4ff27d14d",
# "keyfrom": "fanyi.web",
#
#
# salt:  需要看一下 是否需要加密
# sign： 也要看一下是否需要加密

# 首要解决的问题是salt和sign生成的过程
'''
salt = o.salt = i = r + parseInt(10 * Math.random(), 10)
"" + (new Date).getTime() + parseInt(10 * Math.random(), 10)  js
= "" + int(time.time() * 1000) + random.randint(0, 9)

o = r.generateSaltSign(t) = r(t)
r.generateSaltSign(t) = t.generateSaltSign(t) = r(t)
{
    ts: r,
    bv: t,
    salt: i,
    sign: n.md5("fanyideskweb" + e + i + "@6f#X3=cCuncYssPsuRUE")
}


e = t = "apple" 需要翻译的内容

var r = function(e)
    {
            var
        t = n.md5(navigator.appVersion),
            r = "" + (new Date).getTime(),
                i = r + parseInt(10 * Math.random(), 10);
        return {
            ts: r,
            bv: t,
            salt: i,
            sign: n.md5("fanyideskweb" + e + i + "@6f#X3=cCuncYssPsuRUE")
    }
};


长度比较：
15585816225096   python
15585148976393   js
15585822104216


sign = o.sign
= n.md5("fanyideskweb" + e + i + "@6f#X3=cCuncYssPsuRUE")
= md5("fanyideskweb" + "apple" + salt + "@6f#X3=cCuncYssPsuRUE")
'''