python爬取网易云热评

在爬取了妹子图之后，感觉总想再爬点什么东西增加编程兴趣，因为平时喜欢听歌嘛，然后就想到了爬取网易云热评，一首歌无数个故事23333…

首先打开网易云音乐网页版，找到自己想要爬取的那首歌（我这里搜的是隔壁老樊的-我曾）

打开检查（我这里使用的是谷歌浏览器），发现网易云的评论在 （R_SO_4_1336856777?csrf_token=） 里面，如下图所示，comments(评论)和hotComments(热门评论)，我们这里要爬取的是热门评论

找到歌曲url

找到目标文件的url

我们开始写代码，完整代码如下：

import requests
import json

def get_url(url):
    name_id = url.split('=')[1]             # url字符串以:分割，:左边是[0],右边是[1]
    headers = {
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
        'referer':'https://music.163.com/song?id=1336856777'
        }
        # 表单
    params = "lWLwYJlLNz2IDzMNv2dNDppn+/3ogEeb3XzeDHaUWlx69bYP7DXCrcAM05cwFhGaeJn0Etv+KB2/iAfPt2t8Ao9IJc/Xc/e9FqOZxZNArV/YqKfLASIlddvfem6zSz46ObQ60873SzQH08n2CLC9V13do8+zfumljxFseyvEevx4n2DjTvCCVTxQMUHZlKc6"
    encSecKey = "d759a811702000683476ab39777480509717372ceb62f41d6051dc6472fe5bb4842941a6c09cdc17b261c30973c13072a5e8eec705b8a68495206983e6a601faf67438b60e54bbc713a8dd4df964eae8ba11fc7c68cc416a811aed65f8f77245f3b943616116bbd90f6f0ff7ae543d63e773f00c0464022b7cb828bc286529ea"
    data = {
        "params":params,
        "encSecKey":encSecKey
    }

    # 使用.format格式化字符串，更灵活，更方便的爬取目标url
    target_url = "https://music.163.com/api/v1/resource/comments/R_SO_4_{}?csrf_token=".format(name_id)

    res = requests.get(target_url, headers=headers, data = data)
    return res

def get_hot_comments(res):
    comments_json = json.loads(res.text)
    hot_comments = comments_json['hotComments']
    with open("热评.txt", 'w', encoding='utf-8') as f:
        for each in hot_comments:
            f.write(each['user']['nickname'] + ':\n')
            f.write(each['content'] + '\n')
            f.write("----------------------------------\n")

def main():
    url = "https://music.163.com/#/song?id=1336856777"
    res = get_url(url)
    get_hot_comments(res)
    

if __name__ == "__main__":
    main()

为了避免网站反爬，加入了头并提交表单：

headers = {
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
        'referer':'https://music.163.com/song?id=1336856777'
        }
        # 表单
    params = "lWLwYJlLNz2IDzMNv2dNDppn+/3ogEeb3XzeDHaUWlx69bYP7DXCrcAM05cwFhGaeJn0Etv+KB2/iAfPt2t8Ao9IJc/Xc/e9FqOZxZNArV/YqKfLASIlddvfem6zSz46ObQ60873SzQH08n2CLC9V13do8+zfumljxFseyvEevx4n2DjTvCCVTxQMUHZlKc6"
    encSecKey = "d759a811702000683476ab39777480509717372ceb62f41d6051dc6472fe5bb4842941a6c09cdc17b261c30973c13072a5e8eec705b8a68495206983e6a601faf67438b60e54bbc713a8dd4df964eae8ba11fc7c68cc416a811aed65f8f77245f3b943616116bbd90f6f0ff7ae543d63e773f00c0464022b7cb828bc286529ea"
    data = {
        "params":params,
        "encSecKey":encSecKey
    }

    # 使用.format格式化字符串，更灵活，更方便的爬取目标url
    target_url = "https://music.163.com/api/v1/resource/comments/R_SO_4_{}?csrf_token=".format(name_id)

    res = requests.get(target_url, headers=headers, data = data)

用到的库：

import requests
import json

ps：不足之处还请广大读者指出
ps：python小白的一次爬虫记录