python爬虫之爬取手机微博评论(图文并排,炒鸡详细!!!)

B站有视频讲解，有不动的地方，可以去看视频
一、手机微博爬虫（一）数据抓取
二、手机微博爬虫（二）解析、保存数据
三、手机微博爬虫（三）数据存到数据库
四、手机微博爬虫（四）词云制作

第一步:登录手机微博官网,截图如下所示:

第二步:分析提交的参数,找到任意一条微博内容,本文选取的内容是微博推送的,找到开发者模式,点开network选项卡,刷新页面,一直向下翻,找出规律,截图如下所示:

注意事项:就此条微博来说,前17页数据中提交的参数max_id_type = 0,但是17页以后就变了,改成了1,这点大家注意即可

第三步:分析数据藏匿的地方,截图如下所示:

第四步:写代码

import requests
import re
import os

class Main():
    url = 'https://m.weibo.cn/comments/hotflow?'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.9 Safari/537.36',
        'cookie': 'ALF=1586350518; _T_WM=18754506867; WEIBOCN_FROM=1110006030; MLOGIN=1; SCF=AhxeeyC_-eMbxpwb8KLVrN7-04xwJvc9ATC-09VkEqDz38nXmtqkQbVSt5VvmWZcNldfr8k_Nf8P4SEdVDS_E84.; SUB=_2A25zaEzqDeRhGeBP61oT8SvMwz2IHXVQk1SirDV6PUNbktANLWTikW1NRWvMdQ0ctsSre5zQ_DZmBR8BBLNsxlVv; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWNVTTW1HmFFlVFFRI_IKF55JpX5KMhUgL.FoqpehnEeK-71h22dJLoIE5LxKML12-L12zLxKBLB.2L1-2LxK-LBo.LBoeN1hzpSntt; SUHB=0gHiwhGI-JYGEg; SSOLoginState=1584151738; M_WEIBOCN_PARAMS=oid%3D4481983306764793%26luicode%3D20000061%26lfid%3D4481983306764793%26uicode%3D20000061%26fid%3D4481983306764793; XSRF-TOKEN=c1db22',
        'Referer': 'https://m.weibo.cn/detail/4481983306764793'
    }
    params = {}
    list_username = []
    list_text = []
    list_all = []
    file = './'

    def __init__(self):
        num_page = int(input('请输入您要爬取的微博内容的页数:\n'))
        ID_MID = input('请输入您要爬取的微博内容的ID:\n')
        return_info = ('0','0')
        for i in range(0,num_page):
            print(f'正在爬取第{i + 1}页数据')
            self.params = {
                'id': ID_MID,
                'mid': ID_MID,
                'max_id': return_info[0],
                'max_id_type': return_info[1]
            }
            return_info = self.get_max_id()
        self.parseData()
        self.writeData()

    def get_max_id(self):
        res = requests.get(url = self.url,headers = self.headers,params = self.params).json()['data']
        max_id = res['max_id']
        max_id_type = res['max_id_type']
        data = res['data']
        # print(data)
        for i in range(0,len(data)):
            text = data[i]['text']
            username = data[i]['user']['screen_name']
            self.list_text.append(text)
            self.list_username.append(username)
        return max_id,max_id_type

    def parseData(self):
        for i in range(0, len(self.list_text)):
            res = re.sub('''[ ,/,",',=,.,\],\[,\-,_,;,:,?,%,&,+]''', '', self.list_text[i])
            res = re.sub('<\w*>', '', res)
            res = re.sub('@\w*', '', res)
            self.list_text[i] = res
        self.list_all = list(zip(self.list_username, self.list_text))

    def writeData(self):
        for i in self.list_all:
            if not os.path.exists(self.file):
                os.mkdir(self.file)
            with open(f'{self.file}/weibocomments1.txt', 'a', encoding='utf-8') as fp:
                fp.write(i[0])
                fp.write(' ')
                fp.write(i[1])
                fp.write('\n')

# 4481983306764793
if __name__ == '__main__':
    Main()

第五步:爬取效果展示:

第六步:完结撒花