简单python爬取微博并写入excel

​ 写了一个简单的Python爬取指定用户微博的内容和图片，目前比较简陋，之前有在github上参考别人写的爬虫，发现现在微博好像使用的是Ajax的方式来渲染数据，这也太方便了，直接请求接口，然后解析数据不就能得到我们想要的数据了吗？？？？

​ ok，开始操作

首先，我们进入微博，打开检查，观察数据，这里我用的方法比较蠢，是一个个看返回内容是什么。。。等后面学习到了更高深的技术了在和大家分享吧，

经过手动查看，发现mymblog接口是获取内容的接口，然后操作，我们使用请求数据的第三方库是requests，先将请求头和获取内容的地址还有获取信息的地址定义好

    def __init__(self):
        self.headers = {
            'cookie': '',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
            'referer': 'https://weibo.com/u/3214393887'
        }
        self.user_info_url = 'https://weibo.com/ajax/profile/info?uid=3214393887'
        self.content_url = 'https://weibo.com/ajax/statuses/mymblog?uid=5541182601&page={}&feature=0'
        self.session = requests.session()
        self.user = User()
        self.contents = []

这里的请求头可以在浏览器中cv，不再赘述,

这里是我保存微博内容的对象，

class Weibo:
    def __init__(self):
        self.id = ''
        self.user_id = ''
        self.screen_name = ''

        self.content = ''
        self.article_url = ''

        self.original_pictures = []
        self.retweet_pictures = None
        self.original = None
        self.video_url = ''

        self.publish_place = ''
        self.publish_time = ''
        self.publish_tool = ''

        self.up_num = 0
        self.retweet_num = 0
        self.comment_num = 0

    def __str__(self):
        """打印一条微博"""
        result = self.content + '\n'
        result += u'微博发布位置：%s\n' % self.publish_place
        result += u'发布时间：%s\n' % self.publish_time
        result += u'发布工具：%s\n' % self.publish_tool
        result += u'点赞数：%d\n' % self.up_num
        result += u'转发数：%d\n' % self.retweet_num
        result += u'评论数：%d\n' % self.comment_num
        # result += u'url：https://weibo.cn/comment/%s\n' % self.id
        return result

解析数据的接口没啥好讲的，就是对症下药。。。

    def get_data(self, since_id,page):
        contentInfo = self.session.get(url=(self.content_url + '&since_id=' + since_id).format(page), headers=self.headers).content.decode()
        resultDict = json.loads(contentInfo)
        dataDict = resultDict['data']
        dataList = dataDict['list']
        index = 0
        for data in dataList:
            index += 1
            wb = Weibo()
            wb.id = data['id']
            wb.user_id = data['user']['id']
            wb.content = data['text_raw']
            wb.publish_time = data['created_at']
            wb.publish_tool = data['source']
            wb.screen_name = data['user']['screen_name']
            # wb.article_url = data['']
            pic_ids = data['pic_ids']
            try:
                pic_infos = data['pic_infos']
                for id in pic_ids:
                    info = pic_infos[id]
                    wb.original_pictures.append(info['original']['url'])
            except:
                print('没有图片')
            #获取图片url

            self.contents.append(wb)

        if 'since_id' in dataDict:
            page += 1
            return dataDict['since_id'],page
        else:
            return ''

额，这里其实是有个问题的，这个微博的内容好像是下拉触发换页的，然后返回内容里面有一个since_id字段应该是用来判断是否还有下一页的，然后你在发请求的时候需要指定page参数的值所以，所以需要+1,

数据下来后，可以保存到Excel或者数据库，这里为了直观是存入Excel的，目前还有蛮多功能没有实现，比如视频下载，评论拉取和点赞数，后面慢慢完善吧

    def save_as_excel(self):
        wb =  openpyxl.Workbook()
        sheet = wb.get_active_sheet()
        headers = ['时间', '发布工具', '内容', '图片', '发布地点', '点赞数', '评论']
        index = 0
        for header in headers:
            index += 1
            sheet.cell(row=1, column=index).value = header

        row = 1
        for weibo in self.contents:
            row += 1
            sheet.cell(row=row, column=1).value =  weibo.publish_time
            sheet.cell(row=row, column=2).value =  weibo.publish_tool
            sheet.cell(row=row, column=3).value =  weibo.content
            pics = ''
            for pic in  weibo.original_pictures:
                pics += pic + '\n'
            sheet.cell(row=row, column=4).value  = pics
            sheet.cell(row=row, column=5).value =  weibo.publish_place
            sheet.cell(row=row, column=6).value =  ''
            sheet.cell(row=row, column=7).value =  ''

        wb.save('小七.xlsx')

下载图片,根据前面获取到的链接下载图片，还是使用reuquest模块进行下载

    def down_load_pic(self):
        for content in self.contents:
            text = content.content
            imgs  = content.original_pictures
            index = 0
            for img in imgs:
                index += 1
                filename =  hashlib.md5((text+str(index)).encode()).hexdigest() + '.jpg'
                dowload = self.session.get(url=img, headers=self.headers)
                with open('./imgs/'+filename, 'wb') as f:
                    print('./imgs/'+filename)
                    f.write(dowload.content)

最后，可以继续完善

import requests
import lxml
from user import User
import json
from  weibo import Weibo
import  hashlib
import openpyxl
import time

class WeiBo(object):
    def __init__(self, url):
        self.url = url
        self.headers = {
            'cookie': '',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
            'referer': 'https://weibo.com/u/3214393887'
        }
        self.user_info_url = 'https://weibo.com/ajax/profile/info?uid=5541182601'
        self.content_url = 'https://weibo.com/ajax/statuses/mymblog?uid=5541182601&page={}&feature=0'
        self.session = requests.session()
        self.user = User()
        self.contents = []

    def get_data(self, since_id,page):
        contentInfo = self.session.get(url=(self.content_url + '&since_id=' + since_id).format(page), headers=self.headers).content.decode()
        resultDict = json.loads(contentInfo)
        dataDict = resultDict['data']
        dataList = dataDict['list']
        index = 0
        for data in dataList:
            index += 1
            wb = Weibo()
            wb.id = data['id']
            wb.user_id = data['user']['id']
            wb.content = data['text_raw']
            wb.publish_time = data['created_at']
            wb.publish_tool = data['source']
            wb.screen_name = data['user']['screen_name']
            # wb.article_url = data['']
            pic_ids = data['pic_ids']
            try:
                pic_infos = data['pic_infos']
                for id in pic_ids:
                    info = pic_infos[id]
                    wb.original_pictures.append(info['original']['url'])
            except:
                print('没有图片')
            #获取图片url

            self.contents.append(wb)

        if 'since_id' in dataDict:
            page += 1
            return dataDict['since_id'],page
        else:
            return ''

    def save_as_excel(self):
        wb =  openpyxl.Workbook()
        sheet = wb.get_active_sheet()
        headers = ['时间', '发布工具', '内容', '图片', '发布地点', '点赞数', '评论']
        index = 0
        for header in headers:
            index += 1
            sheet.cell(row=1, column=index).value = header

        row = 1
        for weibo in self.contents:
            row += 1
            sheet.cell(row=row, column=1).value =  weibo.publish_time
            sheet.cell(row=row, column=2).value =  weibo.publish_tool
            sheet.cell(row=row, column=3).value =  weibo.content
            pics = ''
            for pic in  weibo.original_pictures:
                pics += pic + '\n'
            sheet.cell(row=row, column=4).value  = pics
            sheet.cell(row=row, column=5).value =  weibo.publish_place
            sheet.cell(row=row, column=6).value =  ''
            sheet.cell(row=row, column=7).value =  ''

        wb.save('小七.xlsx')






    def get_user_info(self):
        userInfo = self.session.get(url=self.user_info_url, headers=self.headers).content.decode()
        resultDict = json.loads(userInfo)
        dataDict = resultDict['data']
        userDict = dataDict['user']

        self.user.id = userDict['id']
        self.user.nickname = userDict['screen_name']
        self.user.avator_img = userDict['profile_image_url'] + userDict['profile_url']
        self.user.gender = '女' if userDict['gender'] == 'f' else '男'
        self.user.description = userDict['description']
        self.user.location = userDict['location']
        self.user.following = userDict['followers_count']
        self.user.followers = userDict['friends_count']
        self.user.weibo_num = userDict['statuses_count']
    def down_load_pic(self):
        for content in self.contents:
            text = content.content
            imgs  = content.original_pictures
            index = 0
            for img in imgs:
                index += 1
                filename =  hashlib.md5((text+str(index)).encode()).hexdigest() + '.jpg'
                dowload = self.session.get(url=img, headers=self.headers)
                with open('./imgs/'+filename, 'wb') as f:
                    print('./imgs/'+filename)
                    f.write(dowload.content)

    def parse_data(self):
        #解析内容
        """时间 发布工具 内容 图片 发布地点 点赞数 评论"""
    def run(self):
        # 获取用户信息
        self.get_user_info()
        # print(self.user)

        # 获取微博内容
        since_id,page = self.get_data('',1)
        while since_id != '' and page < 5:
            since_id,page = self.get_data(since_id,page)
        # print(self.contents)
        self.save_as_excel()
        #下载图片
        # self.down_load_pic()

if __name__ == '__main__':
    heyi = WeiBo('')
    heyi.run()