本文的主要目标是获取微博评论数据,具体包括微博评论链接、总评论数、用户ID、用户昵称、评论时间、评论内容、用户详情链接等。
实现代码如下所示:
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 8 16:39:07 2017
@author: Administrator
"""
import requests
from bs4 import BeautifulSoup
import json
import time
import random
import pymysql.cursors
def crawlDetailPage(url,page,i):
#读取微博网页的JSON信息
req = requests.get(url)
jsondata = req.text
data = json.loads(jsondata)
#评论数
commentCounts = data['total_number']
print("第{}页第{}条微博的评论数为{}".format(page,i+1,commentCounts))
#循环输出每一页的微博信息
for comment in data['data']:
userId = comment['user']['id']
userName = comment['user']['screen_name']
commentTime = comment['created_at']
commentText = comment['text']
userProfileUrl = comment['user']['profile_url']
print("用户{}创建于:{}".format(userName,commentTime))
print("评论内容为:{}".format(commentText))
print("用户详情链接为:{}".format(userProfileUrl))
'''
数据库操作
'''
#获取数据库链接
connection = pymysql.connect(host = 'localhost',
user = 'root',
password = '123456',
db = 'weibo',
charset = 'utf8mb4')
try:
#获取会话指针
with connection.cursor() as cursor:
#创建sql语句
sql = "insert into `comment` (`commentUrl`,`commentCounts`,`userId`,`userName`,`commentTime`,`commentText`,`userProfileUrl`) values (%s,%s,%s,%s,%s,%s,%s)"
#执行sql语句
cursor.execute(sql,(url,commentCounts,userId,userName,commentTime,commentText,userProfileUrl))
#提交数据库
connection.commit()
finally:
connection.close()
def crawl(url,page):
#读取微博网页的JSON信息
req = requests.get(url)
jsondata = req.text
data = json.loads(jsondata)
#print(data['cards'][5]['scheme'])
#获取每一页的数据
content = data['cards']
#print(content[6]['scheme'])
#循环输出每一页微博的详情链接
for i in range(2,11):
contentId = content[i]['mblog']['id']
#contentUrl = "https://m.weibo.cn/status/" + contentId
commentUrl = "https://m.weibo.cn/api/comments/show?id=" +str(contentId)
#print("第{}条微博的详情链接为:{}".format(i+1,commentUrl))
crawlDetailPage(commentUrl,page,i)
t = random.randint(11,13)
print("休眠时间为:{}s".format(t))
time.sleep(t)
for i in range(1,2):
print("正在获取第{}页微博数据:".format(i))
#知乎官方微博数据的JSON链接
url = "https://m.weibo.cn/api/container/getIndex?uid=1939498534&type=uid&value=1939498534&containerid=1076031939498534&page=" + str(i)
crawl(url,i)
#设置休眠时间
time.sleep(random.randint(31,33))
部分运行结果如下图所示:
MySQL数据库中的数据存储如下图所示: