爬取电视剧的评论信息,并存储到本地。
# coding:utf-8
import urllib.request
import re
import http.cookiejar
import json
import time
cursor_id="6374452475944677749"
url="https://video.coral.qq.com/varticle/2451377986/comment/v2?callback=_varticle2451377986commentv2&orinum=10&oriorder=o&pageflag=1&cursor="
temp_headers = {"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Accept-Encoding":"utf-8","Accept-Language":"zh-CN,zh;q=0.8","User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}
cjar = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))
headers = []
for key,value in temp_headers.items():
item = (key,value)
headers.append(item)
opener.addheaders=headers
urllib.request.install_opener(opener)
def get_data(cursor_id):
url_new = url+cursor_id
# 获得的原始数据,为json格式
data = urllib.request.urlopen(url_new).read().decode("utf-8")
#对获得的原始数据进行格式化处理,转换为标准的json格式
temp_data = data.replace('_varticle2451377986commentv2(','').replace(')',"")
#将接送格式的数据转化为python字典格式
temp = json.loads(temp_data,encoding="utf-8")
# print(temp)
# print(temp['data']['userList'])
# print(len(temp['data']['userList']))
# print(type(temp))
# # print(len(temp))
# # for i in temp:
# # print(i)
# for i in temp['data']:
# print(i)
#提取需要的数据
id_first = temp['data']['first'] #第一个id
id_last = temp['data']['last'] #最后一个id
#每一个评论的数据列表
new_comment_data_list=[]
data_list = temp['data']['oriCommList']
for data in data_list:
# print(data)
comment_data = data['content']
new_comment_data_list.append(comment_data)
print(new_comment_data_list)
# 获取评论者姓名
nick_name_list = temp['data']['userList']
# print(nick_name_list)
new_nick_name_list = []
for k,v in nick_name_list.items():
nick_name = v['nick']
new_nick_name_list.append(nick_name)
print(new_nick_name_list)
print("++++++++++++++++++++++++++++++++++++++++++++")
print(id_last)
for i in range(len(new_comment_data_list)):
with open('comment.txt','a+',encoding="utf-8") as fr:
fr.write(new_comment_data_list[i])
fr.write('\n')
return id_last
# 获取数据
while True:
try:
cursor_id = get_data(cursor_id)
time.sleep(1)
except:
print("信息有误")