腾讯视频短评爬取--斗罗大陆

########################### 斗罗大陆短评爬取 #################################

import urllib.request
import re

cursor = "6393416078681683802"  # 爬虫开始爬取的第一条评论的编号
orinum = "10"                   # 每页十条评论
orirepnum = "0"                 # 每条评论下的回复数（同样为评论）
vid = "2369303789"              # 斗罗大陆的编号

# 通过Fiddler抓取JS文件，并提取其评论的URL，如下所示：
# https://video.coral.qq.com/varticle/1743283224/comment/v2?callback=_varticle1743283224commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6360110638996097228&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=132&_=1552440991386
# https://video.coral.qq.com/varticle/1743283224/comment/v2?callback=_varticle1743283224commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6411116569243258049&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=132&_=1552440991387

# 总结url的规律
url = "https://video.coral.qq.com/varticle/" + vid + "/comment/v2?orinum=" + orinum + "&oriorder=o&pageflag=1&cursor=" + cursor + "&orirepnum=" + orirepnum

# 模拟浏览器，并将报头安装为全局
headers = {
           'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
           "Content-Type":"application/javascript"
           }
headall =[]
for k,v in headers.items():
    item = (k,v)
    headall.append(item)
opener = urllib.request.build_opener()
opener.addheaders = headall
urllib.request.install_opener(opener)

f=open("D:\\Py_Projects\\腾讯视频（短评）评论爬取-斗罗大陆.txt","w+",encoding="utf-8",errors='ignore')
# 爬取20页的评论
for j in range(0,20):
    print("第%d页" % (j+1))
    thispage = urllib.request.urlopen(url).read().decode("utf-8")
    comentpat = '"content":"(.*?)"'
    lastpat ='"last":"(.*?)"'
    commentall = re.compile(comentpat).findall(thispage)
    last = re.compile(lastpat,re.S).findall(thispage)[0]
    url =  "https://video.coral.qq.com/varticle/" + vid + "/comment/v2?orinum=" + orinum + "&oriorder=o&pageflag=1&cursor=" + last + "&orirepnum=" + orirepnum
    try:
        for i in range(0,len(commentall)):
            print("第"+str(j+1)+"-"+str(i+1)+"条评论内容为：")
            print(eval('u"' + commentall[i]+'"'))
            print(eval('u"' + commentall[i]+'"'),file = f)
    except Exception as e:
        print(e)
f.close()