Python爬取豆瓣短评

豆瓣短评.py

from multiprocessing import Process #多进程
import threading #多线程
from lxml import etree
import requests
import time
import random
import ciyun as cy





# //*[@id="comments"]/div[1]/div[2]/h3/span[2]/a

start=time.clock()

# print(etree.tostring(htmlelement, encoding="utf-8").decode("utf-8")

def paqu(id,startpage,endpage,save):
    j = 1
    with open("/Users/zhangpengxu/Desktop/爬取影评/paquresult/"+str(save)+".txt", 'w', encoding='utf-8') as f:
        for a in range(startpage,endpage):#爬取前两页
            for next in range(1,21):#每页20条评论
                url="https://movie.douban.com/subject/"+str(id)+"/comments?start="+str(20*a) #哪吒之魔童降世
                # htmlelement = etree.parse(url, parser=parser)
                result=requests.get(url).content   #使用content 使用text的时候经常解析不出来
                req = etree.HTML(result)  #使用xpath解析
                links1=req.xpath('//*[@id="comments"]/div['+str(next)+']/div[2]/h3/span[2]/a/text()')  #评论ID
                links2=req.xpath('//*[@id="comments"]/div['+str(next)+']/div[2]/p/span/text()')