利用python爬虫程序爬取豆瓣影评

        获取至少两个页面的豆瓣影评，包括每一条影评的标题、评论，然后将多条影评信息存入CSV文件。

具体代码如下：

#doubanyingping.py
import requests
from bs4 import BeautifulSoup
import re
def getHTMLText(url):
    kv={'cookie':'viewed="26987890"; bid=LtZlXs-lxWE; __gads=ID=2f05664627755103-228fcabf2eca0085:T=1625664681:RT=1625664681:S=ALNI_MbErnNq9E-gQKSusKFmubZ33JrR1A; gr_user_id=23ab9cd5-12ea-402d-af1f-f82384037c1a; douban-fav-remind=1; ll="108291"; dbcl2="220543507:+d6RaThYFJg"; push_noty_num=0; push_doumail_num=0; _vwo_uuid_v2=D2BBDC590CDF4A6D94F22488FC0507F3C|f20d9349bb838c0ffda7ad25fbd40373; __utmz=30149280.1631793610.4.3.utmcsr=link.csdn.net|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmz=223695111.1631793610.2.2.utmcsr=link.csdn.net|utmccn=(referral)|utmcmd=referral|utmcct=/; ck=re1Y; __utma=30149280.516384608.1625664683.1632279401.1632391188.7; __utmb=30149280.0.10.1632391188; __utmc=30149280; __utma=223695111.1494505190.1631782526.1632279401.1632391188.5; __utmb=223695111.0.10.1632391188; __utmc=223695111; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1632391194%2C%22https%3A%2F%2Flink.csdn.net%2F%3Ftarget%3Dhttps%253A%252F%252Fmovie.douban.com%252Ftop250%253Fstart%253D0%2526filter%253D%22%5D; _pk_id.100001.4cf6=f45412f108713ba2.1631782532.5.1632391194.1632279427.; _pk_ses.100001.4cf6=*',
       'user-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36"}
    try:
        r = requests.get(url,headers=kv,timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillReviewList(rlist,html):
    soup = BeautifulSoup(html,"html.parser")
    for div in soup.find_all('div',{"class":"main-bd"}):
        rlist.append([div.find('a',{"href":re.compile("https:")}).text.strip("\n "),\
                     div.find('div',{"class":"short-content"}).text.strip("影评可能有\
                      剧透\n \xa0(展开)这篇剧评可能有剧透")])
        
def main():
    rinfo=[]
    start_url='https://movie.douban.com/review/best/?start='
    for i in range(5):
        try:
            url = start_url+str(20*i)
            html = getHTMLText(url)
            fillReviewList(rinfo,html)
        except:
            continue
    fw=open("D:/rinfo3.csv","w",encoding='utf-8-sig')
    fw.write(",".join(["title","review"])+"\n")
    for r in rinfo:
        fw.write(','.join(r)+'\n')
    fw.close()
main()

运行截图如下：

python使用多线程进行爬豆瓣电影top250海报图片，附源码加运行结果_小赵同学的博客-CSDN博客