xpath_爬取豆瓣电影TOP250

爬取豆瓣电影TOP250

爬取豆瓣电影Top250的电影信息：电影名称，电影评分，评价人数，电影短评

源代码：

import csv
import lxml.etree as etree
import requests    


def get_content(url):
    try:
        user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
        response = requests.get(url,headers={'User-Agent':user_agent})
        response.raise_for_status()     # 如果返回的状态码不是200，则抛出异常
        response.encoding = response.apparent_encoding      # 根据响应信息判断网页的编码格式，便于response.text知道如何解码
    except Exception as e:
        print('爬取错误')
    else:
        print('爬取成功')
        return response.text

def parser_content(html):
    # 1. 将html内容转化为xpath可以解析/匹配的格式
    selector = etree.HTML(html)
    # 2.获取每个电影的详细信息，存储在li标签 <ol class="grid_view"><li></li></ol>
    movieDetails = selector.xpath('//ol[@class="grid_view"]/li')
    # 3.获取需要的电影信息
    for movieDetail in movieDetails:
        # 电影名称 <span class="title">肖申克的救赎</span>
        movieName = movieDetail.xpath('.//span[@class="title"]/text()')[0]
        # 电影评分 <span class="rating_num" property="v:average">9.6</span>
        movieScore = movieDetail.xpath('.//span[@class="rating_num"]/text()')[0]
        # 评价人数 <div class="star"><span>1323431人评价</span></div>
        movieCommentNum = movieDetail.xpath('.//div[@class="star"]/span/text()')[1]
        # 电影短评 <span class="inq">希望让人自由。</span>
        movieCommentObj = movieDetail.xpath('.//span[@class="inq"]/text()')
        if movieCommentObj:
            movieComment = movieCommentObj[0]
        else:
            movieComment = '无短评'
        movieInfo.append((movieName,movieScore,movieCommentNum,movieComment))

    return movieInfo

def save_csv(movieInfo):
    with open('doc/douban_xpath.csv','w') as f:
        writer = csv.writer(f)
        writer.writerows(movieInfo)
    print('csv文件保存成功......')




if __name__ == '__main__':
    doubanTopPage = 10
    perPage = 25
    movieInfo = []
    for page in range(1,doubanTopPage+1):
        url = 'https://movie.douban.com/top250?start=%s' %((page-1)*perPage)
        content = get_content(url)
        parser_content(content)


    save_csv(movieInfo)

爬取结果保存到csv文件中：