Scrapy 获取电影详细信息

import scrapy
from scrapy import Selector, Request
import sys
sys.path.append(r'D:\编程\pycharm\py文件\scrapy\spider2107\spider2107')
from items import MovieItem



class MovieSpider(scrapy.Spider):
    # 爬虫名
    name = 'movie'
    # 爬取网站的域名
    allowed_domains = ['movie.douban.com']
    # 入口url
    start_urls = ['https://movie.douban.com/top250']
    def start_requests(self):
        for page in range(1):
            yield scrapy.Request(url=f'https://movie.douban.com/top250?start={page*25}&filter=')



    def parse(self, response):
        # 首先抓取电影列表
        movie_list = response.xpath("//ol[@class='grid_view']/li")
        for selector in movie_list:
            # 遍历每个电影列表，从其中精准抓取所需要的信息并保存为item对象
            detail_url = selector.css('div.info > div.hd > a::attr(href)').extract_first()
            # print(detail_url)
            item = MovieItem()
            item['name'] = selector.xpath(".//span[@class='title']/text()").extract_first()
            item['ranting'] = selector.xpath(".//span[@class='rating_num']/text()").extract_first()
            item['inq'] = selector.xpath(".//span[@class='inq']/text()").extract_first()

            yield Request(
                url=detail_url,
                callback=self.parse_detail,
                cb_kwargs={'item': item}
            )

    def parse_detail(self, response, **kwargs):
        item = kwargs['item']

        item['duration'] = response.xpath(".//span[@property='v:runtime']/text()").extract_first()
        item['comment'] = response.xpath(".//span[@property='v:summary']/text()").extract_first()

        yield item