Scrapy请求传参

scrapy.Request(url=url, callback=self.parse_item, meta={'item': item}, headers=headers)

url: 要请求的地址

callback：响应成功之后的回调函数

meta: 参数传递

headers: 定制头信息，一般不用

# -*- coding: utf-8 -*-
'''
要抓取的数据部分在一级页面、一部分在二级页面
'''
import scrapy
from movie.items import MovieItem

class MvSpider(scrapy.Spider):
# 爬虫文件的名字
name = 'mv'
# 允许的域名
allowed_domains = ['www.ygdy8.net'] # 起始的url start_urls = ['http://www.ygdy8.net/html/gndy/dyzz/index.html'] def parse(self, response): a_list = response.xpath('//div[@class="co_content8"]//table//a') for a in a_list: title = a.xpath('./text()').extract_first() href = 'http://www.ygdy8.net' + a.xpath('./@href').extract_first() movie = MovieItem(title=title) # callback是执行的方法 meta是响应时候携带的参数 yield scrapy.Request(url=href,callback=self.parse_detail,meta={'movie':movie}) def parse_detail(self,response): movie = response.meta['movie'] src = response.xpath('//div[@id="Zoom"]//p[1]/img[1]/@src').extract_first() movie['src']=src yield movie

 

转载于:https://www.cnblogs.com/huanggaoyu/p/10656674.html