import scrapy
from testSpider.items import TestspiderItem
class MeinvSpider(scrapy.Spider):
name = 'meinv'
allowed_domains = ['www.3gbizhi.com']
start_urls = ['http://www.3gbizhi.com/meinv/']
page = 2
page_url = 'https://www.3gbizhi.com/meinv/index_%d.html'
def parse(self, response, **kwargs):
ul = response.xpath('/html/body/div[4]/ul/li | /html/body/div[5]/ul/li')
for div in ul:
name = div.xpath('./a/div[2]/text()').extract_first()
item = TestspiderItem()
item['file_name'] = name
yield item
if self.page <= 5:
new_url = self.page_url % self.page
self.page += 1
# 在解析过程中产生新的url,需要对新的url再次发起请求时,yield 手动调用scrapy.Request方法对象,
yield scrapy.Request(url=new_url, callback=self.parse)
总结:两个地方会用到item
第一用yield 返回将item返回给管道
第二用yield手动发起Requets请求或者是FormRequests请求,即封装新的url。