python增量爬虫_python网络爬虫——增量式爬虫

#1.spider文件

importscrapyfrom movieAddPro.items importMovieaddproItemfrom scrapy.linkextractors importLinkExtractorfrom scrapy.spiders importCrawlSpider, Rulefrom redis importRedisclassMovieaddSpider(CrawlSpider):

name= 'movieadd'

#allowed_domains = ['www.xxx.com']

start_urls= ['https://www.4567tv.tv/frim/index1.html']

link= LinkExtractor(allow=r'.frim/index1-\d+.html')

rules=(

Rule(link, callback='parse_item', follow=True),

)#创建reids连接对象

conn = Redis(host='127.0.0.1',port=6379)#解析电影的名称和详情页的url

defparse_item(self, response):

li_list= response.xpath('/html/body/div[1]/div/div/div/div[2]/ul/li')for li inli_list:

title= li.xpath('./div/a/@title').extract_first()#获取详情页url

detail_url = 'https://www.4567tv.tv' + li.xpath('./div/a/@href').extract_first()

item=MovieaddproItem()

item['title'] =title#判断该详情页的url是否进行请求发送

ex = self.conn.sadd('movieadd_detail_urls',detail_url)if ex == 1: #说明detail_url之前不存在redis的set集合中，需要发送请求

print('已有新数据更新，正在爬取数据......')yield scrapy.Request(url=detail_url,callback=self.parse_detail,meta={'item':item})else:print('暂无新数据更新......')defparse_detail(self,response):

item= response.meta['item']

desc= response.xpath('/html/body/div[1]/div/div/div/div[2]/p[5]/span[3]/text()').extract_first()

item['desc'] =descyielditem--------------------------------------------------------------------------------

#2.pipelines文件

classMovieaddproPipeline(object):defprocess_item(self, item, spider):

dic={'title':item['title'],'desc':item['desc']

}print(dic)

conn=spider.conn

conn.lpush('movieadd_data',dic)returnitem--------------------------------------------------------------------------------

#3.items文件

importscrapyclassMovieaddproItem(scrapy.Item):

title=scrapy.Field()

desc=scrapy.Field()--------------------------------------------------------------------------------

#4.setting文件

BOT_NAME= 'movieAddPro'SPIDER_MODULES= ['movieAddPro.spiders']

NEWSPIDER_MODULE= 'movieAddPro.spiders'USER_AGENT= 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'ROBOTSTXT_OBEY=False

LOG_LEVEL= 'ERROR'ITEM_PIPELINES={'movieAddPro.pipelines.MovieaddproPipeline': 300,

}