目录
将爬取数据直接存入到自定义文件中
import scrapy
import json
import time
class DoubanSpider(scrapy.Spider): # 继承scrapy.Spider模板类
name = 'douban' # spider的名字,必须有,且唯一,用来唯一标记每一个爬虫文件
allowed_domains = ['movie.douban.com'] # 用于限制爬虫爬去网站的域名(可以添加多个),对不是该域名下的网站进行过滤
start_urls = ['https://movie.douban.com/top250'] # 初始的url列表
num = 1
def parse(self, response):
movie_data = response.xpath('//div[@class="info"]')
if movie_data:
with open('db.txt','a',encoding='utf-8') as f:
for idx, data_one in enumerate(movie_data):
movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()
# 写入自定义文件
data = {}
data['电影名'] = movie_name
data['导演'] = movie_director
data['评分'] = score
f.write(json.dumps(data,ensure_ascii=False)+'\n')
yield data
time.sleep(5)
print("&&&&&"*20)
self.url = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
da = scrapy.Request(self.url,callback=self.parse)
self.num += 1
yield da
else:
return
将爬取数据通过管道存放到对应文件中
1、编写爬虫项目文件
import scrapy
import time
from ..items import MovieItem
class DoubanSpider(scrapy.Spider): # 继承scrapy.Spider模板类
name = 'douban' # spider的名字,必须有,且唯一,用来唯一标记每一个爬虫文件
allowed_domains = ['movie.douban.com'] # 用于限制爬虫爬去网站的域名(可以添加多个),对不是该域名下的网站进行过滤
start_urls = ['https://movie.douban.com/top250'] # 初始的url列表
num = 1
def parse(self, response):
movie_data = response.xpath('//div[@class="info"]')
if movie_data:
for idx, data_one in enumerate(movie_data):
movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()
# 写入管道文件
item = MovieItem()
item['movie_name'] = movie_name
item['movie_director'] = movie_director
item['score'] = score
yield item # 返回item对象
time.sleep(5)
print("&&&&&"*20)
self.url = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
dt = scrapy.Request(self.url,callback=self.parse)
yield dt
self.num += 1
else:
return
2、数据字段文件(items.py)中创建对应字段
import scrapy
class MovieItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
movie_name = scrapy.Field()
movie_director = scrapy.Field()
score = scrapy.Field()
movie_synopsis = scrapy.Field()
3、管道文件(pipelines.py)中创建管道数据处理方法
import json
from itemadapter import ItemAdapter
class MoviePipeline:
def open_spider(self,spider):
self.f = open('douban.txt','w',encoding='utf-8')
def process_item(self, item, spider):
self.f.write(json.dumps(dict(item),ensure_ascii=False)+'\n')
return item
def close_spider(self,spider):
self.f.close()
4、配置文件(settings.py)中设置相关内容
robots.txt规则
ROBOTSTXT_OBEY = True
添加请求头信息
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'
}
开启管道:在此设置中为类分配的整数值决定了它们运行的顺序:按照从较低值到较高值的顺序进行
ITEM_PIPELINES = {
'movie.pipelines.MoviePipeline': 300,
}
次级页面内容获取
import scrapy
import time
from ..items import MovieItem
class DoubanSpider(scrapy.Spider): # 继承scrapy.Spider模板类
name = 'douban' # spider的名字,必须有,且唯一,用来唯一标记每一个爬虫文件
allowed_domains = ['movie.douban.com'] # 用于限制爬虫爬去网站的域名(可以添加多个),对不是该域名下的网站进行过滤
start_urls = ['https://movie.douban.com/top250'] # 初始的url列表
num = 1
def parse(self, response):
movie_data = response.xpath('//div[@class="info"]')
if movie_data:
for idx, data_one in enumerate(movie_data):
movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()
# 写入管道文件
item = MovieItem()
item['movie_name'] = movie_name
item['movie_director'] = movie_director
item['score'] = score
# 获取次级页面内容
next_url = data_one.xpath('.//div[@class="hd"]/a/@href').get()
print(next_url)
yield scrapy.Request(next_url,callback=self.get_nextdata,meta={'datas':item})
time.sleep(5)
print("&&&&&"*20)
self.url = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
dt = scrapy.Request(self.url,callback=self.parse)
yield dt
self.num += 1
else:
return
def get_nextdata(self,response):
item = MovieItem()
data = response.meta['datas']
item.update(data)
time.sleep(1)
movie_synopsis = response.xpath('.//div[@class="related-info"]//span[@property="v:summary"]/text()').get().strip()
item['movie_synopsis'] = movie_synopsis
yield item