爬虫—scrapy框架（二）简单代码实战

目录

将爬取数据直接存入到自定义文件中

将爬取数据通过管道存放到对应文件中

次级页面内容获取

---

将爬取数据直接存入到自定义文件中

import scrapy
import json
import time

class DoubanSpider(scrapy.Spider):  # 继承scrapy.Spider模板类
    name = 'douban'  # spider的名字，必须有，且唯一，用来唯一标记每一个爬虫文件
    allowed_domains = ['movie.douban.com']  # 用于限制爬虫爬去网站的域名（可以添加多个），对不是该域名下的网站进行过滤
    start_urls = ['https://movie.douban.com/top250']  # 初始的url列表
    num = 1

    def parse(self, response):
        movie_data = response.xpath('//div[@class="info"]')

        if movie_data:
            with open('db.txt','a',encoding='utf-8') as f:
                for idx, data_one in enumerate(movie_data):
                    movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
                    movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
                    score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()
                    # 写入自定义文件
                    data = {}
                    data['电影名'] = movie_name
                    data['导演'] = movie_director
                    data['评分'] = score
                    f.write(json.dumps(data,ensure_ascii=False)+'\n')
                    yield data

                time.sleep(5)
                print("&&&&&"*20)
                self.url  = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
                da = scrapy.Request(self.url,callback=self.parse)
                self.num += 1
                yield da
        else:
            return

将爬取数据通过管道存放到对应文件中

1、编写爬虫项目文件

import scrapy
import time
from ..items import MovieItem

class DoubanSpider(scrapy.Spider):  # 继承scrapy.Spider模板类
    name = 'douban'  # spider的名字，必须有，且唯一，用来唯一标记每一个爬虫文件
    allowed_domains = ['movie.douban.com']  # 用于限制爬虫爬去网站的域名（可以添加多个），对不是该域名下的网站进行过滤
    start_urls = ['https://movie.douban.com/top250']  # 初始的url列表
    num = 1

    def parse(self, response):
        movie_data = response.xpath('//div[@class="info"]')

        if movie_data:
            for idx, data_one in enumerate(movie_data):
                movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
                movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
                score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()

                # 写入管道文件
                item = MovieItem()
                item['movie_name'] = movie_name
                item['movie_director'] = movie_director
                item['score'] = score
                yield item   # 返回item对象

            time.sleep(5)
            print("&&&&&"*20)

            self.url  = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
            dt = scrapy.Request(self.url,callback=self.parse)
            yield dt
            self.num += 1
            
        else:
            return

2、数据字段文件（items.py）中创建对应字段

import scrapy

class MovieItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    movie_name = scrapy.Field()
    movie_director = scrapy.Field()
    score = scrapy.Field()
    movie_synopsis = scrapy.Field()

3、管道文件（pipelines.py）中创建管道数据处理方法

import json
from itemadapter import ItemAdapter

class MoviePipeline:
    def open_spider(self,spider):
        self.f = open('douban.txt','w',encoding='utf-8')
    def process_item(self, item, spider):
        self.f.write(json.dumps(dict(item),ensure_ascii=False)+'\n')
        return item
    def close_spider(self,spider):
        self.f.close()

4、配置文件（settings.py）中设置相关内容

robots.txt规则

ROBOTSTXT_OBEY = True

添加请求头信息

DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'
}

开启管道：在此设置中为类分配的整数值决定了它们运行的顺序：按照从较低值到较高值的顺序进行

ITEM_PIPELINES = {
   'movie.pipelines.MoviePipeline': 300,
}

次级页面内容获取

import scrapy
import time
from ..items import MovieItem

class DoubanSpider(scrapy.Spider):  # 继承scrapy.Spider模板类
    name = 'douban'  # spider的名字，必须有，且唯一，用来唯一标记每一个爬虫文件
    allowed_domains = ['movie.douban.com']  # 用于限制爬虫爬去网站的域名（可以添加多个），对不是该域名下的网站进行过滤
    start_urls = ['https://movie.douban.com/top250']  # 初始的url列表
    num = 1

    def parse(self, response):
        
        movie_data = response.xpath('//div[@class="info"]')

        if movie_data:
            for idx, data_one in enumerate(movie_data):
                movie_name = data_one.xpath('./div[@class="hd"]/a/span/text()')[0].extract()
                movie_director = data_one.xpath('./div/p/text()')[0].extract().strip().replace('\xa0',' ')
                score = data_one.xpath('./div//span[@class="rating_num"]/text()')[0].extract()

                # 写入管道文件
                item = MovieItem()
                item['movie_name'] = movie_name
                item['movie_director'] = movie_director
                item['score'] = score
 
                # 获取次级页面内容
                next_url = data_one.xpath('.//div[@class="hd"]/a/@href').get()
                print(next_url)
                yield scrapy.Request(next_url,callback=self.get_nextdata,meta={'datas':item})

            time.sleep(5)
            print("&&&&&"*20)
            self.url  = 'https://movie.douban.com/top250?start={}&filter='.format(self.num*25)
            dt = scrapy.Request(self.url,callback=self.parse)
            yield dt
            self.num += 1
            
        else:
            return

    def get_nextdata(self,response):
        item = MovieItem()
        data = response.meta['datas']
        item.update(data)
        time.sleep(1)

        movie_synopsis = response.xpath('.//div[@class="related-info"]//span[@property="v:summary"]/text()').get().strip()
        item['movie_synopsis'] = movie_synopsis
        yield item