scrapy

最新推荐文章于 2023-04-10 06:45:00 发布

数星星啦

最新推荐文章于 2023-04-10 06:45:00 发布

阅读量153

点赞数

分类专栏： scrapy 爬虫

本文链接：https://blog.csdn.net/gcyangxin/article/details/108705577

版权

scrapy 同时被 2 个专栏收录

1 篇文章 0 订阅

订阅专栏

爬虫

1 篇文章 0 订阅

订阅专栏

环境

Scrapy 2.3.0

python3.7

0.24.6中文入门教程

调试SHELL

分为全局和项目only，cd project 则可用项目only

命令行工具使用

菜鸟教程--xpath使用

#启动
scrapy shell [url|file]

#帮助
scrapy shell -h

#设置
settings.__dict__ #获取所有设置，包括user agent等headers
setting.get('USER_AGENT')
'Scrapy/2.3.0 (+https://scrapy.org)'

#测试糗事百科，糗百会禁止爬虫，增加user_agent。注意参数的名字大小写和下划线。
scrapy shell --set=USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/40.0.2214.111 Chrome/40.0.2214.111 Safari/537.36" https://www.qiushibaike.com/
>>response
<200 https://www.qiushibaike.com/>
>>response.headers
{b'Server': [b'openresty'],
 b'Date': [b'Mon, 21 Sep 2020 03:16:14 GMT'],
 b'Content-Type': [b'text/html; charset=UTF-8'],
 b'Set-Cookie': [b'_xsrf=2|46b435ed|e701954a6c5afd19da708aac09740f52|1600658174; Path=/'],
 b'Vary': [b'User-Agent, Accept-Encoding'],
 b'Etag': [b'"87f963dd02a197ab0c2cf4f2395d84f9631edaa0"']}
#xpath 返回List
In [1]: response.xpath('//title')
Out[1]: [<Selector xpath='//title' data=u'<title>Open Directory - Computers: Progr'>]

In [2]: response.xpath('//title').extract()
Out[2]: [u'<title>Open Directory - Computers: Programming: Languages: Python: Books</title>']

In [3]: response.xpath('//title/text()')
Out[3]: [<Selector xpath='//title/text()' data=u'Open Directory - Computers: Programming:'>]

In [4]: response.xpath('//title/text()').extract()
Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books']

In [5]: response.xpath('//title/text()').re('(\w+):')
Out[5]: [u'Computers', u'Programming', u'Languages', u'Python']
>>response.xpath('//span[@class="next"]/../@href').extract() #获取属性
['/8hr/page/2/']

/html/head/title: 选择HTML文档中 <head> 标签内的 <title> 元素
/html/head/title/text(): 选择上面提到的 <title> 元素的文字
//td: 选择所有的 <td> 元素
//div[@class="mine"]: 选择所有具有 class="mine" 属性的 div 元素
#CSS选择器
response.css('title::text').extract()
response.css('base::attr(href)').extract()
#rebot验证
#有些网站允许爬虫，但是会设定权限，可根据rebot.txt结果查看 user_agent
#scrapy  shell --set=USER_AGENT="Baiduspider"   http://www.sohu.com/a/337634404_100032610 
>>
>>
>>
>>
>>

文字

糗事百科

#settings.py

DEFAULT_REQUEST_HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/40.0.2214.111 Chrome/40.0.2214.111 Safari/537.36"
}
ITEM_PIPELINES = {
   'study.pipelines.StudyPipeline': 300, # 权重值越小，越优先执行！
}

#pipeliens.py
#重写管道，scrapy会按照顺序调用
from itemadapter import ItemAdapter
class StudyPipeline:
    def __init__(self):
        self.f = open('./ret.txt', 'w')

    def open_spider(selfs, spider):
        print('start!')
    def process_item(self, item, spider):
        txt=item['author']+'\t'+item['body']+'\t'+\
            item['laugh']+' laughs'+'\t'+\
            item['comments']+' commnets'+'\n'
        self.f.write(txt)

        return item
    def close_spider(self,spider):
        self.f.close()
        print("over!")
#import  scrapy
from study.items import QiuShiItem

#qiushi_spider.py
class QiuShi(scrapy.Spider):
    name='qiushibaike'
    allowed_domains=['qiushibaike.com']

    start_urls=['http://www.qiushibaike.com']


    def parse(self, response):
        item = QiuShiItem()
        for sel in response.xpath('//div[@class="recmd-right"]'):
            item['author'] =sel.xpath('.//span[@class="recmd-name"]/text()').extract()[0]
            item['body'] = sel.xpath('./a/text()').extract()[0]
            item['laugh']= sel.xpath('.//div[@class="recmd-num"]/span[1]/text()').extract()[0]
            item['comments']=sel.xpath('.//div[@class="recmd-num"]/span[last()-1]/text()').extract()[0]
            # item=QiuShi.encodeItem(item)
            yield item
        next_page=response.xpath('//span[@class="next"]/../@href').extract()[0]
        # print(next_page)
        if next_page:
            next=self.start_urls[0] +next_page
            yield scrapy.Request(next,callback=self.parse)
        else:
            return

图片

默认保存的名字用md5/sha1等hash算法计算出来,可重写file_path函数

#settings.py
ITEM_PIPELINES = {
   'pictest.pipelines.PictestPipeline': 300,
}
IMAGES_STORE='./pics' #
IMAGES_URLS_FIELD='pic_url'#item里字段名称修改
USER_AGENT = 'Baiduspider'#souhu允许百度爬虫

#pipelines.py
from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
from scrapy.http import Request
from scrapy.exceptions import DropItem

class PictestPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for url in item['pic_url']:
            yield Request(url)

    def item_completed(self, results, item, info) :
        image_path = [x['path'] for ok, x in results if ok]
        if not image_path :
            raise DropItem('Item contains no images')
        item['image_paths'] = image_path
        return item
    #def file_path(self, request, response=None, info=None):
    #    PictestPipeline.pic_name+=1
    #   return (str(PictestPipeline.pic_name)+'.jpg')

#results >>(True, {'url': 'http://5b0988e595225.cdn.sohucs.com/images/20190830/1780022790ac42c4939c6ba7774cd9ed.jpeg', 'path': 'full/80db58946bd9c706b959529b2140e8e8fecc0360.jpg', 'checksum': '209e953b433db95eb84ed51650117ebc', 'status': 'downloaded'})

#items.py
import scrapy
class PictestItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pic_url=scrapy.Field()#可以自定义字段名字，默认为 ‘image_urls’，settings设置
    images_result_field=scrapy.Field()#默认为‘images_result_field’
    image_pathd=scrapy.Field()

#pic_spider.py
import  scrapy
from pictest.items import PictestItem
class Pic(scrapy.Spider):
    name='pictest'
    allowed_domains = ['sohu.com']
    start_urls = ['http://www.sohu.com/a/337634404_100032610']
    def parse(self, response):
        item = PictestItem()
        item['pic_url']=response.xpath('//article//img/@src').extract()
        yield item

双管道

1.用scrapy框架的时候，一定要先明白执行的顺序，代码已写好，程序开始运行~

SPIDERS的yeild将request发送给ENGIN
ENGINE对request不做任何处理发送给SCHEDULER
SCHEDULER( url调度器)，生成request交给ENGIN
ENGINE拿到request，通过MIDDLEWARE进行层层过滤发送给DOWNLOADER
DOWNLOADER在网上获取到response数据之后，又经过MIDDLEWARE进行层层过滤发送给ENGIN
ENGINE获取到response数据之后，返回给SPIDERS，SPIDERS的parse()方法对获取到的response数据进行处理，解析出items或者requests
将解析出来的items或者requests发送给ENGIN
ENGIN获取到items或者requests，将items发送给ITEMPIPELINES，将requests发送给SCHEDULER
注意！只有当调度器中不存在任何request了，整个程序才会停止，（也就是说，对于下载失败的URL，Scrapy也会重新下载。）

#main.py 调试模式
#!/usr/bin/env python
#-*- coding:utf-8 -*-

from scrapy.cmdline import execute
import os
import sys

#添加当前项目的绝对地址
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
execute(['scrapy', 'crawl', 'douban250'])

#settings.py

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/40.0.2214.111 Chrome/40.0.2214.111 Safari/537.36"
# Obey robots.txt rules 豆瓣禁止爬虫访问图片
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
   'douban250.pipelines.Douban250Pipeline': 100,
   'douban250.pipelines.Douban250AvatarPipeline': 200,
}
DOWNLOAD_DELAY = 5 #每次request的间隔
IMAGES_STORE='./pic'

#items.py
import scrapy
class Douban250Item(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    director=scrapy.Field()
    kinds=scrapy.Field()
    time=scrapy.Field()
    detail=scrapy.Field()
    intro=scrapy.Field()
    score=scrapy.Field()
    persons=scrapy.Field()
    image_urls=scrapy.Field()
    image_path=scrapy.Field()

#pipelines.py
from itemadapter import ItemAdapter
from scrapy.pipelines.images import  ImagesPipeline
from scrapy.http import Request
from scrapy.exceptions import DropItem
from  .settings import IMAGES_STORE

class Douban250Pipeline:

    def process_item(self, item, spider):

        fileName='%s/%s.txt'%(IMAGES_STORE,item['name'])
        txt="{} \n{}\n类型:{}\n简介:{}\n{}分  {}\n{}"\
            .format(item['name'],item['director'].strip(),item['kinds'].strip(),
                    item['detail'],item['score'],
                    item['persons'],item['intro'].strip())
        with open(fileName,'w') as f:
            f.write(txt)
        return item


class Douban250AvatarPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
       url=item['image_urls']

       yield Request(url,cb_kwargs={'name':item['name']})

    def item_completed(self, results, item, info):
        images_path = [x['path'] for ok,x in results if ok]
        if not images_path:
            raise DropItem("item contains no images")
        item['image_path'] = images_path
        return item

    def file_path(self, request, response=None, info=None):
        return '%s.jpg'%(request.cb_kwargs['name'])



#douban_spider.py
from scrapy import Spider,Request
from douban250.items import Douban250Item

class DouBan(Spider):
    name = 'douban250'
    start_urls = ['https://movie.douban.com/top250?start=50&filter=']
    def parse(self, response, **kwargs):
        item=Douban250Item()
        for one in response.xpath('//div[@class="item"]'):
            item['name']=one.xpath('.//div[@class="pic"]/a/img/@alt').extract_first()
            item['image_urls']=one.xpath('.//div[@class="pic"]/a/img/@src').extract_first()
            item['director'],item['kinds']= one.xpath('.//div[@class="bd"]/p[@class=""]/text()').extract()
            # item['time']=''
            item['detail']= one.xpath('.//span[@class="inq"]/text()').extract_first()
            item['score'], item['persons']= one.xpath('.//div[@class="star"]/span/text()').extract()
            # item['persons']=one.xpath('.//div[@class="star"]/span/text()')[1::2].extract()
            intro_link=one.xpath('.//div[@class="pic"]/a/@href').extract_first()

            # yield会把Request给SCHEDULER-ENGIN-MIDDLEWARE-DOWNLOADER-response
            #MIDDLEWARE-ENGINE-SPIDERS（parse_intro)-ENGIN-items-ITEMPIPELINES

            yield Request(intro_link,callback=self.parse_intro,meta=item)
        next_page=response.xpath('//span[@class="next"]/a/@href').extract_first()
        # print(next_page)
        if next_page:
            
            yield Request(response.urljoin(next_page),callback=self.parse)
        else:
            return
    def parse_intro(self,response,**kwargs):
        item=response.meta
        item['intro']=response.xpath('//span[@property="v:summary"]/text()').extract_first()
        yield item

数星星啦

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
scrapy

环境Scrapy 2.3.0python3.70.24.6中文入门教程调试SHELL分为全局和项目only，cd project 则可用项目only命令行工具使用菜鸟教程--xpath使用#启动scrapy shell [url|file]#帮助scrapy shell -h#设置settings.__dict__ #获取所有设置，包括user agent等headerssetting.get('USER_AGENT')'Scrapy/2.3.0 (+ht
复制链接

扫一扫

专栏目录