python scrapy框架项目，管道文件代码以及几种文件的保存方式_scrapy返回一个str给管道进行保存-CSDN博客

本文链接：https://blog.csdn.net/crq_zcbk/article/details/81587392

保存为json格式的终端操作代码：

scrapy crawl 爬虫文件名 -o 随便起的文件名.json -s FEEN_EXPORT_ENCODING=utf-8

import scrapy
# 在同级文件夹路径下找到指定的文件items
# 所以要回到上级文件夹路径来找  ..回到上级路径
from ..items import MokoItem
class MokokoSpider(scrapy.Spider):
    name = 'mokoko'
    allowed_domains = ['moko.cc']
    #通常会修改start_rl
    start_urls = ['http://www.moko.cc/channels/post/153/1.html']

    def parse(self, response):
        # print(response.text)
        ul_list=response.xpath('//ul[@class="post small-post"]')
        print(ul_list)
        all_items=[]
        for ul in ul_list:
            # 初始化一个item对象
            item= MokoItem()
            # xpath对象获取内容都是列表
            # 返回的内容为：scrapy.selector
            # 如果对象类型为scrapy.selector  那么这个对象可以
            # 被继续迭代  也可以被xpath继续寻找里面的内容
            title=ul.xpath('.//div[@class="cover"]/@cover-text')
            # print(title)
            # print(type(title))
            # 将xpath对象转化为列表对象   [0]取出里面的元素
            title=title.extract()[0]
            # print(title)
            # 如果对象的类型为list  那么这个对象可以迭代
            # 但是不能再继续使用xpath
            # print(type(title))

            clicknum=ul.xpath('.//li[last()]/span/text()').extract()[0]

            imgsrc=ul.xpath('.//img/@src2').extract()[0]
            item['title']=title
            item['imgsrc'] = imgsrc
            item['clicknum'] = clicknum
            yield item

#项目需求是将斗鱼网页代码用json格式获取里面的图片链接下载并在终端保存为json格式


#项目需求是   将斗鱼网页代码用json格式获取里面的图片链接下载  并在终端保存为json格式


import scrapy
from  ..items import DoudouyuItem
import json
class TupianSpider(scrapy.Spider):
    name = 'tupian'
    allowed_domains = ['api.douyucdn.cn']

    start_urls = ['http://api.douyucdn.cn/api/v1/getverticalRoom?limit=20&offset=']

    def parse(self, response):

        jsobj = json.loads(response.text)
        for src in jsobj["data"]:
            item = DoudouyuItem()
            src = src["room_src"]
            # print(src)
            item['src']=[src]
            yield item

        # print('正在获取第一页')
            for x in range(0,500,20):
                url='http://api.douyucdn.cn/api/v1/getverticalRoom?limit=20&offset=' + str(x)
                yield scrapy.Request(url=url,callback=self.parse)





如果只下载图片只需要在settings.py里将下载管道解注释然后按照如下代码进行：

ITEM_PIPELINES = {
   # 'doudouyu.pipelines.DoudouyuPipeline': 300,
   #专门负责下载图片的管道
    'scrapy.pipelines.images.ImagesPipeline':1
}
IMAGES_STORE='tutuqian'
IMAGES_URLS_FIELD='src'

项目需求将http://pic.netbian.com/4kmeishi/中的图片下载下来

项目需求  将http://pic.netbian.com/4kmeishi/中的图片下载下来





# -*- coding: utf-8 -*-
import scrapy

from  ..items import ImagenetItem
class ImageSpider(scrapy.Spider):
    name = 'image'
    allowed_domains = ['pic.netbian.com']
    start_urls = ['http://pic.netbian.com/4kmeishi/']

    def parse(self, response):
        img_list = response.xpath('//ul[@class="clearfix"]/li/a/img/@src')
        print(img_list)
        for img in img_list:
            item=ImagenetItem()
            src = 'http://pic.netbian.com/' + img.extract()
            # print(src)
            item['src']=[src]
            yield item
        next_url=response.xpath('//div[@class="page"]/a[text()="下一页"]/@href').extract()
        if len(next_url)!=0:

            print('*****************************')
            url='http://pic.netbian.com/'+next_url[0]
            yield scrapy.Request(url=url,callback=self.parse)
            # print(next_url)



ITEM_PIPELINES = {
   # 'imagenet.pipelines.ImagenetPipeline': 300,

#ImagesPipeline要与管道中的方法名字一样

    'scrapy.pipelines.images.ImagesPipeline':1
}
#图片的存储路径
IMAGES_STORE='imagessss'
IMAGES_URLS_FIELD='src'

存储为表格

scrapy crawl 爬虫名 -o 爬虫名.csv

存储为Excel

scrapy crawl 爬虫名 -o 爬虫名.xml

存储为json并且转码为中文

scrapy crawl 爬虫名 -o 爬虫名.json -s FEED_EXPORT_ENCODING=utf-8

保存TXT文本

项目需求  将贴吧中楼主的内容全部保存为TXT文本




# -*- coding: utf-8 -*-
import scrapy
# from..items import XiaoshuospiderItem

class ZhigengniaoSpider(scrapy.Spider):
    name = 'zhigengniao'
    allowed_domains = ['tieba.baidu.com']
    start_urls = ['https://tieba.baidu.com/p/5815118868?pn=']

    def parse(self, response):
        info_list = response.xpath('//div[@class="l_post l_post_bright j_l_post clearfix  "]')
        for info in info_list:
            name_list = info.xpath('.//ul[@class="p_author"]/li/a/text()').extract()
            for name in name_list:
                if name == '乔深沉':
                    content_list = info.xpath('.//div[@class="p_content  "]/cc/div/text()')
                    for con in content_list:
                        # item = XiaoshuospiderItem()
                        con = con.extract()

                        # item['con'] = con
                        # yield item

                        with open('xiaoshuo.txt','a',encoding='utf-8')as f:
                            f.write(con)
                            f.write('\n')
        next_url = response.xpath('//li[@class="l_pager pager_theme_5 pb_list_pager"]/a[text()="下一页"]/@href').extract()
        if len(next_url) != 0:
            url = 'https://tieba.baidu.com' + next_url[0]

            yield scrapy.Request(url=url, callback=self.parse)

项目需求将小说和图片同时保存在本地文件

项目需求  将小说和图片同时保存在本地文件



# -*- coding: utf-8 -*-
import scrapy

from ..items import QishuItem
class XiaoshuoSpider(scrapy.Spider):
    name = 'xiaoshuo'
    allowed_domains = ['qisuu.la']
    start_urls = ['https://www.qisuu.la/']

    def parse(self, response):
        # print(response.text)
        #获取所有小说的类型
        type_list=response.xpath('//div[@class="nav"]/a/@href').extract()
        # print(type_list)
        #列表里面第一个是首页  将首页  去掉
        del type_list[0]
        #输出url
        # print(response.url)
        for type in type_list:
            #拼接每一个类型的url地址
            # 在这个方法里面  response.url 为start_url
            url=response.url+type[1:]
            print(url)
            yield  scrapy.Request(url=url,callback=self.get_content_with_type_url)
    #用来找到每一种类型对应的小说
    def get_content_with_type_url(self,response):

        # print(response.text)
        #找到类型中  第一页所有小说详情页链接地址
        book_list=response.xpath('//div[@class="listBox"]/ul/li/a/@href').extract()
        print('************************')
        # print(book_list)
        for book in book_list:
            # 在这个方法里面  response.url 为: https://www.qisuu.la/soft/sort0(x)/
            url='https://www.qisuu.la'+book
            yield scrapy.Request(url=url,callback=self.get_detail_with_book_url)
            #获取每一本书的内容详情
    def get_detail_with_book_url(self,response):#extract_first  ：转化成列表同时取出第一个元素
        item=QishuItem()
        #获取小说标题
        name=response.xpath('//div[@class="detail_right"]/h1/text()').extract_first('')

        info_list=response.xpath('//div[@class="detail_right"]/ul/li/text()').extract()
        #获取需要下载的小说图片地址
        imageurl=response.xpath('//div[@class="detail_pic"]/img/@src').extract_first('')
        imageurl='https://www.qisuu.la'+imageurl
        #获取小说的下载地址
        downloadurl=response.xpath('//div[@class="showDown"]/ul/li[3]/script').extract_first('').split(',')[1].strip("'")
        print(downloadurl)
        print(imageurl)
        item['imageurl']=[imageurl]
        item['downloadurl']=[downloadurl]
        item['name']=name
        clicknum=info_list[0]
        item['clicknum']=clicknum
        filesize = info_list[1]
        item['filesize'] = filesize
        booktype = info_list[2]
        item['booktype'] = booktype
        updatetime = info_list[3]
        item['updatetime'] = updatetime
        bookstatus = info_list[4]
        item['bookstatus'] = bookstatus
        bookauthor = info_list[5]
        item['bookauthor'] = bookauthor
        print('//')
        print(info_list)
        yield item










ITEM_PIPELINES = {
   'qishu.pipelines.QishuPipeline': 300,
    #图片下载管道
    'scrapy.pipelines.images.ImagesPipeline':1,
    #文件（文字）下载管道
    'scrapy.pipelines.files.FilesPipeline':2
}
IMAGES_STORE='file/image'
IMAGES_URLS_FIELD='imageurl'

FILES_STORE='file/book'
FILES_URLS_FIELD='downloadurl'

将json格式添加标准

项目需求    将文件进行相应的保存   然后将json格式补充完整


# -*- coding: utf-8 -*-
import scrapy

import re
from ..items import HongxiuxiuItem
class XiuxiuSpider(scrapy.Spider):
    name = 'xiuxiu'
    allowed_domains = ['hongxiu.com']
    start_urls = ['https://www.hongxiu.com/all?gender=2&catId=-1']

    def parse(self, response):
        type_list=response.xpath('//ul[@type="category"]/li/a/@href').extract()
        del type_list[0]
        for type in type_list:

            url='https://www.hongxiu.com'+type
            split=re.compile(r'.*?catId=(.*?)&.*?',re.S)
            catId=re.findall(split,url)
            print(catId)
            yield scrapy.Request(url=url,meta={'type':catId[0]},callback=self.get_content_with_type_url)
    def get_content_with_type_url(self,response):
        catId=response.meta['type']
        for page_num in range(1,11):
            url='https://www.hongxiu.com/all?pageNum='+str(page_num)+'&pageSize=10&gender=2&catId='+catId+'&isFinish=-1&isVip=-1&size=-1&updT=-1&orderBy=0'
            print(url)
            yield scrapy.Request(url=url,callback=self.get_book_with_url)
    def get_book_with_url(self,response):
        detail_list=response.xpath('//div[@class="book-info"]/h3/a/@href').extract()
        for book in detail_list:
            url='https://www.hongxiu.com'+book
            print('********************************************************')
            print(url)
            yield scrapy.Request(url=url,callback=self.get_detail_with_url)
    def get_detail_with_url(self,response):
        type = response.xpath('//div[@class="crumbs-nav center1020"]/span/a[2]/text()').extract_first('')
        print(type)
        name = response.xpath('//div[@class="book-info"]/h1/em/text()').extract_first('')
        print(name)
        author = response.xpath('//div[@class="book-info"]/h1/a/text()').extract_first('')
        print(author)
        total = response.xpath('//p[@class="total"]/span/text()').extract_first('') + response.xpath(
            '//p[@class="total"]/em/text()').extract_first('')
        print(total)
        love = response.xpath('//p[@class="total"]/span[2]/text()').extract_first('') + response.xpath(
            '//p[@class="total"]/em[2]/text()').extract_first('')
        print(love)
        cilk = response.xpath('//p[@class="total"]/span[3]/text()').extract_first('') + response.xpath(
            '//p[@class="total"]/em[3]/text()').extract_first('')
        print(cilk)
        introduce = response.xpath('//p[@class="intro"]/text()').extract_first('')
        print(introduce)
        url = 'https:' + response.xpath('//div[@class="book-img"]//img/@src').extract_first('')
        url = url.replace('\r', '')
        print(url)

        item=HongxiuxiuItem()
        item['type']=type
        item['name'] = name
        item['author'] = author
        item['total'] = total
        item['love'] = love
        item['cilk'] = cilk
        item['introduce'] = introduce
        item['url']=[url]
        yield item





# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import codecs
import os
import json
class HongxiuxiuPipeline(object):
    def __init__(self):
        self.file=codecs.open(filename='book.json',mode='w+',encoding='utf-8')
        self.file.write('"book_list":[')
    def process_item(self, item, spider):
        res=dict(item)
        str=json.dumps(res,ensure_ascii=False)
        self.file.write(str)
        self.file.write(',\n')
        return item
    def close_spider(self,spider):
        self.file.seek(-1,os.SEEK_END)
        self.file.truncate()

        self.file.seek(-1,os.SEEK_END)
        self.file.truncate()

        self.file.write(']')
        self.file.close()



ITEM_PIPELINES = {
   'hongxiuxiu.pipelines.HongxiuxiuPipeline': 300,
    'scrapy.pipelines.images.ImagesPipeline':1
}
IMAGES_STORE='TUTUTUTUTU'
IMAGES_URLS_FIELD='url'

项目要求将图片进行下载然后将每个图片模块的名称作为下载图片后的存储文件名称

项目要求   将图片进行下载  然后将每个图片模块的名称作为下载图片后的存储文件名称





# -*- coding: utf-8 -*-
import scrapy

from ..items import SucaiItem
class TubiaoSpider(scrapy.Spider):
    name = 'tubiao'
    allowed_domains = ['sc.chinaz.com']
    start_urls = ['http://sc.chinaz.com/']

    def parse(self, response):
        icon_url=response.xpath('//li[@class="nos"]/a[3]/@href').extract_first('')
        full_url='http://sc.chinaz.com'+icon_url
        yield scrapy.Request(url=full_url,callback=self.parse_icon_url)
    def parse_icon_url(self,response):
        a_list=response.xpath('//ul[@class="pngblock imgload"]/li/span/a')
        for a in a_list:
            href=a.xpath('@href').extract_first('')
            title=a.xpath('text()').extract_first('')
            print(title)
            # meta:负责传递往下一个方法发送的内容
            yield scrapy.Request(url=href,meta={'title':title},callback=self.get_detail_with_url)
    def get_detail_with_url(self,reseponse):

        title=reseponse.meta['title']
        img_list=reseponse.xpath('//div[@class="png_sl"]/div/img/@src').extract()
        for img in img_list:
            item=SucaiItem()
            item['title']=title
            item['img']=[img]
            yield item





# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import scrapy
# 系统中下载图片的管道
from scrapy.pipelines.images import ImagesPipeline
# 系统管道有下载图片的功能  我们的管道继承了系统的管道也有了
# 下载图片的功能
class SucaiPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        # print('管道方法执行了')
        # 这个方法会循环执行
        # 前面每次传入一个item  这个item被交给了引擎
        # 引擎又交给了管道来执行   管道里面有很多个方法
        # 这些方法会依次执行
        yield scrapy.Request(url=item['img'][0],meta={'item':item})
    # 管道里面提供了一系列的内置方法，这些方法会自动从第一个执行到最后一个
    def file_path(self, request, response=None, info=None):
        print('/8*/*/*/*965327-**/-*-/-*/*/*/***/*/**/**/*/')
        item = request.meta['item']
        print(item['title'])
        print(item['img'])
        # 设置图片的路径为     类型名称/url地址
        image_name=item['img'][0].split('/')[-1]
        path='%s/%s' % (item['title'],image_name)
        return path






DOWNLOAD_DELAY = 0.3#将其解注释


ITEM_PIPELINES = {
   'sucai.pipelines.SucaiPipeline': 300,

}
IMAGES_STORE='imagesssssss'

下载4k风景图片

下载4k风景图片



# -*- coding: utf-8 -*-
import scrapy
from ..items import TuwangItem
class BianSpider(scrapy.Spider):
    name = 'bian'
    allowed_domains = ['pic.netbian.com']
    start_urls = ['http://pic.netbian.com/4kfengjing/']

    def parse(self, response):
        img_list=response.xpath('//ul[@class="clearfix"]/li//img/@src').extract()
        for img in img_list:
            url='http://pic.netbian.com'+img
            print(url)
            item=TuwangItem()
            item['url']=[url]
            yield item
        next_url=response.xpath('//div[@class="page"]/a[text()="下一页"]/@href').extract()
        if len(next_url) != 0:
            url='http://pic.netbian.com'+next_url[0]
            yield scrapy.Request(url=url,callback=self.parse)








ITEM_PIPELINES = {
   'tuwang.pipelines.TuwangPipeline': 300,
    'scrapy.pipelines.images.ImagesPipeline':1
}
IMAGES_STORE='tutupian'
IMAGES_URLS_FIELD='url'

项目要求将小说爬取出来然后在管道进行json格式的保存

项目要求     将小说爬取出来然后在管道进行json格式的保存






# -*- coding: utf-8 -*-
import scrapy
from ..items import XiaoshuoItem

class XiaoxioashuoSpider(scrapy.Spider):
    name = 'xiaoxiaoshuo'
    allowed_domains = ['readnovel.com']
    start_urls = ['https://www.readnovel.com/']

    def parse(self, response):
        # book_list=response.xpath('//div[@class="book-info"]')
        book_list=response.css('.book-info')
        print(book_list)
        for book in book_list:
            # 获取小说名称
            name=book.xpath('.//h4/a/@title').extract_first('')
            if len(name) ==0:
                name = book.xpath('.//h3/a/@title').extract_first('')

            des=book.xpath('.//p/text()').extract_first('')

            author=book.xpath('.//div[@class="state-box cf"]/a/text()').extract_first('')

            type=book.xpath('.//div[@class="state-box cf"]/i/text()').extract_first('')
            item=XiaoshuoItem()
            item['name']=name
            item['des']=des
            item['author']=author
            item['type']=type
            yield item



# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# 用来打开指定文件  并且对文件进行转码  防止出现乱码问题
import codecs
import json
import os
class XiaoshuoPipeline(object):
    def __init__(self):
        # w：写文件     r:读文件
        # w+：读写文件    r+：读写文件
        # 前者读写文件  如果文件不存在  则创建
        # 后者读写文件  如果文件不存在  则抛出异常
        self.file=codecs.open(filename='book.json',mode='w+',encoding='utf-8')
        self.file.write('"list:["')
    # 如果想要将数据写入本地或者使用数据库的时候  这个方法需要保留
    def process_item(self, item, spider):
        # 将item对象转化为一个字典对象
        res=dict(item)
        # dumps  将字典对象转化成字符串  ASCII编码是否可用
        # 如果直接将字典形式的数据写入文件当中，会发生错误
        # 所以需要将字典形式的值  转化成字符串写入文件当中
        str = json.dumps(res,ensure_ascii=False)
        # 将数据写入到文件当中
        self.file.write(str)
        self.file.write(',\n')
    def open_spider(self,spider):
        pass
    def close_spider(self,spider):
        # 删除文件当中最后一个字符
        # -1 表示偏移量至文件的末尾
        # SEEK_END  定位到文件的最后一个字符
        self.file.seek(-1,os.SEEK_END)
        # 开始执行
        self.file.truncate()
        self.file.seek(-1, os.SEEK_END)
        self.file.truncate()

        self.file.write(']')
        self.file.close()


#解注释
ITEM_PIPELINES = {
   'xiaoshuo.pipelines.XiaoshuoPipeline': 300,
}

项目要求将数据爬取出来将其保存到数据库中

# -*- coding: utf-8 -*-
import scrapy

from ..items import HonghongxiuxiuItem
class HongxiuxiuSpider(scrapy.Spider):
    name = 'hongxiuxiu'
    allowed_domains = ['hongxiu.com']
    start_urls = ['https://www.hongxiu.com/finish?gender=2&catId=-1']

    def parse(self, response):
        li_list=response.xpath('//div[@class="right-book-list"]/ul/li')
        for li in li_list:
            img='https'+li.xpath('.//div[@class="book-img"]/a/img/@src').extract_first('')
            print(img)
            name=li.xpath('.//div[@class="book-info"]/h3/a/text()').extract_first('')
            print(name)
            author=li.xpath('.//div[@class="book-info"]/h4/a/text()').extract_first('')
            print(author)
            intro=li.xpath('.//p[@class="intro"]/text()').extract_first('')
            print(intro)
            item=HonghongxiuxiuItem()
            item['img']=img
            item['name']=name
            item['author']=author
            item['intro']=intro
            yield item









# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import sqlite3
class HonghongxiuxiuPipeline(object):
    def process_item(self, item, spider):
        return item
class HongXiuDBPipeline(object):
    def open_spider(self,spider):
        self.connect=sqlite3.connect('hongxiuDB')
        self.cursor=self.connect.cursor()
        self.cursor.execute('create table if not exists bookTable(name text,author text,img text, intro text)')
        self.connect.commit()
    def process_item(self,item,spider):
        self.cursor.execute('insert into bookTable (name,author,img,intro)VALUES ("{}","{}","{}","{}")'.format(item['name'],item['author'],item['img'],item['intro']))
        self.connect.commit()
    def close_spider(self,spider):
        self.cursor.close()
        self.connect.close()






ITEM_PIPELINES = {
   'honghongxiuxiu.pipelines.HonghongxiuxiuPipeline': 300,
    'honghongxiuxiu.pipelines.HongXiuDBPipeline':1
}

项目要求将数据爬取出来进行单位统一

# -*- coding: utf-8 -*-
import scrapy
# https://search.51job.com/list/170200,000000,0000,00,9,99,java,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=
from ..items import JobItem
class JavaSpider(scrapy.Spider):
    name = 'zhiye'
    allowed_domains = ['search.51job.com']
    start_urls = ['https://search.51job.com/list/170200,000000,0000,00,9,99,java,2,1.html?','https://search.51job.com/list/170200,000000,0000,00,9,99,python,2,1.html?']

    def parse(self, response):
        # 1.分离当前页面所有数据  存储到item中
        # 2.获取下一页链接，请求

        div_list=response.xpath('//div[@id="resultList"]/div[@class="el"]')
        for div in div_list:
            # contains：只要属性包含某个值
            jobname=div.xpath('.//p[contains(@class,"t1")]/span/a/@title').extract_first('')
            print(jobname)
            #公司名称
            companyname=div.xpath('.//span[@class="t2"]/a/@title').extract_first('')
            print(companyname)
            cityname=div.xpath('.//span[@class="t3"]/text()').extract_first('')
            print(cityname)
            salary=div.xpath('.//span[@class="t4"]/text()').extract_first('')
            print(salary)
            min_salary=0
            max_salary=0
            if u'年' in salary:
                money = salary.split('万')[0].split('-')
                min_salary=int(money[0])/12
                min_salary='%.1f'% min_salary
                max_salary='%.1f'% (int(money[1])/12)
            elif u'万' in salary:
                money =salary.split('万')[0].split('-')
                min_salary=money[0]
                max_salary=money[1]
            elif u'千' in salary:
                money =salary.split('千')[0]
                if '-' in money:
                    min_salary=float(money.split('-')[0])*0.1
                    max_salary=float(money.split('-')[1])*0.1
                else:
                    min_salary=0
                    max_salary=float(money)*0.1
            elif u'日' in salary:
                money=salary.split('元')
                min_salary=0
                max_salary=int(money[0])*30/10000
            else:
                min_salary=0
                max_salary=0
            date=div.xpath('.//span[@class="t5"]/text()').extract_first('')
            item = JobItem()
            item['jobname']=jobname
            item['companyname'] = companyname
            item['cityname'] = cityname
            item['min_salary'] = min_salary
            item['max_salary'] = max_salary
            item['date'] = date
            yield item

        next_url=response.xpath('//li[@class="bk"]/a[text()="下一页"]/@href').extract()
        print('**********************************************')
        print(next_url)
        if len(next_url) != 0:
            print(next_url[0])
            yield scrapy.Request(url=next_url[0],callback=self.parse)

项目要求:爬取数据将数据存入到MYSQL中

# -*- coding: utf-8 -*-
import scrapy
from ..items import DianyingItem

class TiantangSpider(scrapy.Spider):
    name = 'tiantang'
    allowed_domains = ['ygdy8.net']
    start_urls = ['http://www.ygdy8.net/html/gndy/index.html']

    def parse(self, response):
        detail_list=response.xpath('//div[@class="co_area2"]//tr')
        for datail in detail_list:
            url = 'http://www.ygdy8.net'+datail.xpath('.//td[1]/a[2]/@href').extract_first('')
            print(url)
            yield scrapy.Request(url=url,callback=self.detail_info)
    def detail_info(self,response):
        title=response.xpath('//div[@class="title_all"]//font/text()').extract_first('')
        href =response.xpath('//tr[@style="WORD-WRAP : break-word"]/a/@href').extract_first('')
        print(href)
        item=DianyingItem()
        item['title']=title
        item['href']=href
        yield item






import pymysql
class DianyingPipeline(object):
    def __init__(self):
        self.connect=pymysql.connect(host='localhost',user='root',password='666666',db='movie',port=3306)
        self.cursor=self.connect.cursor()
    def process_item(self, item, spider):
        self.cursor.execute('insert into movieTable(title,href)VALUES ("{}","{}")'.format(item['title'],item['href']))
        self.connect.commit()
        return item
    def close_spider(self,spider):
        self.cursor.close()
        self.connect.close()



ITEM_PIPELINES = {
   'dianying.pipelines.DianyingPipeline': 300,
}

项目要求：将爬取的数据存入json格式并下载图片

# -*- coding: utf-8 -*-
import scrapy

from ..items import HongxiuItem
class HongxiuxiuSpider(scrapy.Spider):
    name = 'hongxiuxiu'
    allowed_domains = ['hongxiu.com']
    start_urls = ['https://www.hongxiu.com/all?catId=30008']

    def parse(self, response):
        li_list=response.xpath('//div[@class="right-book-list"]/ul/li')
        print(li_list)
        for li in li_list:
            img = 'https:'+li.xpath('.//div[@class="book-img"]/a/img/@src').extract_first('')
            title=li.xpath('.//div[@class="book-img"]/a/img/@alt').extract_first('')
            author=li.xpath('.//div[@class="book-info"]/h4/a/text()').extract_first('')
            intro=li.xpath('.//div[@class="book-info"]/p[@class="intro"]/text()').extract_first('')
            item=HongxiuItem()
            item["img"]=[img]
            item["title"]=title
            item["author"]=author
            item["intro"]=intro
            yield item






import scrapy
import codecs
import os
import json
from scrapy.pipelines.images import ImagesPipeline
class HongxiuPipeline(object):
    def __init__(self):
        self.file=codecs.open(filename='hongxiu.json',mode='w+',encoding='utf-8')

    def process_item(self, item, spider):
        res=dict(item)
        str=json.dumps(res,ensure_ascii=False)
        self.file.write(str)
        self.file.write('\n')
        return item
class HongXiuDownloadPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        url=item['img'][0]
        yield scrapy.Request(url=url,meta={'item':item})

    def file_path(self, request, response=None, info=None):
        item=request.meta['item']
        bookname=item['title']
        path= bookname+'.jpg'
        return path






ITEM_PIPELINES = {
   'hongxiu.pipelines.HongxiuPipeline': 300,
    'hongxiu.pipelines.HongXiuDownloadPipeline':1

}
IMAGES_STORE='imgggg'

项目要求：爬取51job里的python和Java工作的总页数和链接

# -*- coding: utf-8 -*-
import scrapy
import re

class JobSpider(scrapy.Spider):
    name = 'job'
    allowed_domains = ['51job.com']
    start_urls = ['https://search.51job.com/list/170200,000000,0000,00,9,99,java,2,1.html','https://search.51job.com/list/170200,000000,0000,00,9,99,Python,2,1.html']
    def parse(self, response):
        total_page=response.xpath('//div[@class="p_in"]/span[1]/text()').extract_first('')
        print(total_page)
        # 使用正则取出页码里面的所有的数字
        res=re.compile(r'\d+')
        # 得到的结果是一个对象 ，从对象中取出匹配的结果
        result=re.findall(res,total_page)[0]
        print(result)
        # 获取请求的网页
        url=response.url
        print(url)
        if 'java' in url:
            for page in range(1,int(result)+1):
                java_url='https://search.51job.com/list/170200,000000,0000,00,9,99,java,2,{}.html'.format(page)
                yield scrapy.Request(url=java_url,callback=self.get_detail_with_page)

        else:
            for page in range(1,int(result)+1):
                python_url='https://search.51job.com/list/170200,000000,0000,00,9,99,Python,2,{}.html'.format(page)
                yield scrapy.Request(url=python_url,callback=self.get_detail_with_page)
    def get_detail_with_page(self,response):
        print(response.url)

项目要求：爬取动态数据将数据存入Excel表中，首先应该在中间件里面写上如下代码：

from scrapy.http.response.html import HtmlResponse
from scrapy import signals
class taobaospidermiddleware(object):
    def process_request(self,request,spider):
        if spider.name=='shishang':
            spider.driver.get(request.url)
            spider.driver.implicitly_wait(10)
            response=HtmlResponse(url=spider.driver.current_url,
                                  request=request,
                                  body=spider.driver.page_source,
                                  encoding='utf-8')
            return response

然后到爬虫文件里输入如下代码：

# -*- coding: utf-8 -*-
import scrapy
from ..items import TaobaoItem
from selenium import webdriver
class ShishangSpider(scrapy.Spider):
    name = 'shishang'
    allowed_domains = ['taobao.com']
    start_urls = ['https://s.taobao.com/search?q=%E6%97%B6%E5%B0%9A%E7%94%B7%E9%9E%8B&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306&bcoffset=6&ntoffset=6&p4ppushleft=1%2C48&s=0']
    def __init__(self):
        self.driver=webdriver.PhantomJS()
    def parse(self, response):

        content_list=response.xpath('//div[@class="ctx-box J_MouseEneterLeave J_IconMoreNew"]')
        for x in content_list:
            name=x.xpath('.//div[@class="row row-2 title"]/a').xpath('string(.)').extract()[0].strip('\n').replace(' ','').strip('\n')
            price=x.xpath('.//div[@class="price g_price g_price-highlight"]/strong/text()').extract_first('')
            dian_name=x.xpath('.//div[@class="shop"]/a/span[2]/text()').extract_first('')
            item=TaobaoItem()
            item['name']=name
            item['price']=price
            item['dian_name']=dian_name
            yield item

item文件代码省略

然后就是管道文件将数据存入Excel，代码如下：

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from openpyxl import Workbook
class TaobaoPipeline(object):
    def __init__(self):
        self.wb=Workbook()
        self.ws=self.wb.active
        self.ws.append(['名称','价格','店铺'])
    def process_item(self, item, spider):
        line=[item['name'],item['price'],item['dian_name']]
        self.ws.append(line)
        self.wb.save('时尚男鞋.xlsx')
        return item
    def spider_closed(self,spider):
        pass

然后到setting文件中将下载中间件和item管道解注释并修改，代码如下：

DOWNLOADER_MIDDLEWARES = {
   'taobao.middlewares.taobaospidermiddleware': 543,
}








ITEM_PIPELINES = {
   'taobao.pipelines.TaobaoPipeline': 300,
}