Scrapy之Crawlspider实例：爬取沪江网校所有课程信息

点击此处查看要爬取的网页
如图，利用Crawlspider将所有的课程信息爬取下来，并存入MongoDB数据库


整体思路清晰，简单，不过有一个要注意的点，在插入到数据库的时候
items.py

import scrapy

class HujiangwangxiaoItem(scrapy.Item):

    _id = scrapy.Field() # 插入到MongoDB数据库最好一定要加上这个
    title = scrapy.Field()
    good_info = scrapy.Field()
    summary = scrapy.Field()
    now_price = scrapy.Field()
    orgin_price = scrapy.Field()
    end_time = scrapy.Field()
    coures_nums = scrapy.Field()
    teachers = scrapy.Field()

hujiang.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from hujiangwangxiao.items import HujiangwangxiaoItem
import random

class HujiangSpider(CrawlSpider):
    name = 'hujiang'
    allowed_domains = ['hujiang.com']
    start_urls = ['https://class.hujiang.com/?utm_campaign=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1%E5%93%81%E4%B8%93&utm_adgroup=%E6%A0%87%E9%A2%98&utm_source=baidu_pz&utm_medium=cpc&sem_source=sem_baidu_pz&utm_adkeyword=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1']

    rules = (
        Rule(LinkExtractor(restrict_xpaths='//div[@class="cate-list"]//a',unique=True), callback='parse_item', follow=False),
    )

    def parse_item(self, response):
        item = HujiangwangxiaoItem()
        selector = Selector(response)
        course_text = selector.xpath('//div[@class="course-txt-box"]')
        for content in course_text:
            title = content.xpath('div[1]/a[1]/text()').extract_first().strip()
            good_info = content.xpath('a[1]/span/text()').extract_first()
            summary = content.xpath('a[1]/text()').extract_first().strip()
            now_price = content.xpath('p/span[1]/span[2]/text()').extract_first()
            orgin_price = content.xpath('p/span[2]/text()').extract_first()
            if orgin_price:
                orgin_price = orgin_price.strip()
            else :
                pass
            end_time = content.xpath('div[2]/span[1]/text()').extract_first().split('总')[0]
            coures_nums = content.xpath('div[2]/span[1]/text()').extract_first().split('总')[1]
            teachers = content.xpath('div[2]/span[2]/text()').extract_first()
			# 可能会出现重复的名字，id后面加上一个randomint可以很大程度避免重复id
            item['_id'] = title + str(random.randint(1,1000))
            item['title'] = title
            item['good_info'] = good_info
            item['summary'] = summary
            item['now_price'] = now_price
            item['orgin_price'] = orgin_price
            item['end_time'] = end_time
            item['coures_nums'] = '总' + coures_nums
            item['teachers'] = teachers
            yield item
            

没加id这个键的时候会出现KeyError: ‘HujiangwangxiaoItem does not support field: _id’
这个报错信息，我仔细检查了item的键是没有写错的，所以就是id重复了，造成的问题

piplins.py

import pymongo

class HujiangwangxiaoPipeline(object):

    def process_item(self, item, spider):
        client = pymongo.MongoClient('localhost', 27017)
        mydb = client['mydb']
        hujiangkecheng = mydb['hujiangkecheng']
        hujiangkecheng.insert_one(item)
        print(item)
        return item
        

srttings.py就不写了，一样的。
结果

有一千七百多个课程