Scrapy框架实例-2（沪江网网课信息）

今天以一个框架来结束今天的爬虫学习，准备去整高数了。

本次的实例是沪江网校的信息，和实例1一样都是爬取各种分类。

不再过多展示页面信息了，直接开代码！！

items:

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class HujiangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    price = scrapy.Field()
    main = scrapy.Field()
    teacher = scrapy.Field()

settings基本都一样，改一下请求头，robots协议。。。。

spider：（hj）

# -*- coding: utf-8 -*-
import scrapy
from hujiang.items import HujiangItem
import time


class HjSpider(scrapy.Spider):
    name = 'hj'
    allowed_domains = ['hujiang.com']
    start_urls = ['https://class.hujiang.com/?utm_campaign=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1%E5%93%81%E4%B8%93&utm_adgroup=%E6%A0%87%E9%A2%98&utm_source=baidu_pz&utm_medium=cpc&sem_source=sem_baidu_pz&utm_adkeyword=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1']

    def parse(self, response):
        url_list = []
        infos = response.xpath('/html/body/div[2]/div[1]/div[3]/div[2]/div[1]/div/div/ul/li')
        for info in infos:
            url_1 = info.xpath('h3/a/@href').extract_first()
            url_2 = info.xpath('h3/span/a/@href').extract()
            #链接存在于两个标签内，分别取出来放到列表里，然后再遍历列表
            url_2_1 = url_2[0]
            url_2_2 = url_2[-1]
            url_list.append(url_1)
            url_list.append(url_2_1)
            url_list.append(url_2_2)
        #遍历url列表，访问网页
        for url in url_list:
            requests = scrapy.Request(url,method="get",callback=self.get_Html)
            yield requests

    def get_Html(self, response):
        infos = response.xpath('//*[@id="listData"]/ul/li')
        for info in infos:
            item = HujiangItem()
            #标题信息
            item['title'] = info.xpath('div/div[2]/div[1]/a/text()').extract_first()
            #价格
            price = info.xpath('div/div[2]/p/span[1]/span/text()').extract()
            if price[0] == "免费":
                item['price'] = "免费"
            else:
                item['price'] = price[0] + price[1]
            #概要信息
            item['main'] = info.xpath('div/div[2]/div[2]/span[1]/text()').extract_first()
            #教师信息（可能为空，需要判断）
            item['teacher'] = info.xpath('div/div[2]/div[2]/span[2]/text()').extract_first()
            if item['teacher']:
                item['teacher'] = item['teacher']
            else:
                item['teacher'] = "此处未展示教师师信息！！"
            #print(title.strip())
            #print(teacher)
            yield item
        time.sleep(1)

pipelines：

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import csv

class HujiangPipeline(object):
    def __init__(self):
        with open("F://hujiang.csv",'ab+') as self.fp:
            self.fp.write(codecs.BOM_UTF8)

        self.fp = open("F://hujiang.csv",'a+',newline='',encoding='utf-8')
        self.write = csv.writer(self.fp)
        self.write.writerow(['标题','价格','概要','老师'])
    def process_item(self, item, spider):
        self.write.writerow(
            [item['title'].strip(), item['price'], item['main'], item['teacher']]
        )
        return item
    def close(self):
        self.fp.close()

最后就可以看到我们想要的信息进入了我们的文件中，

结果展示：