今天以一个框架来结束今天的爬虫学习,准备去整高数了。
本次的实例是沪江网校的信息,和实例1一样都是爬取各种分类。
不再过多展示页面信息了,直接开代码!!
items:
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class HujiangItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
price = scrapy.Field()
main = scrapy.Field()
teacher = scrapy.Field()
settings基本都一样,改一下请求头,robots协议。。。。
spider:(hj)
# -*- coding: utf-8 -*-
import scrapy
from hujiang.items import HujiangItem
import time
class HjSpider(scrapy.Spider):
name = 'hj'
allowed_domains = ['hujiang.com']
start_urls = ['https://class.hujiang.com/?utm_campaign=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1%E5%93%81%E4%B8%93&utm_adgroup=%E6%A0%87%E9%A2%98&utm_source=baidu_pz&utm_medium=cpc&sem_source=sem_baidu_pz&utm_adkeyword=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1']
def parse(self, response):
url_list = []
infos = response.xpath('/html/body/div[2]/div[1]/div[3]/div[2]/div[1]/div/div/ul/li')
for info in infos:
url_1 = info.xpath('h3/a/@href').extract_first()
url_2 = info.xpath('h3/span/a/@href').extract()
#链接存在于两个标签内,分别取出来放到列表里,然后再遍历列表
url_2_1 = url_2[0]
url_2_2 = url_2[-1]
url_list.append(url_1)
url_list.append(url_2_1)
url_list.append(url_2_2)
#遍历url列表,访问网页
for url in url_list:
requests = scrapy.Request(url,method="get",callback=self.get_Html)
yield requests
def get_Html(self, response):
infos = response.xpath('//*[@id="listData"]/ul/li')
for info in infos:
item = HujiangItem()
#标题信息
item['title'] = info.xpath('div/div[2]/div[1]/a/text()').extract_first()
#价格
price = info.xpath('div/div[2]/p/span[1]/span/text()').extract()
if price[0] == "免费":
item['price'] = "免费"
else:
item['price'] = price[0] + price[1]
#概要信息
item['main'] = info.xpath('div/div[2]/div[2]/span[1]/text()').extract_first()
#教师信息(可能为空,需要判断)
item['teacher'] = info.xpath('div/div[2]/div[2]/span[2]/text()').extract_first()
if item['teacher']:
item['teacher'] = item['teacher']
else:
item['teacher'] = "此处未展示教师师信息!!"
#print(title.strip())
#print(teacher)
yield item
time.sleep(1)
pipelines:
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import csv
class HujiangPipeline(object):
def __init__(self):
with open("F://hujiang.csv",'ab+') as self.fp:
self.fp.write(codecs.BOM_UTF8)
self.fp = open("F://hujiang.csv",'a+',newline='',encoding='utf-8')
self.write = csv.writer(self.fp)
self.write.writerow(['标题','价格','概要','老师'])
def process_item(self, item, spider):
self.write.writerow(
[item['title'].strip(), item['price'], item['main'], item['teacher']]
)
return item
def close(self):
self.fp.close()
最后就可以看到我们想要的信息进入了我们的文件中,
结果展示: