Python之Scrapy爬虫学习日记(二)Scrapy实战
使用scrapy爬取阳光问政里的意见信息,并存储到MongoDB中
参考B站教程
爬取投诉分类下的所有投诉标题以及详情,包含翻页、图片地址提取等
网页xpath
解析
1、发起请求,起始页:
start_urls = ['http://d.wz.sun0769.com/index.php/question/questiontype?type=4&page=0']
2、接收回应这里可以看到所有的投诉都在tr
标签下
tr_list = response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")
3、翻页的话通过页面中下一页按钮>
来判断,最后一页是没有这个按钮的
next_url = response.xpath("//a[text()='>']/@href").extract_first()
4、spider部分的源码
import scrapy
from sun.items import SunItem
class YgSpider(scrapy.Spider):
name = 'yg'
allowed_domains = ['sun0769.com']
start_urls = ['http://d.wz.sun0769.com/index.php/question/questiontype?type=4&page=0']
def parse(self, response):
# 分组
tr_list = response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")
for tr in tr_list:
item = SunItem()
# 网页解析,元素解析
item['title'] = tr.xpath("./td[2]/a[@class='news14']/@title").extract_first()
item['publish_date'] = tr.xpath("./td[last()]/text()").extract_first()
item['href'] = tr.xpath("./td[2]/a[@class='news14']/@href").extract_first()
item['user'] = tr.xpath("./td[4]/text()").extract_first()
item['department'] = tr.xpath("./td[2]/a[@class='t12h']/text()").extract_first()
yield scrapy.Request(
item['href'],
callback=self.parse_data1,
meta={'item': item} # 这里是一个meta字典,可以方便的将详情页返回的数据加入到item中
)
# 翻页
next_url = response.xpath("//a[text()='>']/@href").extract_first()
if next_url is not None:
yield scrapy.Request(
next_url,
callback=self.parse
)
# 处理详情页面内的数据,并将其返回给item
def parse_data1(self, response):
item = response.meta['item']
item["content"] = response.xpath("//div[@class='contentext']//text()").extract()
item["content_img"] = response.xpath("//div[@class='textpic']//img/@src").extract()
item["content_img"] = ["http://wz.sun0769.com"+i for i in item["content_img"]]
# print(item)
yield item # yield给item就会经过pipelines进行数据的处理
5、items源码
import scrapy
from scrapy import Field
class SunItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = Field()
href = Field()
publish_date = Field()
department = Field()
user = Field()
content_img = Field()
content = Field()
6、pipeline源码,对数据进行操作(修饰、存储)
import re
import pymongo
import logging
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(levelname)s: %(message)s',
) # 定义一个log的样式
logger = logging.getLogger(__name__)
class SunPipeline(object):
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls, crawler):
return cls(mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE'))
def open_spider(self, spider):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db=self.client[self.mongo_db]
def closen_spider(self, spider):
self.client.close()
def process_item(self, item, spider):
item["content"] = self.process_content(item["content"]) # 调用process_content对获取到的字符串进行处理
# 这里是MongoDB的更新操作,如果查询到了就使用第一个参数作为查询条件,第二个通过$set指定更新的条件第三个表示如果存在则更新,不存在则插入
self.db['user'].update({'title': item['title']}, {'$set': item}, True)
# print(item)
# logger.info('success')
return item
def process_content(self,content):
content = [re.sub(r"\xa0|\t|\s", "", i) for i in content] # 替换文本中的\a0 \t \s等字符
content = [i for i in content if len(i) > 0] # 去除列表中的空字符串
return content
7、setting中需要添加(取消注释)的内容
LOG_LEVEL = "INFO" # 显示的日志等级
LOG_FILE = "./log.log" # 生成日志的保存位置
# Crawl responsibly by identifying yourself (and your website) on the user-agent
# 请求头,在浏览器网页检查元素,Network中可以找到
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, ' \
'like Gecko) Chrome/78.0.3904.108 Safari/537.36'
ITEM_PIPELINES = {
'sun.pipelines.SunPipeline': 300,
}
# 定义MongoDB的域名和id
MONGO_URI = 'localhost'
MONGO_DATABASE = 'sun0769'
8、新建一个begin.py
用于运行spider
from scrapy import cmdline
cmdline.execute('crapy crawl yg'.split())
9、大功告成
MongoDB compass界面查看数据
词频热点图
最后的效果图(选取5000个title
的内容)
使用
wordcloud 词云
和
jieba 分词
实现
生成词云部分的源码如下:
1、处理从数据库提取的字符并生成词云
from collections import Counter
from os import replace
import pymongo
import re
import jieba
from gen_wordcloud import genwordcloud
# 链接到数据库
my_client = pymongo.MongoClient("mongodb://localhost:27017")
my_db = my_client["sun0769"]
my_col = my_db["user"]
my_title ={"_id": 0, "title": 1} # 限定获取对象的元素,只获取title
my_result = my_col.find({}, my_title).limit(5000) # 获取前5000个对象
word_list = [] # 定义空数组用于存储词条
for x in my_result:
str = x['title']
str = re.sub("[A-Za-z]", "", str)
str.replace(" ", "").strip().lstrip()
str_cut = jieba.cut(str, cut_all=True)
# print(",".join(str_cut))
for word in str_cut:
word_list.append(word)
word_dict = dict(Counter(word_list))
genwordcloud(word_dict)
2、生成词云的模块
from os import path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import os
from wordcloud import WordCloud, STOPWORDS
def genwordcloud(dict):
d = path.dirname(__file__) if "__file__" in locals() else os.getcwd()
# text = open(path.join(d, 'alice.txt')).read() # 生成词云的源文本
alice_mask = np.array(Image.open(path.join(d, "bg_mask.png"))) # 背景图片路径
font_path = "C:\Windows\Fonts\叶根友毛笔行书.ttf" # 字体所在的地址,设置一个中文字体地址,不然会乱码
stopwords = set(STOPWORDS) # 设置停止关键词
stopwords.add("said") # 添加停止关键词
wc = WordCloud(background_color="black", # 设置背景颜色
max_words=300, # 最大显示词数
mask=alice_mask, # 背景图片
# stopwords=stopwords, # 设置停止关键词
contour_width=0, # 设置图片边线的宽度
contour_color='black', # 边线颜色
width=1000, height=1000, # 图片长宽,设置背景图片时不使用
margin=1, # 词语边缘距离
font_path=font_path, # 设置字体
)
# generate word cloud
# wc.generate(text) # 生成词云
wc.generate_from_frequencies(dict)
# store to file
wc.to_file(path.join(d, "test.png")) # 最后保存的词云图片
# show 显示结果图片和原始图片
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.figure()
plt.imshow(alice_mask, cmap=plt.cm.gray, interpolation='bilinear')
plt.axis("off")
plt.show()