Python之Scrapy爬虫学习日记（二）Scrapy实战

Python之Scrapy爬虫学习日记（二）Scrapy实战

使用scrapy爬取阳光问政里的意见信息，并存储到MongoDB中
参考B站教程

爬取投诉分类下的所有投诉标题以及详情，包含翻页、图片地址提取等

网页xpath解析

1、发起请求，起始页：
start_urls = ['http://d.wz.sun0769.com/index.php/question/questiontype?type=4&page=0']
2、接收回应这里可以看到所有的投诉都在tr标签下
tr_list = response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")

3、翻页的话通过页面中下一页按钮>来判断，最后一页是没有这个按钮的
next_url = response.xpath("//a[text()='>']/@href").extract_first()

4、spider部分的源码

import scrapy
from sun.items import SunItem

class YgSpider(scrapy.Spider):
    name = 'yg'
    allowed_domains = ['sun0769.com']
    start_urls = ['http://d.wz.sun0769.com/index.php/question/questiontype?type=4&page=0']

    def parse(self, response):
        # 分组
        tr_list = response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")
        for tr in tr_list:
            item = SunItem()
            # 网页解析，元素解析
            item['title'] = tr.xpath("./td[2]/a[@class='news14']/@title").extract_first()
            item['publish_date'] = tr.xpath("./td[last()]/text()").extract_first()
            item['href'] = tr.xpath("./td[2]/a[@class='news14']/@href").extract_first()
            item['user'] = tr.xpath("./td[4]/text()").extract_first()
            item['department'] = tr.xpath("./td[2]/a[@class='t12h']/text()").extract_first()

            yield scrapy.Request(
                item['href'],
                callback=self.parse_data1,
                meta={'item': item}  # 这里是一个meta字典，可以方便的将详情页返回的数据加入到item中
            )

            # 翻页
            next_url = response.xpath("//a[text()='>']/@href").extract_first()
            if next_url is not None:
                yield scrapy.Request(
                    next_url,
                    callback=self.parse
                )

    # 处理详情页面内的数据，并将其返回给item
    def parse_data1(self, response):
        item = response.meta['item']
        item["content"] = response.xpath("//div[@class='contentext']//text()").extract()
        item["content_img"] = response.xpath("//div[@class='textpic']//img/@src").extract()
        item["content_img"] = ["http://wz.sun0769.com"+i for i in item["content_img"]]
        # print(item)
        yield item  # yield给item就会经过pipelines进行数据的处理

5、items源码

import scrapy
from scrapy import Field

class SunItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = Field()
    href = Field()
    publish_date = Field()
    department = Field()
    user = Field()
    content_img = Field()
    content = Field()
    

6、pipeline源码，对数据进行操作（修饰、存储）

import re
import pymongo
import logging

logging.basicConfig(level=logging.INFO,
                    format='%(levelname)s: %(levelname)s: %(message)s',
                    )  # 定义一个log的样式

logger = logging.getLogger(__name__)


class SunPipeline(object):

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db


    @classmethod
    def from_crawler(cls, crawler):
        return cls(mongo_uri=crawler.settings.get('MONGO_URI'),
                   mongo_db=crawler.settings.get('MONGO_DATABASE'))

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db=self.client[self.mongo_db]

    def closen_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        item["content"] = self.process_content(item["content"])  # 调用process_content对获取到的字符串进行处理
        # 这里是MongoDB的更新操作，如果查询到了就使用第一个参数作为查询条件，第二个通过$set指定更新的条件第三个表示如果存在则更新，不存在则插入
        self.db['user'].update({'title': item['title']}, {'$set': item}, True)
        # print(item)
        # logger.info('success')
        return item

    def process_content(self,content):
        content = [re.sub(r"\xa0|\t|\s", "", i) for i in content]  # 替换文本中的\a0 \t \s等字符
        content = [i for i in content if len(i) > 0]  # 去除列表中的空字符串
        return content

7、setting中需要添加（取消注释）的内容

LOG_LEVEL = "INFO"  # 显示的日志等级
LOG_FILE = "./log.log"  # 生成日志的保存位置

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# 请求头，在浏览器网页检查元素，Network中可以找到
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, ' \
             'like Gecko) Chrome/78.0.3904.108 Safari/537.36'
ITEM_PIPELINES = {
   'sun.pipelines.SunPipeline': 300,
}
# 定义MongoDB的域名和id
MONGO_URI = 'localhost'

MONGO_DATABASE = 'sun0769'

8、新建一个begin.py用于运行spider

from scrapy import cmdline
cmdline.execute('crapy crawl yg'.split())

9、大功告成
MongoDB compass界面查看数据

词频热点图

最后的效果图（选取5000个title的内容）

使用
wordcloud 词云
和
jieba 分词
实现
生成词云部分的源码如下：
1、处理从数据库提取的字符并生成词云

from collections import Counter
from os import replace

import pymongo
import re
import jieba
from gen_wordcloud import genwordcloud

# 链接到数据库
my_client = pymongo.MongoClient("mongodb://localhost:27017")
my_db = my_client["sun0769"]
my_col = my_db["user"]

my_title ={"_id": 0, "title": 1}  # 限定获取对象的元素，只获取title

my_result = my_col.find({}, my_title).limit(5000)  # 获取前5000个对象

word_list = []  # 定义空数组用于存储词条

for x in my_result:
    str = x['title']
    str = re.sub("[A-Za-z]", "", str)
    str.replace(" ", "").strip().lstrip()
    str_cut = jieba.cut(str, cut_all=True)
    # print(",".join(str_cut))
    for word in str_cut:
        word_list.append(word)
    word_dict = dict(Counter(word_list))
genwordcloud(word_dict)

2、生成词云的模块

from os import path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import os

from wordcloud import WordCloud, STOPWORDS


def genwordcloud(dict):
    d = path.dirname(__file__) if "__file__" in locals() else os.getcwd()
    # text = open(path.join(d, 'alice.txt')).read()  # 生成词云的源文本
    alice_mask = np.array(Image.open(path.join(d, "bg_mask.png")))  # 背景图片路径
    font_path = "C:\Windows\Fonts\叶根友毛笔行书.ttf"  # 字体所在的地址，设置一个中文字体地址，不然会乱码
    stopwords = set(STOPWORDS)  # 设置停止关键词
    stopwords.add("said")  # 添加停止关键词

    wc = WordCloud(background_color="black",  # 设置背景颜色
                   max_words=300,  # 最大显示词数
                   mask=alice_mask,  # 背景图片
                   # stopwords=stopwords,  # 设置停止关键词
                   contour_width=0,  # 设置图片边线的宽度
                   contour_color='black',  # 边线颜色
                   width=1000, height=1000,  # 图片长宽，设置背景图片时不使用
                   margin=1,  # 词语边缘距离
                   font_path=font_path,  # 设置字体
                   )

    # generate word cloud
    # wc.generate(text)  # 生成词云
    wc.generate_from_frequencies(dict)

    # store to file
    wc.to_file(path.join(d, "test.png"))  # 最后保存的词云图片

    # show 显示结果图片和原始图片
    plt.imshow(wc, interpolation='bilinear')
    plt.axis("off")
    plt.figure()
    plt.imshow(alice_mask, cmap=plt.cm.gray, interpolation='bilinear')
    plt.axis("off")
    plt.show()