Python-scrapy爬取问政平台

spider

import logging

import scrapy

logger = logging.getLogger(__name__)


class CqwenzhengSpider(scrapy.Spider):
    name = 'cqWenZheng'
    allowed_domains = ['cqnews.net']
    again_url = "http://cqwz.cqnews.net/ask/searchResult?status=1&lastId="
    again_detail_url = "http://cqwz.cqnews.net/ask/askDetail?id="

    def start_requests(self):
        urls = [
            # 'http://cqwz.cqnews.net/ask/searchResult?status=1&lastId=287527'
            'http://cqwz.cqnews.net/ask/searchResult?status=1'
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse, method="POST", dont_filter=False)

    def parse(self, response):
        # 获取settings中的数据
        # self.settings["xxxx"]
        # self.settings.get("xxxx")
        assert isinstance(response, scrapy.http.response.text.TextResponse)
        json_data = response.json()  # type: dict
        if json_data.get("data") is None:
            logger.warning("运行结束")
            return

        last_id = json_data["data"]["dataList"][-1]["id"]
        url = CqwenzhengSpider.again_url + str(last_id)
        yield scrapy.Request(url=url, callback=self.parse, method="POST", dont_filter=False)
        for i in json_data["data"]["dataList"]:
            i["detail_url"] = CqwenzhengSpider.again_detail_url + str(i["id"])
            logger.warning(i)
            yield i

pilelines

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import pymongo
from pymongo import MongoClient
from pymongo.errors import DuplicateKeyError


class CqWenzhengPipeline:
    # 创建一个mongo连接
    def __init__(self):
        mongoClient = MongoClient(host="123", port=222)
        # 验证账号密码
        mongoClient.test.authenticate("test", "test")
        # 选择账号权限对应的数据库
        self.mongo_data_base = mongoClient["test"]
        # 因为是测试 所以每次创建连接时 参数特定集合的数据
        self.mongo_data_base.drop_collection("cqWenZheng")
        # 设置索引 简单索引 默认升序 这里设置id为唯一索引
        self.mongo_data_base["cqWenZheng"].create_index([("id", pymongo.ASCENDING)], unique=True)
        self.mongo_data_base["cqWenZheng"].create_index([("title", pymongo.ASCENDING)])

    def process_item(self, item, spider):
        # 获取settings中的数据
        # spider.settings["xxxx"]
        # spider.settings.get("xxxx")
        try:
            self.mongo_data_base["cqWenZheng"].insert_one(item)
        except DuplicateKeyError as a:
            print("重复建:{}".format(item["id"]))
        return item