scrapy携带参数post爬取

个人笔记

# -*- coding: utf-8 -*-
import scrapy
import re
import time
from datetime import datetime, date, timedelta
from scrapy.http import Request
from fzggw.utils import *
from fzggw.items import FgwNewsItem
from fzggw.save_images import save_img
from fzggw.constants import *
import snowflake.client
from fzggw.replace_emoji import remove_emoji
import json



class GjcxGgwSpider(scrapy.Spider):
    name = 'gjcx_ggw'
    start_urls = ['http://sc.ndrc.gov.cn//policy/advancedQuery?']

    def get_form_data(self, page):
          return  {
                    'pageNum':f"{page}",
                    'pageSize':'10',
                    'timeStageId':'',
                    'areaFlagId':'',
                    'areaId':'',
                    'zoneId':'',
                    'businessPeopleId':'',
                    'startDate':'',
                    'endDate':'',
                    'industryId':'',
                    'unitName':'',
                    'issuedno':'',
                }

    def start_requests(self):
        form_data = self.get_form_data(1)
        yield scrapy.FormRequest(method='post', formdata=form_data,
                                 url=self.start_urls[0],dont_filter=True)

    def parse(self, response):
        yesterday = (date.today() + timedelta(days=-1)).strftime("%Y-%m-%d")
        today = date.today().strftime("%Y-%m-%d")
        back_html = response.text
        item_json = json.loads(back_html.strip())
        item_list = item_json['data']['list']
        for info in item_list:
            item = FgwNewsItem()
            item["title"] = ''.join(info['title'])
            item["url"] = (
                    "http://sc.ndrc.gov.cn//zhengcekuDetail.html?id=" + info["id"]
                )
            item['data'] = info['publishtime']

            print(item)

        depth = response.meta.get('depth', 0)
        print(depth)
        page = depth + 2
        print(page)
        yield scrapy.FormRequest(method='post', formdata=self.get_form_data(page),
                                 url=self.start_urls[0],dont_filter=True)

scrapy有可以直接post的scrapy.FormRequest(method=‘post’，不需要指定。当然指定也没错。
dont_filter=True 这个好像是过滤什么的，具体我也忘了。建议带上，不然会出bug。

此为翻页代码

运行

# import time
import time
import os

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

process = CrawlerProcess(get_project_settings())


if __name__ == '__main__':
    while True:
        os.system('py -m scrapy crawl hunan')
        os.system('py -m scrapy crawl zhibang')
        time.sleep(7200)