Scrapy发送POST请求

一、发送post请求需要将start_urls注释，然后重写start_requests方法
二、使用yield scrapy.FormRequest(url=post_url, formdata=formdata, callback=self.parse)

# -*- coding: utf-8 -*-
import scrapy

class BufySpider(scrapy.Spider):
    name = 'bufy'
    allowed_domains = ['fanyi.baidu.com']
    # 发送post请求，需要注释掉start_urls
    # start_urls = ['http://fanyi.baidu.com/']

    def start_requests(self):
        post_url = 'http://fanyi.baidu.com/sug'
        # 表单数据
        formdata = {
            'kw': 'wolf',
        }
        # 发送请求
        yield scrapy.FormRequest(url=post_url, formdata=formdata, callback=self.parse)

    # 此方法用于保存响应数据
    def parse(self, response):
        content = response.text
        obj = json.loads(content)
        str = json.dumps(obj,ensure_ascii=False)
        with open('fanyi.json','w',encoding='utf-8')as fp:
            fp.write(str)

 

转载于:https://www.cnblogs.com/huanggaoyu/p/10657059.html