1. 请求参数
2. GET请求
def parse(self, response, **kwargs):
yield scrapy.Request(
url="", # 网址
method="GET", # 请求方法
headers={"Usre-Agent": "values"}, # 请求头
cookies={"key": "values"}, # cookies
callback=self.parse_detail, # 回调方法
meta={"key": "values"}, # meta参数
cb_kwargs={"key": "values"} # cb_kwargs参数
)
3. POST
使用 scrapy.FormRequest
发起 POST 请求
scrapy.FormRequest
用于发送表单数据。你可以在 formdata
参数中指定要发送的表单数据。
import scrapy
class MySpider(scrapy.Spider):
name = 'my_spider'
def start_requests(self):
url = 'http://example.com/login'
formdata = {
'username': 'myusername',
'password': 'mypassword',
}
yield scrapy.FormRequest(
url, formdata=formdata,
callback=self.after_login
)
def after_login(self, response):
# Check login success before proceeding
if "Welcome" in response.text:
self.log('Login successful!')
# Proceed to the next page or perform further actions
else:
self.log('Login failed')
使用 scrapy.Request
发起自定义 POST 请求
scrapy.Request
可以用于发送更灵活的 POST 数据,如 JSON 数据或自定义的请求头。
import scrapy
import json
class MySpider(scrapy.Spider):
name = 'my_spider'
def start_requests(self):
url = 'http://example.com/api'
data = {
'key1': 'value1',
'key2': 'value2',
}
headers = {
'Content-Type': 'application/json',
}
yield scrapy.Request(
url,
method='POST',
body=json.dumps(data), # 转换为JSON
headers=headers,
callback=self.parse_response
)
def parse_response(self, response):
# Handle the response
data = json.loads(response.text)
self.log(f'Received data: {data}')