3.requests爬取数据

下载包

下载：pip install requests

发送请求

1.一个类型，六个方法

response的属性以及类型
    类型 ：models.Response
        r.text : 获取网站源码
        r.encoding ：访问或定制编码方式
        r.url ：获取请求的url
        r.content ：响应的字节类型
        r.status_code ：响应的状态码
        r.headers ：响应的头信息

使用案例：

import requests

# 发送get请求
url = "http://www.baidu.com"
response = requests.get(url)

# 设置响应的编码格式
response.encoding = "utf-8" 

print(type(response)) # 返回<class 'requests.models.Response'>类型

2.发送get请求

优势：

• 请求头中的中文不需要编码
• 不需要定制请求

import requests

# 发送请求
base_url = "http://www.baidu.com/s?"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
data = {
    "wd": "北京"
}
response = requests.get(url=base_url, params=data, headers=headers)
print(response.text)

3.发送post请求

import requests

# 发送请求
url = "https://fanyi.baidu.com/sug"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
}
data = {
    'kw': '视力'
}
# url   请求地址
# data  请求参数
# kwargs 字典
response = requests.post(url=url, data=data, headers=headers)
print(response.text)

4.使用代理

# 发送请求
url = "http://2022.ip138.com/"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
data = {
}

proxy = {
    'http': '114.232.74.184:20013'
}
# url   请求地址
# data  请求参数
# kwargs 字典
response = requests.get(url=url, params=data, headers=headers, proxies=proxy)

5.动态验证码解决：session

由于重新发送请求时，验证码会刷新，因此使用session再次申请验证码，使用session再次发送post请求登录。

import requests
from bs4 import BeautifulSoup

# __VIEWSTATE: kEhKIpaynwSfbUK5FVzV3GJbNbAwApdhR5YhC1A58svypJfOETAtv4Jy8l0MEK0Qs4crRHu+jYniVPpGoLSRro/SF9WYVgkwEyWm8p6wW16Nm0qqPG+FL0Ar4SLYZ/GOvI6aDqKlkjTiVN0ilfPFGh0DoFw=
# __VIEWSTATEGENERATOR: C93BE1AE
# from: http://so.gushiwen.cn/user/collect.aspx
# email: 13223674058
# pwd: afsd54645
# code: daf
# denglu: 登录

# 这是登录页面的url地址
url = 'https://so.gushiwen.cn/user/login.aspx?from=http://so.gushiwen.cn/user/collect.aspx'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
response = requests.get(url=url, headers=headers)
response.encoding = 'utf-8'
content = response.text

soup = BeautifulSoup(content, 'lxml')

# 获取__VIEWSTATE
viewstate = soup.select('#__VIEWSTATE')[0].attrs.get('value')
# 获取__VIEWSTATEGENERATOR
viewstategeneration = soup.select('#__VIEWSTATEGENERATOR')[0].attrs.get('value')
# 获取code_name
code = soup.select("#imgCode")[0].attrs.get("src")
code_url = "https://so.gushiwen.cn" + code

# urllib.request.urlretrieve(code_url, filename="./photo/code.jpg")  # 这是错误的，因为不是用一次会话，验证码刷新了
# 使用session解决上述问题
session = requests.session()
response_code = session.get(code_url)
# 图片的下载需要使用二进制数据
content_code = response_code.content
with open("./photo/code.jpg", "wb") as file:
    file.write(content_code)
code_name = input("请输入你的验证码：")

# 点击登录
url_post = 'https://so.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fso.gushiwen.cn%2fuser%2fcollect.aspx'
data_post = {
    '__VIEWSTATE': viewstate,
    '__VIEWSTATEGENERATOR': viewstategeneration,
    'from': 'http://so.gushiwen.cn/user/collect.aspx',
    'email': 'scut_lff@163.com',
    'pwd': '******',
    'code': code_name,
    'denglu': '登录'
}
headers_post = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}

response_post = session.post(url=url_post, data=data_post, headers=headers_post)
response_post.encoding = 'utf-8'
content_post = response_post.text
with open("./html_file/yzm.html", "w", encoding="utf-8") as file:
    file.write(content_post)