No23.爬虫之requests的用法

1、request.get的用法

import requests

params = {"wd":"中国"}
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
}
# response.text查看响应内容返回的是unicode格式的字符串
# response.content 返回的是bytes类型
# response.url 查看url
# response.encoding查看响应头的编码
# response.status查看状态码

response = requests.get("https://www.baidu.com/s", params=params, headers=headers)
# print(type(response.text))
# print(response.text)
print(type(response.content))
print(response.content.decode("utf-8"))
print(response.url)
print(response.status_code)
print(response.encoding)

2、requests.post的用法

import requests

url = "https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false"
data = {
    "first":"true",
    "pn":1,
    "kd":"python"
}
headers = {
    "Referer":"https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
}

response = requests.post(url, data=data, headers=headers)
print(response.json()) # 将一个json类型变成一个字典类型

3、requests库之使用代理

import requests

url = "http://httpbin.org/ip"
proxy = {
    "http":"106.75.9.39:8080"
}
response = requests.get(url, proxies=proxy)
print(response.content)

4、requests库之处理cookie

import requests

# response = requests.get("https://www.baidu.com/")
# print(response.cookies.get_dict())  # 查看cookie的值并返回字典

url = "http://www.renren.com/"
data = {
    "email":"847045382@qq.com",
    "password":"swh980629"
}
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
}
session = requests.session()
session.post(url, data=data, headers=headers)
response = session.get("http://www.renren.com/967928364/profile")
with open("renren.html", "w", encoding="utf-8") as f:
    f.write(response.text)

5、requests库之处理不信任的SSL证书

import requests

# 将参数verify设置为False即可
# resp = requests.get(url, verify=False)