爬虫之request模块基本使用

一、基本使用发送get请求

import requests
# 爬取页面源代码
url="https://bbs.csdn.net/?spm=1000.2115.3001.6068"
res=requests.get(url)
res.encoding="utf-8"
# 以utf-8格式解码
print(res.text)3#得到页面源代码

import requests
content=input("请输入要搜索的内容\n")
url=f"https://cn.bing.com/search?q={content}"
res=requests.get(url)
print(res.text)

最简单的反爬处理

import requests
# 爬取页面源代码
content=input("请输入要搜索的内容\n")
url=f"https://cn.bing.com/search?q={content}"
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"}
res=requests.get(url,headers=headers)
print(res.text)

print(res.request.headers)#可以查看请求头信息

二、发送post请求

import requests
import json
url="https://fanyi.baidu.com/sug"
h={"kw":input("输入单词\n")}
res=requests.post(url,data=h,)
print(res.text)
print(res.json())

三、处理很多参数的get请求

import requests
url="https://movie.douban.com/j/chart/top_list"
data={
"type": "13",
"interval_id": "100:90",
"action":"",
"start": "0",
"limit": "20"
}
headers={"User-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"}#处理一个小反爬
resp=requests.get(url,params=data,headers=headers)
print(resp.text)