requests爬取网页源代码（爬虫）

注释里写的很清楚，大家看注释就可以了。

代码中关于ua字典内容的处理我在上篇urllib3里说的很清楚了，也有图文的教程，大家如果不懂可以去看我的上篇文章。

import requests
import chardet
#指定url
url='https://image.baidu.com/search/index?tn=baiduimage&ps=1&ct=201326592&lm=-1&cl=2&nc=1&ie=utf-8&dyTabStr=MCwzLDIsNiwxLDQsNSw3LDgsOQ%3D%3D&word=%E4%BB%93%E9%BC%A0'
#设置请求头
ua={
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cache-Control":"max-age=0",
"Connection":"keep-alive",
"Cookie":"winWH=%5E6_1536x722; BDIMGISLOGIN=0; BIDUPSID=E13BF43F2E6123BE5A9421171ED470E5; PSTM=1633156200; __yjs_duid=1_c992e7eb62695c285914fe1d41b022501633165312820; BDUSS=jl5VEhGcXBsRFA3UG1tWExBUHRNYUNXNDB6eDVDU3o0SlZULUszc2IxMzZnUWRpRVFBQUFBJCQAAAAAAAAAAAEAAADFloSywM~Lvrv6v6qztbChZ28AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPr032H69N9hc; BDUSS_BFESS=jl5VEhGcXBsRFA3UG1tWExBUHRNYUNXNDB6eDVDU3o0SlZULUszc2IxMzZnUWRpRVFBQUFBJCQAAAAAAAAAAAEAAADFloSywM~Lvrv6v6qztbChZ28AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPr032H69N9hc; H_WISE_SIDS=107316_110085_127969_131862_132547_179347_184716_189037_189325_189755_190248_190627_191067_191247_191287_191371_192206_192387_193284_194085_194511_194520_194600_195038_195187_195342_196426_196514_197241_197470_197471_197478_197711_197782_197957_198265_198649_199023_199082_199176_199305_199440_199468_199490_199571_199598_199752_199845_200274_200349_200490_200743_201055_201107_201191_201328_201548_201554_201577_201581_201600_201701_201733_201947_201969_201978_201996_202059_202177_202283_202392_202476_202545_202553_202823_202915_202923_202969_203253_203520_203534_203576_203605_203688_203787; BAIDUID=BCB9B949EFF4B79A1AE6DF590C38F2BF:FG=1; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; MCITY=-%3A; H_PS_PSSID=31254_26350; BAIDUID_BFESS=BCB9B949EFF4B79A1AE6DF590C38F2BF:FG=1; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; BDRCVFR[tox4WRQ4-Km]=mk3SLVN4HKm; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm; ab_sr=1.0.1_Mzk0OTM2N2MxYTcyNzcxYjY4OWMwMzU2ZmYyZjE4MDg1MWQ5OGQ3NWM2MDI3MGNhZmM4NDkxMjZkOWE0MTQ4YjBlZGU3ZWE2ZGQ3YzVmZWE2MGQ4NjBiYjk4NTZkZjMzYjBhOTY0MDkyOWVhMGFlMjEzZTkwZDEwOTdmYzFjOWE2ZTY3ZTVlOTBhNGYzNDE2NTA1YWY1MGI2YmI3MTcxZWQxNzMxNjEwNzFkYTU0NzE0ODQwMWJjN2MyMWYxMTg5; delPer=0; PSINO=1; BA_HECTOR=ag2k212g2laha12g5d1h33j8u0q",
"Host":"image.baidu.com",
"Referer":"https://www.baidu.com/",
"sec-ch-ua-mobile":"?0",
"Sec-Fetch-Dest":"document",
"Sec-Fetch-Mode":"navigate",
"Sec-Fetch-Site":"same-origin",
"Sec-Fetch-User":"?1",
"Upgrade-Insecure-Requests":"1",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
}
#为避免因等待服务器响应而造成程序永久失去响应，需要对程序设置一个时间限制，超过该时间程序就会自动停止等待,时间设置过短会报错
#生成get请求
rq=requests.get(url,headers=ua,timeout=2)
print('超时为2时：',rq)
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#查看结果类型
print('结果类型：',type(rq),'\n\n')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#查看状态码
print('状态码：',rq.status_code,'\n\n')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#查看编码
#注意：这个是通过服务器返回的HTTP头部信息猜测网页编码，当猜测错误时，需要指定编码，避免返回网页内容解析错误
print('编码：',rq.encoding,'\n\n')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#指定编码
#rq.encoding='utf-8'
'''手动指定的方法并不灵活，无法自适应爬取过程中不同网页的编码，而使用 chardet库的方法比较简便灵活。chardet 库是一个非常优秀的宇符串/文件编码检测模块。
chardet 库的 detect 方法可以检测给定字符串的编码'''
print('detect方法检测结果：',chardet.detect(rq.content))
rq.encoding=chardet.detect(rq.content)['encoding']
print('改变后的编码为',rq.encoding)
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#查看响应头
print('响应头为：',rq.headers,'\n\n')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------')
#查看获取的内容
print(rq.text)