Python爬虫

一、request库的基本使用方法

基本实例

import requests url= 'https://www.baidu.com/' response = requests.get(url) print(type(response)) print(response.status_code)#状态码 print(type(response.text)) print(response.text)#打开网页源代码 print(response.cookies)#获取cookies

 

各种请求方式

import requests url= 'https://www.baidu.com/' requests.get(url) requests.put(url) requests.delete(url) requests.head(url) requests.options(url)

 

带参数的GET请求

import requests data={ } reponse = requests.get(url,params=data)

 

解析JSON

import requests import json reponse = requests.get(url) print(requests.json()) print(json.loads(reponse.text))

 

获取二进制数据和保存

import requests import json reponse = requests.get(url) print(reponse.text) print(reponse.content)

 

import requests import json reponse = requests.get(url) with open(' ',' ') as f: f.write(reponse.content) f.close()

 

添加headers

import requests import json headers = { } response = requests.get(url,headers=headers)

 

基本POST请求

mport requests import json data = { } headers={ } response = requests.post(url,data=data,headers=headers)

 

Reponse属性

import requests url= 'https://www.baidu.com/' response = requests.get(url) print(type(response)) print(response.status_code)#状态码 print(type(response.text)) print(response.text)#打开网页源代码 print(response.cookies)#获取cookies print(response.history) print(response.url)

 

文件上传

import requests files = {'file':open('','rb')} reponse = requests.post(url,files=files)

 

维持会话

import requests s = requests.session() s.get(url_1) response = s.get(url_2)

 

证书认证

import requests from requests.packages import urllib3 urllib3.disable_warnings()#消除警告 response = requests.get(url,verify=False)

 

代理

import requests proxies = { "http": "https": } requests.get(url,proxies=proxies)

 

 pip3 install 'requests[socks]' 使用socks代理

import requests from requests.exceptions import ReadTimeout try: response = requests.get(url,timeout= ) except ReadTimeout: print("time out")

 

认证设置

import requests from requests.auth import HTTPBasicAuth response = requests.get(url,auth=HTTPBasicAuth('',''))

 

import requests response = requests.get(url,auth=('',''))

 

异常处理

 

原文来自：https://www.cnblogs.com/yinghualuowu/p/8563363.html

还可以参考：https://www.cnblogs.com/chenshy/p/10704185.html