Python爬虫之requests模块基础用法

1.get 请求

requests.get(url,headers=headers)

2.get带参数 :自动转码

params = {"kw":2}
requests.get(url,params = params,headers=headers)

3.post请求

data={"kw":2}
requests.post(url,headers=headers)

4.post带参数 :自动转码

formdata={"kw":2}
requests.post(url,data = formdata,headers=headers)

5.ssl --ca的证书 忽略证书认证

requests.get(url,verfy = False)

6.proxy --代理

proxy = {"http":"IP:port"}requests.get(url,proxies = proxy)
# 6.1私密代理
proxy = {"http":"username:password@IP：post"}
requests.get(url,proxies = proxy)

7.webauth --web客户端验证

auth = ("username","pwd")
requests.get(url,auth = auth)

8.cookie --session

# 1.创建session 还是为了保存cookie
session = requests.session()

# 2.发送登录 -- cookie
session.post(url,data = data)

# 3.携带Cookie值访问那些登录后才可以访问的页面
session.get(data_url)
......

9.headers查询参数

response = requests.get("http://www.baidu.com/s?", params = kw, headers = headers)
# 查看响应内容，response.text 返回的是Unicode格式的数据
print response.text
# 查看响应内容，response.content返回的字节流数据
print respones.content	
# 查看完整url地址
print response.url
# 查看响应头部字符编码
print response.encoding
# 查看响应码
print response.status_code