爬虫常用的requests库的学习使用

import requests

response = requests.get('https://www.baidu.com')
print(type(response))
print(response.status_code)
print(type(response.text))
print(response.text)
print(response.cookies)

import requests

response = requests.get('http://httpbin.org/get')
print(response.text)

out:
{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ea294c1-d93fdab007f1da937cc4752d"
  }, 
  "origin": "120.243.219.224", 
  "url": "http://httpbin.org/get"
}

import requests

data = {
    'name':'germey',
    'age':22
}
response = requests.get('http://httpbin.org/get',params=data)
print(response.text)

out:
{
  "args": {
    "age": "22", 
    "name": "germey"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ea295d4-cbe65534628d5d25cbe08be9"
  }, 
  "origin": "120.243.219.224", 
  "url": "http://httpbin.org/get?name=germey&age=22"
}

import requests
response = requests.get('http://httpbin.org/get?name=germey&age=22')
print(response.text)

out:
{
  "args": {
    "age": "22", 
    "name": "germey"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ea2969e-979e9ce4eb21e150a642b7a0"
  }, 
  "origin": "120.243.219.224", 
  "url": "http://httpbin.org/get?name=germey&age=22"
}

# 获取cookie
import requests

response = requests.get('https://www.zhihu.com')
print(response.cookies)
for key,value in response.cookies.items():
    print(key+"="+value)

out:
<RequestsCookieJar[<Cookie _xsrf=tiI9FY2iowanTyCFwEpW1D1Jtr6sickh for .zhihu.com/>]>
_xsrf=tiI9FY2iowanTyCFwEpW1D1Jtr6sickh

模拟登录：

import requests

requests.get('http://httpbin.org/cookies/set/number/123456789')
response = requests.get('http://httpbin.org/cookies')
print(response.text)  # 因为两次是不同的游览器进行的请求，所以得不到原来的那个cookie值

out:
{
  "cookies": {}
}

import requests
#  模拟登录验证
s = requests.Session()
s.get('http://httpbin.org/cookies/set/number/123456789')
response = s.get('http://httpbin.org/cookies')
print(response.text)   # 用requests.Session()来进行会话维持，所以能得到之前那次请求的cookie值

{
  "cookies": {
    "number": "123456789"
  }
}

证书验证:

import requests

response = requests.get('https://www.12306.cn')
print(response.status_code)

out:
200

import requests
from requests.packages import urllib3
urllib3.disable_warnings()  # 用来消除警告信息
response = requests.get('https://www.12306.cn',verify=False) # 利用参数verify来设定是否验证
print(response.status_code)

out:
200

代理设置:

第一种：
import requests

proxies = {
    'http':'http://127.0.0.1:9742',
    'https':'https://127.0.0.1:9742',
}
response = requests.get('https://www.taobao.com',proxies=proxies)
print(response.status_code)

第二种：
import requests

proxies = {
    'http':'http://user:password@127.0.0.1:9742',
}
response = requests.get('https://www.taobao.com',proxies=proxies)
print(response.status_code)


第三种：
pip install requests[socks]

import requests

proxies = {
    'http':'socks5://127.0.0.1:9742',
    'https':'socks5://127.0.0:9742'
}

response = requests.get('https://www.taobao.com',proxies=proxies)
print(response.status_code)

超时设置

import requests

response = requests.get('https://www.taobao.com',timeout=1)
print(response.status_code)

out:
200

异常处理:

import requests
from requests.exceptions import ReadTimeout, ConnectionError, RequestException
try:
    response = requests.get('http://httpbin.org/get',timeout=0.5)
    print(response.status_code)
except ReadTimeout:
    print('Timeout')
except ConnectionError:
    print('Connection error')
except RequestException:
    print('Error')

out:
Connection error