5 requests

json()

import requests


r = requests.get('http://httpbin.org/get', params)
print(type(r.text))
print(r.json())
print(type(r.json()))

<class 'str'>
{'args': {'age': '25', 'name': 'germey'}, 'headers': {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Host': 'httpbin.org', 'User-Agent': 'python-requests/2.24.0', 'X-Amzn-Trace-Id': 'Root=1-60e4393e-23d3e2310e22acd452420eb0'}, 'origin': '218.17.40.15', 'url': 'http://httpbin.org/get?name=germey&age=25'}
<class 'dict'>

1. r.json() 将字典形式的字符串转化为字典。是 requests 中的方法，
   如果 r 不是字典形式的字符串，那么会报错。

verify=0,disable_warnings()

import requests
import re


requests.packages.urllib3.disable_warnings()
r = requests.get('https://static1.scrape.cuiqingcai.com', verify=0)
pattern = re.compile('<h2.*?>(.*?)</h2>', re.S)
titles = re.findall(pattern, r.text)
print(titles)

['霸王别姬 - Farewell My Concubine', '这个杀手不太冷 - Léon', '肖申克的救赎 - The Shawshank Redemption', '泰坦尼克号 - Titanic', '罗马假日 - Roman Holiday', '唐伯虎点秋香 - Flirting Scholar', '乱世佳人 - Gone with the Wind', '喜剧之王 - The King of Comedy', '楚门的世界 - The Truman Show', '狮子王 - The Lion King']

1. verify=0 会有警告，requests.packages.urllib3.disable_warnings() 加上就没了

with open(name, ‘wb’) as f:

import requests


r = requests.get('https://github.com/favicon.ico')
with open('favicon.ico', 'wb') as f:
    f.write(r.content)

1. 'wb' 以二进制写入文件

post upload files

import requests


files = {'file': open('favicon.ico', 'rb')}
r = requests.post('http://httpbin.org/post', files=files)
print(r.text)

1. 'rb' 以二进制文件读取内容，设置 post 中的 files 参数即可上传文件

cookies

get cookies

import requests


r = requests.get('http://www.baidu.com')
print(r.cookies)
for key, value in r.cookies.items():
    print(key + '=' + value)

<RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>
BDORZ=27315

Process finished with exit code 0

1. r.cookies 是一个 RequestsCookieJar 类型
2. 要用 字典 的形式来迭代

use cookies

import requests


headers = {
    'cookie': '_octo=GH1.1.908913242.1625484827; _device_id=bd1dc4af7a1f6fe4512731114a116a52; '
              'user_session=ZX8sJA9s8naIa7rG1wcJG-E0lZ1OJTG7r17_Hh4qU8qm4IPo; '
              '__Host-user_session_same_site=ZX8sJA9s8naIa7rG1wcJG-E0lZ1OJTG7r17_Hh4qU8qm4IPo; logged_in=yes; '
              'dotcom_user=Tpieo; has_recent_activity=1; '
              'color_mode=%7B%22color_mode%22%3A%22light%22%2C%22light_theme%22%3A%7B%22name%22%3A%22light%22%2C'
              '%22color_mode%22%3A%22light%22%7D%2C%22dark_theme%22%3A%7B%22name%22%3A%22dark_dimmed%22%2C'
              '%22color_mode%22%3A%22dark%22%7D%7D; tz=Asia%2FShanghai; '
              '_gh_sess=CuegngaeS57%2FtDFbfI20Ce0w6OqKJwcmjtZ0pGQrtGm3KdjPsctxo7lgh1ELfM2hOxtsks'
              '%2FML3QzZNomY1diB9p9bC9DWJHWjRS9TYuXBUowSHFnG4%2B3twaR136A2xh0rQXOuL%2Fetrk48F6OgtKECYrLraSdg9Lq3UBvV'
              '%2FYS5w3wxoQYqhp6YzN8nWaCaAWH--pD4gWDIEBZGfdtS7--TFXYTfaZdEZm4psvTeXhAg%3D%3D',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Geckoe) '
                  'Chrome/91.0.4472.124 Safari/537.36 '
}
r = requests.get('https://github.com', headers=headers)
print(r.text)

1. 结果中包含个人用户名

use cookies by cookieJar

import requests

cookies = '_octo=GH1.1.908913242.1625484827; _device_id=bd1dc4af7a1f6fe4512731114a116a52; ' \
          'user_session=ZX8sJA9s8naIa7rG1wcJG-E0lZ1OJTG7r17_Hh4qU8qm4IPo; ' \
          '__Host-user_session_same_site=ZX8sJA9s8naIa7rG1wcJG-E0lZ1OJTG7r17_Hh4qU8qm4IPo; logged_in=yes; ' \
          'dotcom_user=Tpieo; has_recent_activity=1; ' \
          'color_mode=%7B%22color_mode%22%3A%22light%22%2C%22light_theme%22%3A%7B%22name%22%3A%22light%22%2C' \
          '%22color_mode%22%3A%22light%22%7D%2C%22dark_theme%22%3A%7B%22name%22%3A%22dark_dimmed%22%2C%22color_mode' \
          '%22%3A%22dark%22%7D%7D; tz=Asia%2FShanghai; ' \
          '_gh_sess=CuegngaeS57%2FtDFbfI20Ce0w6OqKJwcmjtZ0pGQrtGm3KdjPsctxo7lgh1ELfM2hOxtsks' \
          '%2FML3QzZNomY1diB9p9bC9DWJHWjRS9TYuXBUowSHFnG4%2B3twaR136A2xh0rQXOuL%2Fetrk48F6OgtKECYrLraSdg9Lq3UBvV' \
          '%2FYS5w3wxoQYqhp6YzN8nWaCaAWH--pD4gWDIEBZGfdtS7--TFXYTfaZdEZm4psvTeXhAg%3D%3D '
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/91.0.4472.124 Safari/537.36 '
}
jar = requests.cookies.RequestsCookieJar()
for cookie in cookies.split(';'):
    key, value = cookie.split('=', 1)
    jar.set(key, value)
r = requests.get('https://github.com/', cookies=jar, headers=headers)
print(r.text)

1. split(要拆分的字符，拆分的次数) 拆分的次数默认为-1，拆分全部，
   1 会返回含有两个元素的列表，从左往右切。

keep session

without session

import requests

requests.get('http://httpbin.org/cookies/set/number/123456789')
r = requests.get('http://httpbin.org/cookies')
print(r.text)

{
  "cookies": {}
}


Process finished with exit code 0

with session

import requests

s = requests.Session()
s.get('http://httpbin.org/cookies/set/number/123456789')
r = s.get('http://httpbin.org/cookies')
print(r.text)

{
  "cookies": {
    "number": "123456789"
  }
}


Process finished with exit code 0

1. s = requests.Session() 构造一个 requests.Session() 对象，
   后续 s 的请求都是在同一个会话。

timeout

import requests


r = requests.get('http://httpbin.org/get', timeout=1)
r = requests.get('http://httpbin.org/get', timeout=(5, 30))
r = requests.get('http://httpbin.org/get', timeout=None)  # defaut
print(r.status_code)                           # wait all the time       

1. timeout=(5, 30) 第一个是连接的 timeout，第二个是读取的 timeout。
2. timeout=1 包括连接和读取。

身份认证

BasicAuth

import requests


r = requests.get('https://static3.scrape.cuiqingcai.com/', auth=('admin', 'admin'), verify=0)
print(r.status_code)

认证成功返回 200，认证失败返回 401

OAuth

proxies