04 python 爬虫cookie的处理

一.简介

爬虫无法像浏览器一样自动存取和发送cookie,需要我们手动处理

二.处理cookie方法

1.用 requests.utils.dict_from_cookiejar() 把返回的cookies转换成字典

import requests
#得到cookie
def login():
    login_url = 'http://www.xxx.com/login
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01"
    }
    body = {
        "usercode": "liuzz05@****.com",
        "password": "123456"
    }
    try:
        res = requests.post(url=login_url, headers=headers, data=body)
        cookies = res.cookies

        cookie = requests.utils.dict_from_cookiejar(cookies)

        return cookie
    except Exception as err:
        print('获取cookie失败：\n{0}'.format(err))

#使用cookie
import requests
def get_data():
    cookie = login()
    res = requests.get(url=get_data_url, cookies=cookie)
    print(res.text)

2.遍历cookies的键值，拼接成cookie格式

import requests

def login():
    login_url = 'http://www.xxx.com/login
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01"
    }
    body = {
        "usercode": "liuzz05@****.com",
        "password": "123456"
    }
    try:
        res = requests.post(url=login_url, headers=headers, data=body)
        cookies = res.cookies.items()

        cookie = ''
        for name, value in cookies:
            cookie += '{0}={1};'.format(name, value)

        return cookie
    except Exception as err:
        print('获取cookie失败：\n{0}'.format(err))



#使用cookie

def get_data():
    cookie = login()
    headers = {
        "cookie": cookie
    }
    res = requests.get(url=get_data_url, headers=headers)
    print(res.text)

3.直接拼接cookies，这种方法比较傻，前提是要知道cookies的键

import requests
#得到cookie
def login():
    login_url = 'http://www.xxx.com/login
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01"
    }
    body = {
        "usercode": "liuzz05@****.com",
        "password": "123456"
    }
    try:
        res = requests.post(url=login_url, headers=headers, data=body)
        cookies = res.cookies

        phpsessid = cookies['phpsessid']
        env_orgcode = cookies['env_orgcode']
        acw_tc = cookies['acw_tc']
        aliyungf_tc = cookies['aliyungf_tc']
        last_env = cookies['last_env']

        cookie = 'phpsessid={0};env_orgcode={1};acw_tc{2};aliyungf_tc={3};last_env={4}'.format(
            phpsessid, env_orgcode, acw_tc, aliyungf_tc, last_env
        )

        return cookie
    except Exception as err:
        print('获取cookie失败：\n{0}'.format(err))
        
#使用cookie
def get_data():
    cookie = login()
    headers = {
        "cookie": cookie
    }
    res = requests.get(url=get_data_url, headers=headers)
    print(res.text)

4.使用session,正道的光（神器）

使用seesion存储cookie，不再需麻烦的cookie，只需要把请求交给session就可以了

#初始化session
session = requests.Session()
#以后请求直接用session请求就可以了
session.get()
seesion.post()