五 python爬虫之requests高级

需求：模拟浏览器对网站进行登录操作

1 浏览器请求行为

在浏览器中进行登录时，录入完用户名、密码和验证码后，需要点击登录按钮。只有在点击登录按钮后，当前页面才会发起一次网络请求。该次网络请求可以通过抓包工具捕获。

经过抓包分析后，我们就可以模拟浏览器点击登陆按钮后发起的请求操作了。该请求为post请求，对应的url在上图中可以看到，切请求携带的参数也可以在抓到的数据包中获取。
参数分析：

• email：用户名
• password：密码
• icode：验证码（通过超级鹰识别）
• 剩下的参数不需要分析

2 编码

识别验证码图片内容函数封装

#!/usr/bin/env python
# coding:utf-8

import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password =  password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()

模拟登录代码：

# 2020-11-02 模拟网站网登录
# coding:utf-8
import requests
from lxml import etree
from CodeClass import Chaojiying_Client

if __name__ == '__main__':
    # session = requests.session()
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'
    }
    url = 'https://passport.ganji.com/login.php?'
    Login_Page = requests.get(url=url,headers=headers).text
    # print(Login_Page)
    tree = etree.HTML(Login_Page)
    code_src = tree.xpath('//*[@class="label-imgcode"]/img/@src')[0]

    code_img = requests.get(url=code_src,headers=headers).content
    with open('./code.jpg','wb') as fp:
        fp.write(code_img)
        print('保存成功！')
    #超级鹰 进行验证码解析
    chaojiying = Chaojiying_Client('guofan','guo123123','90936')
    im = open('./code.jpg','rb').read()
    result = chaojiying.PostPic(im,1902)['pic_str']
    print(result)

    #模拟登录
    URL='https://passport.ganji.com/login.php?callback=jQuery18204834108743095935_1604402081086'
    data ={
        'username': 'guofan009',
        'password': 'guo123123',
        'setcookie': '14',
        'checkCode': result,
        'next': '/ vip / index.php?_rid = 0.14280194423292',
        'source': 'passport',
        '__hash__': 'IjIGrKPcti4OfhYnoNACFLQ6YGmPMGs9d5M+LN2DiEeyOVDB66J7XO/l3lHol3sa',
    }
    response = requests.post(url=URL,params=data,headers=headers, allow_redirects=False)
    print(response.status_code) #200状态码 请求成功
    login_page_text = response.text
    # print(login_page_text)s
    with open('./Login.html','w',encoding='utf-8') as fp:
        fp.write(login_page_text)
        print('登录成功！')

    #爬取当前用的个人信息
    # detail_url = 'http://www.ganji.com/vip/account/edit_userinfo.php?_rid=0.45347785209969826'
    # datail_page_text = session.get(url=detail_url,headers=headers).text
    # with open('./guofan.html','w',encoding='utf-8') as fp:
    #     fp.write(datail_page_text)

问题：状态码返回200 请求成功 但是爬取下来的页面是登陆页面，页面显示并没有登陆成功