Pyton爬虫实现豆瓣登陆

• 目标：实现豆瓣登陆；
• 方法：使用requests.Session 来处理cookies
• 问题：验证码输入

关于headers

1.实现登陆

# -*- coding: utf-8 -*-
import requests
from HTMLParser import HTMLParser
class DoubanClient(object):
    def __init__(self):
        object.__init__(self)
        #在chrome浏览器进行登陆，登陆之后：右键 检查-Newwork-login-Headers,可以查看
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
            'Origin': 'https://accounts.douban.com'}
        self.session = requests.session()  # 创建一个session
        self.session.headers.update(headers)  # 对头进行定制

    def login(self, usename, pwd, source='index_nav', redir='https://www.douban.com', user_login='登陆'):
        #下面的地址可能有变化，根据实际进行更改
        url = 'https://accounts.douban.com/login'  # 访问登陆的主页面，为了获取验证码id和验证码图片地址
        r = self.session.get(url)
        (captcha_id, captcha_url) = _get_captcha(r.content)  # 解析网页内容，从这个函数中获取验证码
        if captcha_id:
            captcha_solution = raw_input('Please input solution for captcha[%s]:' % captcha_url)  # 用户输入验证
        data = {'source': source,
                'form_email': usename,
                'form_password': pwd,
                'redir': redir,  # 跳转地址
                'user_login': user_login,
                }
        if captcha_id:
            data['captcha-id'] = captcha_id
            data['captcha-solution'] = captcha_solution

        headers = {'Referer': 'https://accounts.douban.com/login',
                   'Host': 'accounts.douban.com'}

        self.session.post(url, data=data, headers=headers)
        print(self.session.cookies.items())
 #下面这个函数用在解析html文件中，tag匹配
def _attr(attrs, attrname):
    for attr in attrs:
        if attr[0] == attrname:
            return attr[1]
    return attr[0]
def _get_captcha(content):
    class CaptchaParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.captcha_id = None
            self.captcha_url = None
        def handle_starttag(self, tag, attrs):
            if tag == 'img' and _attr(attrs, 'id') == 'captcha_image' and _attr(attrs, 'class') == 'captcha_image':
                self.captcha_url = _attr(attrs, 'src')  # 如果是url = src
            if tag == 'input' and _attr(attrs, 'type') == 'hidden' and _attr(attrs, 'name') == 'captcha-id':
                self.captcha_id = _attr(attrs, 'value')

    p = CaptchaParser()
    p.feed(content)
    return p.captcha_id, p.captcha_url
def _get_ck(content):
    class CKParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.ck = None
        def handle_starttag(self, tag, attrs):
            if tag == 'input' and _attr(attrs, 'type') == 'hidden' and _attr(attrs, 'name') == 'ck':
                self.ck = _attr(attrs, 'value')

    p = CKParser()
    p.feed(content)
    return p.ck
c = DoubanClient()
c.login('wangdachui@163.com', 'dashab123')

注：其中登陆的地址可能发生改变，另外注意大小写，一切需要根据看到的headers