requsets的过程

因为requests基于urllib，所以看一下urllib的请求过程。

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2


# python2
def load_baidu_data():
    url = 'http://www.baidu.com'
    request = urllib2.Request(url)  # 生成request实例
    response = urllib2.urlopen(request)
    data =response.read()
    return data

if __name__ == '__main__':
    print(load_baidu_data())

window，python2中的urllib get请求。

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2


# 自定义urlopen
def user_defined():
    # url
    url = "http://www.baidu.com"
    # headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # request
    request = urllib2.Request(url, headers=headers)

    # 1.创建处理器
    handler = urllib2.HTTPHandler()

    # 2.根据处理器 自定义openner
    opener = urllib2.build_opener(handler)

    # 3.open()发送请求
    response = opener.open(request)

    print response.read()

if __name__ == '__main__':
    user_defined()

实际的过程是创建处理器HTTPHandler，根据处理器自定义opener，根据opener发送请求open。

那有用户代理呢

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2

# 创建 具有 代理IP功能的处理器


def proxy_opener():

    # 1.url
    url = "http://www.baidu.com"
    # 2.headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # 3.request
    request = urllib2.Request(url, headers=headers)

    # 代理ip
    proxy = {"http": ":"}

    # 1.创建具有 proxy处理器
    proxy_handler = urllib2.ProxyHandler(proxy)

    # 2.根据处理器创建opener
    opener = urllib2.build_opener(proxy_handler)

    # 4.open()发送请求
    response = opener.open(request)
    # 5.返回数据
    print response.read()

if __name__ == '__main__':
    proxy_opener()

实际的过程是创建具有代理功能的处理器ProxyHandler，根据处理器自定义opener，根据opener发送请求open。

web认证

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2


def web_auth_open():
    url = ''
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    request = urllib2.Request(url, headers=headers)
    # 密码管理器
    pwd_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
    # 添加用户名密码
    pwd_manager.add_password(None, uri=url, user='', passwd='')
    # 认证处理器
    web_handler = urllib2.HTTPBasicAuthHandler(pwd_manager)
    # 自定义opener
    web_opener = urllib2.build_opener(web_handler)
    # 自定义open
    response = web_opener.open(request)

    print response.read()

if __name__ == '__main__':
    web_auth_open()

实际的过程是，创建密码管理器，创建具有认证功能的处理器HTTPBasicAuthHandler，根据处理器自定义opener，根据opener发送请求open。
cookie

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2

import urllib

import cookielib


# 测试人人网
def cookie_open():
    login_url = ''
    data_url = ""
    # headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
               }
    formdata = {
        "email": "",
        "password": ""
    }
    formdata = urllib.urlencode(formdata)
    login_request = urllib2.Request(login_url, headers=headers, data=formdata) #post
    data_request = urllib2.Request(data_url,headers=headers) # get
    # cookie罐子
    cookie_jar = cookielib.CookieJar()
    # cookie处理器
    cookie_handler = urllib2.HTTPCookieProcessor(cookiejar=cookie_jar)
    # cookie opener
    cookie_opener = urllib2.build_opener(cookie_handler)
    # cookie open 先登录
    cookie_opener.open(login_request)
    # 后去数据页
    data_response = cookie_opener.open(data_request)
    data = data_response.read()
    print data

if __name__ == '__main__':
    data = cookie_open()

实际的过程是，创建cookie_jar，创建具有cookie功能的处理器HTTPCookieProcessor，根据处理器自定义opener，根据opener发送请求open。

忽略ssl认证

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2

import ssl

#如果 网站没有CA证书 访问受限制  可以 告诉系统忽略

def ssl_load_data():

    # 1.url
    url = "https://www.12306.cn/mormhweb/"
    # 2.headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # 3.request
    request = urllib2.Request(url,headers=headers)

    # 4.发送请求
    #主动告诉系统 忽略证书ssl
    context = ssl._create_unverified_context()
    resposne = urllib2.urlopen(request,context=context)

    with open("","w") as f:
        f.write(resposne.read())


if __name__ == '__main__':
    ssl_load_data()