爬虫学习3.2 HTTP请求的python实现--urllib2/urllib实现

1.实现一个完整的请求与响应处理

import urllib2
response = urllib2.urlopen('http://www.zhihu.com')
html = response.read()
print html

request =  urllib2.Request('http://www.zhihu.com')
response1 = urllib2.urlopen(request)
html1 = response1.read()
print html1 + '11111'

#GET请求

2.请求头处理headers处理

import urllib2
import urllib
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
referer = 'http://www.zhihu.com/'
url = 'http://www.zhihu.com/login'
headers = {'User_Agent':user_agent,'Referer':referer}
postdata = {'username':'qiye','password':'qiye_pass'}

data = urllib.urlencode(postdata)
req = urllib2.Request(url,data,headers)
response = urllib2.urlopen(req)
html = response.read()

req.add_header('User_Agent',user_agent)
req.add_header('Referer',referer)
req.add_data(data)

请求头中的 User_Agent包含操作系统，浏览器标识，加密等级

3.Cookie处理

#coding=GBK
import urllib2
import cookielib
'''
cookie = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
response = opener.open('http://www.baidu.com/')
for item in cookie:
    print item.name + ':' + item.value
'''
#寻找cookie的值
opener = urllib2.build_opener()
opener.add_handler(('Cookie','email='+'xxxxxxxxxxx@qq.com'))
req = urllib2.Request('http://www.baidu.com/')
response = opener.open(req)
print response.headers
retdata = response.read()
#自主创建cookie文件

cookie既有本机创建，也有服务器端创建，不同浏览器会产生不同的cookie

4.Timeout设置超时

#Timeout设置超时

import urllib2
import socket
'''
socket.setdefaulttimeout(10)#一种方式
urllib2.socket.setdefaulttimeout(10)#另一种方式
'''
request = urllib2.Request('http://www.baidu.com/')
response = urllib2.urlopen(request,timeout=2)
html = response.read()
print html
#记住分为请求和响应

5.获取HTTP响应码

#coding=GBK
#获取HTTP响应码

import urllib2
try:
    response = urllib2.urlopen('http://baidu.com/')
    print response
    #使用urlopen返回的response对象的getcode()方法可以得到http的返回码
except urllib2.HTTPError as e :
    if hasattr(e,'code'):
        #检查异常对象code的属性
        print 'Error code:',e.code

6.重定向

#coding=GBK
#重定向
import urllib2
'''
response = urllib2.urlopen('http://www.zhihu.cn')
isRedirected = response.geturl() == 'http://www.zhihu.cn'
'''
#要检测是否发生了重定向动作，只需要检查Response的URL和Request的URL是否一致

class RedirectHandler(urllib2.HTTPRedirectHandler):
    def http_error_301(self, req, fp, code, msg, headers):
        pass
    def http_error_302(self, req, fp, code, msg, headers):
        result = urllib2.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, headers)
        result.status = code
        result.newurl = result.geturl()
        return result
opener = urllib2.build_opener(RedirectHandler)
opener.open('http://www.zhihu.cn')
#不想自动重定向，可以自定义HTTPRedirecHandler类

7.Poxy的设置（代理设置）

#coding=GBK
#Poxy的设置
import urllib2
'''
proxy = urllib2.ProxyHandler({'http':'127.0.0.1:8070'})
opener = urllib2.build_opener([proxy,])
urllib2.install_opener(opener)
response = urllib2.urlopen('http://www.zhihu.com/')
print response.read()
'''
#使用 urllib2.build_opener会设置urllib2的全局opener，之后所有的HTTP访问都会使用这个代理。

#如果想用两个不同的Proxy设置，直接调用Opener的open方法代替全局的urlopen方法
proxy = urllib2.ProxyHandler({'http':'127.0.0.1:8087'})
opener = urllib2.build_opener(proxy)
response = opener.open("http://www.zhihu.com/")
print  response.read()