urllib2模块

urllib2模块

标签（空格分隔）： python

---

之前的存在的问题

import urllib2

url = 'http://blog.csdn.net/weiyongxuan/article/details/47193245'

page = urllib2.urlopen(url)

'''
报错 
urllib2.HTTPError: HTTP Error 403: Forbidden

使用urllib2模仿浏览器
'''
print page.read()

模仿浏览器请求

# coding:utf-8
import urllib2

import chardet

url = 'http://blog.csdn.net/weiyongxuan/article/details/47193245'

'''
通过chrome查看的浏览器请求头
Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Accept-Encoding:gzip, deflate, sdch
Accept-Language:zh-CN,zh;q=0.8,en;q=0.6
Cache-Control:max-age=0
Connection:keep-alive
Cookie:uuid_tt_dd=
Referer:http://write.blog.csdn.net/mdeditor
Upgrade-Insecure-Requests:1
User-Agent:Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36
'''

request = urllib2.Request(url)

request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36')

request.add_header('GET', url)

request.add_header('Host', 'blog.csdn.net')

request.add_header('Referer','http://write.blog.csdn.net/mdeditor')

html = urllib2.urlopen(request)

print html.read()

代码整理

# -*- coding : utf-8 -*-
import urllib2

import chardet

url = 'http://blog.csdn.net/weiyongxuan/article/details/47193245'

header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36',
           'Host':'blog.csdn.net',
           'Referer':'http://write.blog.csdn.net/mdeditor',
           'GET': url
}



def getContext(url,header):
    '''
        模仿浏览器，抓取网页
    '''
    request = urllib2.Request(url,headers=header)

    html = urllib2.urlopen(request)

    files = open('\\html.html','w')

    files.write(html.read())

    files.close()

    return files


if __name__=='__main__':
    getContext(url,header)