urllib2升级模块--浏览器可访问爬虫访问返回403说明有防爬虫
#-*-coding:utf-8-*-
#urllib2升级模块--浏览器可访问爬虫访问返回403说明有防爬虫
#F12查看Network(刷新)点击names查看Headers模仿用户登录
import urllib2
url="http://blog.csdn.net/q383700092"
#模仿请求Request
req=urllib2.Request(url)
#复制网页里的Headers里的Request
#用户代理
req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36")
#get访问的url地址
req.add_header("GET",url)
#主机信息
req.add_header("Host","blog.csdn.net")
#从哪里链接过来--有些网站必须是内部网站链接过去才能访问
req.add_header("Referer","http://blog.csdn.net/")
html=urllib2.urlopen(req)
print html.read()
封装版本
#-*-coding:utf-8-*-
#改进版本-封装性更好
import urllib2
url="http://blog.csdn.net/q383700092"
#键对值的字典信息
my_headers={
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36",
"Host":"blog.csdn.net",
"Referer":"http://blog.csdn.net/",
"GET":url
}
#模仿请求Request
req=urllib2.Request(url,headers=my_headers)
html=urllib2.urlopen(req)
print html.read()
#-*-coding:utf-8-*-
#改进版本-封装性更好
import urllib2
import random
url="http://blog.csdn.net/q383700092"
#各种浏览器的userAgent--多个用户头防止被识别
my_headers=["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201"
]
#代理IP(暂无),假的用户头部信息
def get_content(url,headers):
"""
@获取403禁止访问的网页
"""
#随机选择
random_header=random.choice(headers)
print random_header
req=urllib2.Request(url)
req.add_header("User-Agent",random_header)
req.add_header("Host","blog.csdn.net")
req.add_header("Referer","http://blog.csdn.net/")
req.add_header("GET",url)
content=urllib2.urlopen(req).read()
return content
print get_content(url,my_headers)