Python爬虫urllib2笔记(二)

urllib2升级模块--浏览器可访问爬虫访问返回403说明有防爬虫

#-*-coding:utf-8-*-

#urllib2升级模块--浏览器可访问爬虫访问返回403说明有防爬虫
#F12查看Network(刷新)点击names查看Headers模仿用户登录
import urllib2
url="http://blog.csdn.net/q383700092"
#模仿请求Request
req=urllib2.Request(url)
#复制网页里的Headers里的Request
#用户代理
req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36")
#get访问的url地址
req.add_header("GET",url)
#主机信息
req.add_header("Host","blog.csdn.net")
#从哪里链接过来--有些网站必须是内部网站链接过去才能访问
req.add_header("Referer","http://blog.csdn.net/")

html=urllib2.urlopen(req)
print html.read() 

封装版本

#-*-coding:utf-8-*-

#改进版本-封装性更好
import urllib2
url="http://blog.csdn.net/q383700092"
#键对值的字典信息
my_headers={
		"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36",
		"Host":"blog.csdn.net",
		"Referer":"http://blog.csdn.net/",
		"GET":url
		}


#模仿请求Request
req=urllib2.Request(url,headers=my_headers)

html=urllib2.urlopen(req)
print html.read() 

#各种浏览器的userAgent--多个用户头防止被识别

#-*-coding:utf-8-*-

#改进版本-封装性更好
import urllib2
import random

url="http://blog.csdn.net/q383700092"
#各种浏览器的userAgent--多个用户头防止被识别
my_headers=["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
			"User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
			"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
			"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
			"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
			"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201"
			]
#代理IP(暂无)，假的用户头部信息
def get_content(url,headers):
	"""
	@获取403禁止访问的网页
	"""
	#随机选择
	random_header=random.choice(headers)
	print random_header
	req=urllib2.Request(url)
	req.add_header("User-Agent",random_header)
	req.add_header("Host","blog.csdn.net")
	req.add_header("Referer","http://blog.csdn.net/")
	req.add_header("GET",url)

	content=urllib2.urlopen(req).read()
	return content

print get_content(url,my_headers)