python爬取url的某些情形

python网络获取url信息：

1、通过add_header()添加报头：Request.add_header(header)---request.urlopen(req).read()

import urllib.request

url="http://www.aqpta.com/sign/examSign/sign_chkLogin.asp"

#a. 创建Request对象；

req=urllib.request.Request(url)

#b. 添加报头信息

req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36')

#c. urlopen()打开Request对象；

data=urllib.request.urlopen(req).read()

2、通过build_opener()修改报头:request.build_opener().addheaders(head)-----opener.open(url).read()

import urllib.request

url="http://www.aqpta.com/sign/examSign/sign_chkLogin.asp"

a.设置header;

header=('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36')

b. 创建opener对象；

opener=urllib.request.build_opener()

c. 设置opener；

opener.addheaders(header)

d. open()打开url;

data=opener.open(url).read()

3、需创建cookie;  request.build_opener(request.HTTPCookieProcessor(http.cookiejar.CookieJar()))-----request.install_opener(opener) -----opener.open(opener)

import http.cookiejar

cjar=http.cookiejar.CookieJar()

#建立opener对象；

opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))

#安装opener对象；

urllib.request.install_opener(opener)

#打开request;

file=opener.open(req)

data=file.read()

file=open("a","wb")

file.write(data)

file.cose()