python urllib* 获取网页信息

最新推荐文章于 2022-08-15 00:50:52 发布

xluren

最新推荐文章于 2022-08-15 00:50:52 发布

阅读量2.2k

点赞数

分类专栏： python 周边文章标签： python url

本文链接：https://blog.csdn.net/xluren/article/details/37577099

版权

python 周边专栏收录该内容

46 篇文章 0 订阅

订阅专栏

直接上代码吧，从简单到复杂，同时涉及到cookie的使用

import urllib 
import urllib2
import cookielib
import json
########################simple urllib2##############################################################
url="http://www.baidu.com/s"
request=urllib2.Request(url)
print urllib2.urlopen(request).geturl()

######################with params########################################
url='http://www.baidu.com/s'
data={"wd":"hello",}
print urllib.urlencode(data)
request=urllib2.Request(url,urllib.urlencode(data))
print urllib2.urlopen(request).geturl()

#####################with headers#######################################
headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}
url='http://www.baidu.com/s'
request=urllib2.Request(url,urllib.urlencode(data),headers)
print urllib2.urlopen(request).geturl()
    
#############################################method one##################################################
url="https://openapi.hellocdn.com/api/rest/login"
headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}

login_info={"user":"someone@some.com","pass":"someonespassword","output":"json"}
login_info=urllib.urlencode(login_info)

cj=cookielib.CookieJar()
opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
request=urllib2.Request(url,login_info,headers)
response=opener.open(request)
token=json.loads(response.read())["loginResponse"]["session"]["sessionToken"]

url="https://openapi.hellocdn.com/stat/rest/traffic/responseCode/edge"
login_info={"sessionToken":token,"apiKey":"test","fromDate":"20140701","toDate":"20140702","timeInterval":"0","output":"json"}
headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}


login_info=urllib.urlencode(login_info)
request=urllib2.Request(url,login_info,headers)
response=opener.open(request)
print "response",response
print "url",response.geturl()

headers=response.info()
print "headers",headers
for (k,v) in headers.items():
    print k,"#"*5,v

#############################################method two##################################################
from cookielib import LWPCookieJar
import requests
url="https://openapi.hellocdn.com/api/rest/login"
headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}
login_info={"user":"someone@some.com","pass":"someonespassword","output":"json"}

jar = LWPCookieJar('cookies.txt')
r = requests.get(url, cookies=jar,params=login_info,headers=headers)
token=r.json()["loginResponse"]["session"]["sessionToken"]
jar.save()

url="https://openapi.hellocdn.com/stat/rest/traffic/responseCode/edge"
login_info={"sessionToken":token,"apiKey":"test","fromDate":"20140701","toDate":"20140702","timeInterval":"0","output":"json"}
jar = LWPCookieJar('cookies.txt')
jar.load()
r = requests.get(url, cookies=jar,params=login_info,headers=headers)