urllib2和requests的基本使用

最新推荐文章于 2023-11-04 22:43:39 发布

DustHeartQi

最新推荐文章于 2023-11-04 22:43:39 发布

阅读量1.1k

点赞数

分类专栏： python学习文章标签： python urllib2 requests

本文链接：https://blog.csdn.net/qq_25445035/article/details/75094983

版权

python学习专栏收录该内容

2 篇文章 0 订阅

订阅专栏

urllib

不是完整的代码，仅供查询和记忆

import urllib
word={"wd":"python学习"
print urllib.urlencode(word)
"wd=%E4%BC%A0%E6%99%BA%E6%92%AD%E5%AE%A2"
print urllib.unquote("wd=%E4%BC%A0%E6%99%BA%E6%92%AD%E5%AE%A2")
wd=python学习

urllib2

import urllib2

url

url = "http://www.baidu.com/s"
word = {"wd":"python学习"}
word = urllib.urlencode(word)
newurl = url + "? +word

随机添加、修改User-Agent

"User-Agent": 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
ua_list = [
    "Mozilla/5.0 (Windows NT 6.1; ) Apple.... ",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0)... ",
    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X.... ",
    "Mozilla/5.0 (Macintosh; Intel Mac OS... "
]
user_agent = random.choice(ua_list)

随机获取代理

proxy_list = [
    {"http" : "124.88.67.81:80"},
    {"http" : "124.88.67.81:80"},
    {"http" : "124.88.67.81:80"},
    {"http" : "124.88.67.81:80"},
    {"http" : "124.88.67.81:80"}
]
# 随机选择一个代理
proxy = random.choice(proxy_list)

data

发送POST请求时，需要特别注意headers的一些属性：
Content-Length: 144：是指发送的表单数据长度为144，也就是字符个数是144个。
X-Requested-With: XMLHttpRequest ：表示Ajax异步请求。
Content-Type: application/x-www-form-urlencoded ：表示浏览器提交 Web 表单时使用，表单数据会按照 name1=value1&name2=value2 键值对形式进行编码。

formdata = {
    "type":"AUTO",
    "i":"i love python",
    "doctype":"json",
    "xmlVersion":"1.8",
    "keyfrom":"fanyi.web",
    "ue":"UTF-8",
    "action":"FY_BY_ENTER",
    "typoResult":"true"
}
data = urllib.urlencode(formdata)

header,headers,cookie

header = {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}
request.get_header("User-Agent")
request.add_header("Connection","keep-alive")
opener.addeaders = [("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36")]
# addheaders 接受一个列表，里面每个元素都是一个headers信息的元祖,opener将附带headers信息

headers = {
    "Host":"www.renren.com",
    "Connection":"keep-alive",
    "Upgrade-Insecure-Requests":"1",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
    "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language":"zh-CN,zh;q=0.8,en;q=0.6",

    # 便于终端阅读，表示不支持压缩文件
    # Accept-Encoding: gzip, deflate, sdch,

    # 重点：这个Cookie是保存了密码无需重复登录的用户的Cookie，这个Cookie里记录了用户名，密码(通常经过RAS加密)
    "Cookie": "anonymid=ixrna3fysufnwv; depovince=GW; _r01_=1; JSESSIONID=abcmaDhEdqIlM7riy5iMv; jebe_key=f6fb270b-d06d-42e6-8b53-e67c3156aa7e%7Cc13c37f53bca9e1e7132d4b58ce00fa3%7C1484060607478%7C1%7C1484060607173; jebecookies=26fb58d1-cbe7-4fc3-a4ad-592233d1b42e|||||; ick_login=1f2b895d-34c7-4a1d-afb7-d84666fad409; _de=BF09EE3A28DED52E6B65F6A4705D973F1383380866D39FF5; p=99e54330ba9f910b02e6b08058f780479; ap=327550029; first_login_flag=1; ln_uact=mr_mao_hacker@163.com; ln_hurl=http://hdn.xnimg.cn/photos/hdn521/20140529/1055/h_main_9A3Z_e0c300019f6a195a.jpg; t=214ca9a28f70ca6aa0801404dda4f6789; societyguester=214ca9a28f70ca6aa0801404dda4f6789; id=327550029; xnsid=745033c5; ver=7.0; loginfrom=syshome"
}

大多数情况下，我们只用CookieJar()，如果需要和本地文件交互，就用 MozillaCookjar() 或 LWPCookieJar()

import cookielib
cookiejar  = cookielib.Cooliejar()
handler = urllib2.HTTPCookieProssor(cookiejar)
opener = urllib2.build_opener(handler)
opener.open("http://www.baidu.com")

cookie写入文件

filename = "cookie.txt"
cookijar = cookielib.MozillaCookiejar(filename)
handler = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(handler)
response = opener.open("http://www.baidu.com")
cookiejar.save()

从文件中获取cookie

cookiejar = cookielib.MozillaCookiejar()
cookiejar.load("cookie.txt")
handler = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(handler)
response = opener.open("http://www.baidu.com")

cookieStr = " "
for item in cookiejar:
    cookieStr = cookieStr + item.name + "=" + item.value + ";"
##  舍去最后一位的分号
print cookieStr[:-1]

HTTPPasswordMgrWithDefaultRealm() 密码管理对象验证代理密码

HTTPPasswordMgrWithDefaultRealm()类将创建一个密码管理对象，用来保存 HTTP 请求相关的用户名和密码，主要应用两个场景：

验证代理授权的用户名和密码 (ProxyBasicAuthHandler())
验证Web客户端的的用户名和密码 (HTTPBasicAuthHandler())

user = "test"
passwd = "123456"
proxyserver = "61.158.163..130:16816"
passwdmgr = urllib2.HTPPasswordMgrWithDefaultRealm()
passwdmgr.add_password(None,proxyserver,user,passwd)
proxyauth_handler = urllib.ProxyBasicAuthHandler(passwdmgr)

user = "test"
passwd = "123456"
webserver = "http://192.168.199.107"
passwdmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
passwdmgr.add_password(None,webserver,user,passwd)
httpauth_handler = urllib2.HTTPBasicAuthHandler(passwdmgr)

handler

http_hander = urllib2.HTTPHandler()
https_hander = urllib2.HTTPSHandler()
http_handler = urllib2.HTTPHandler(debuglever=1)   //将 Debug Log

打开，这样程序在执行的时候，会把收包和发包的报头在屏幕上自动打印出来，方便调试，有时可以省去抓包的工作。

代理

httpproxy_handler = urllib2.ProxyHandler({"http":"124.88.67.81:80"})
nullproxy_handler = urllib2.ProxyHandler({})
httpproxy_handler = urllib2.ProxyHandler(proxy)

验证

proxyauth_handler = urllib2.ProxyBasicAuthHandler(passwdmgr)
httpauth_handler = urllib2.HTTPBasicAuthHandler(passwdmgr)

cookie:

(import cookielib)
handler = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(handler)

opener:

opener = urllib2.build_opener(http_handler)
opener.addeaders = [("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36")] //addheaders 接受一个列表，里面每个元素都是一个headers信息的元祖, opener将附带headers信息

request:

request = urllib2.Request("http://www.baidu.com")
request = urllib2.Request(url, headers = header)
request.add_header("Connection","keep-alive")
request = urllib2.Request(url, data=data, headers=headers)

response:

response = urllib2.urlopen("http://www.baidu.com")
response = urllib2.urlopen(request)
response = opener.open(request)


html = response.read()
print html