Python3中urllib详细使用方法

urllib是python的一个获取url(Uniform Resource Locators,统一资源定址器)了，我们可以利用它来抓取远程的数据进行保存哦，下面整理了一些关于urllib使用中的一些关于header,代理,超时,认证,异常处理处理方法，下面一起来看看。

python3 抓取网页资源的 N 种方法：

1、最简单

[python] view plain copy

print ?

1. import urllib.request  
2. response = urllib.request.urlopen(’http://python.org/’)  
3. html = response.read()  

import urllib.request
response = urllib.request.urlopen('http://python.org/')
html = response.read()

2、使用 Request

[python] view plain copy

print ?

1. import urllib.request  
2. req = urllib.request.Request(’http://python.org/’)  
3. response = urllib.request.urlopen(req)  
4. the_page = response.read()  

import urllib.request
req = urllib.request.Request('http://python.org/')
response = urllib.request.urlopen(req)
the_page = response.read()

3、发送数据

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import urllib.parse  
3. import urllib.request  
4. url = ’http://localhost/login.php’  
5. user_agent = ’Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)’  
6. values = {  
7. ’act’ : ‘login’,  
8. ’login[email]’ : ‘yzhang@i9i8.com’,  
9. ’login[password]’ : ‘123456’  
10. }  
11. data = urllib.parse.urlencode(values)  
12. req = urllib.request.Request(url, data)  
13. req.add_header(’Referer’, ‘http://www.python.org/’)  
14. response = urllib.request.urlopen(req)  
15. the_page = response.read()  
16. print(the_page.decode(“utf8”))  

#! /usr/bin/env python3
import urllib.parse
import urllib.request
url = 'http://localhost/login.php'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
values = {
'act' : 'login',
'login[email]' : 'yzhang@i9i8.com',
'login[password]' : '123456'
}
data = urllib.parse.urlencode(values)
req = urllib.request.Request(url, data)
req.add_header('Referer', 'http://www.python.org/')
response = urllib.request.urlopen(req)
the_page = response.read()
print(the_page.decode("utf8"))

4、发送数据和header

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import urllib.parse  
3. import urllib.request  
4. url = ’http://localhost/login.php’  
5. user_agent = ’Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)’  
6. values = {  
7. ’act’ : ‘login’,  
8. ’login[email]’ : ‘yzhang@i9i8.com’,  
9. ’login[password]’ : ‘123456’  
10. }  
11. headers = { ’User-Agent’ : user_agent }  
12. data = urllib.parse.urlencode(values)  
13. req = urllib.request.Request(url, data, headers)  
14. response = urllib.request.urlopen(req)  
15. the_page = response.read()  
16. print(the_page.decode(“utf8”))  

#! /usr/bin/env python3
import urllib.parse
import urllib.request
url = 'http://localhost/login.php'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
values = {
'act' : 'login',
'login[email]' : 'yzhang@i9i8.com',
'login[password]' : '123456'
}
headers = { 'User-Agent' : user_agent }
data = urllib.parse.urlencode(values)
req = urllib.request.Request(url, data, headers)
response = urllib.request.urlopen(req)
the_page = response.read()
print(the_page.decode("utf8"))

5、http 错误

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. req = urllib.request.Request(’http://www.111cn.net ’)  
4. try:  
5. urllib.request.urlopen(req)  
6. except urllib.error.HTTPError as e:  
7. print(e.code)  
8. print(e.read().decode(“utf8”))  

#! /usr/bin/env python3
import urllib.request
req = urllib.request.Request('http://www.111cn.net ')
try:
urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
print(e.code)
print(e.read().decode("utf8"))

6、异常处理1

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. from urllib.request import Request, urlopen  
3. from urllib.error import URLError, HTTPError  
4. req = Request(”http://www.111cn.net /”)  
5. try:  
6. response = urlopen(req)  
7. except HTTPError as e:  
8. print(‘The server couldn’t fulfill the request.’)  
9. print(‘Error code: ’, e.code)  
10. except URLError as e:  
11. print(‘We failed to reach a server.’)  
12. print(‘Reason: ’, e.reason)  
13. else:  
14. print(“good!”)  
15. print(response.read().decode(“utf8”))  

#! /usr/bin/env python3
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
req = Request("http://www.111cn.net /")
try:
response = urlopen(req)
except HTTPError as e:
print('The server couldn't fulfill the request.')
print('Error code: ', e.code)
except URLError as e:
print('We failed to reach a server.')
print('Reason: ', e.reason)
else:
print("good!")
print(response.read().decode("utf8"))

7、异常处理2

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. from urllib.request import Request, urlopen  
3. from urllib.error import  URLError  
4. req = Request(”http://www.111cn.net /”)  
5. try:  
6. response = urlopen(req)  
7. except URLError as e:  
8. if hasattr(e, ‘reason’):  
9. print(‘We failed to reach a server.’)  
10. print(‘Reason: ’, e.reason)  
11. elif hasattr(e, ‘code’):  
12. print(‘The server couldn’t fulfill the request.’)  
13. print(‘Error code: ’, e.code)  
14. else:  
15. print(“good!”)  
16. print(response.read().decode(“utf8”))  

#! /usr/bin/env python3
from urllib.request import Request, urlopen
from urllib.error import  URLError
req = Request("http://www.111cn.net /")
try:
response = urlopen(req)
except URLError as e:
if hasattr(e, 'reason'):
print('We failed to reach a server.')
print('Reason: ', e.reason)
elif hasattr(e, 'code'):
print('The server couldn't fulfill the request.')
print('Error code: ', e.code)
else:
print("good!")
print(response.read().decode("utf8"))

8、HTTP 认证

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. # create a password manager  
4. password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()  
5. # Add the username and password.  
6. # If we knew the realm, we could use it instead of None.  
7. top_level_url = ”https://www.111cn.net /”  
8. password_mgr.add_password(None, top_level_url, ‘rekfan’, ‘xxxxxx’)  
9. handler = urllib.request.HTTPBasicAuthHandler(password_mgr)  
10. # create “opener” (OpenerDirector instance)  
11. opener = urllib.request.build_opener(handler)  
12. # use the opener to fetch a URL  
13. a_url = ”https://www.111cn.net /”  
14. x = opener.open(a_url)  
15. print(x.read())  
16. # Install the opener.  
17. # Now all calls to urllib.request.urlopen use our opener.  
18. urllib.request.install_opener(opener)  
19. a = urllib.request.urlopen(a_url).read().decode(’utf8’)  
20. print(a)  

#! /usr/bin/env python3
import urllib.request

# create a password manager password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() # Add the username and password. # If we knew the realm, we could use it instead of None. top_level_url = "https://www.111cn.net /" password_mgr.add_password(None, top_level_url, 'rekfan', 'xxxxxx') handler = urllib.request.HTTPBasicAuthHandler(password_mgr) # create "opener" (OpenerDirector instance) opener = urllib.request.build_opener(handler) # use the opener to fetch a URL a_url = "https://www.111cn.net /" x = opener.open(a_url) print(x.read()) # Install the opener. # Now all calls to urllib.request.urlopen use our opener. urllib.request.install_opener(opener) a = urllib.request.urlopen(a_url).read().decode('utf8') print(a)9、使用代理

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. proxy_support = urllib.request.ProxyHandler({’sock5’: ‘localhost:1080’})  
4. opener = urllib.request.build_opener(proxy_support)  
5. urllib.request.install_opener(opener)  
6. a = urllib.request.urlopen(”http://www.111cn.net ”).read().decode(“utf8”)  
7. print(a)  

#! /usr/bin/env python3
import urllib.request
proxy_support = urllib.request.ProxyHandler({'sock5': 'localhost:1080'})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
a = urllib.request.urlopen("http://www.111cn.net ").read().decode("utf8")
print(a)

10、超时

[python] view plain copy

print ?

1. #! /usr/bin/env python3  
2. import socket  
3. import urllib.request  
4. # timeout in seconds  
5. timeout = 2  
6. socket.setdefaulttimeout(timeout)  
7. # this call to urllib.request.urlopen now uses the default timeout  
8. # we have set in the socket module  
9. req = urllib.request.Request(’http://www.111cn.net /’)  
10. a = urllib.request.urlopen(req).read()  
11. print(a)  

#! /usr/bin/env python3
import socket
import urllib.request

# timeout in seconds timeout = 2 socket.setdefaulttimeout(timeout) # this call to urllib.request.urlopen now uses the default timeout # we have set in the socket module req = urllib.request.Request('http://www.111cn.net /') a = urllib.request.urlopen(req).read() print(a)