urllib库使用_details: cache-hkg17920-hkg 1687153418 1546242916-CSDN博客

本文链接：https://blog.csdn.net/qq_34838643/article/details/105889366

python内置的HTTP请求库，无需额外安装

urllib.request：请求模块
urllib.error：异常处理模块
urllib.parse：url解析模块
urllib.robotparser：robots.txt解析模块

1. request模块

1.1 urlopen

urllib.request.urlopen(url, data,{timeout,}*,cafile=None,capath=None,cadefault=False,context=None)

import urllib.request

response = urllib.request.urlopen('http://www.baidu.com')
print(response.read().decode('utf-8'))

1.1.1 post请求携带参数

import urllib.request

data = bytes(urllib.parse.urlencode({'word':'hello'}),encoding='utf8')
response = urllib.request.urlopen('http://httpbin.org/post',data = data)
print(response.read().decode('utf8'))

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "word": "hello"
  }, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Content-Length": "10", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "Python-urllib/3.7", 
    "X-Amzn-Trace-Id": "Root=1-5ead4538-2a7788aebbb193a604c532bc"
  }, 
  "json": null, 
  "origin": "111.17.130.11", 
  "url": "http://httpbin.org/post"
}

1.1.2 超时设置

import urllib.request

response = urllib.request.urlopen('http://httpbin.org/get',timeout=1)
print(response.read().decode('utf-8'))

{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Host": "httpbin.org", 
    "User-Agent": "Python-urllib/3.7", 
    "X-Amzn-Trace-Id": "Root=1-5ead453d-00dc1456bdda480b31ed0b3e"
  }, 
  "origin": "111.17.130.11", 
  "url": "http://httpbin.org/get"
}

import urllib.request
import urllib.error
import socket

try:
    response = urllib.request.urlopen('http://httpbin.org/get',timeout=0.1)
    print(response.read().decode('utf-8'))
except urllib.error.URLError as e:
    print(e.reason)

timed out

1.2 响应

1.2.1 响应类型

import urllib.request

response = urllib.request.urlopen('https://www.python.org')
print(type(response))

<class 'http.client.HTTPResponse'>

1.2.2 状态码、响应头

import urllib.request
response = urllib.request.urlopen('https://www.python.org')

print(response.status)
print(response.getheaders())
print(response.getheader('Server'))

200
[('Connection', 'close'), ('Content-Length', '49149'), ('Server', 'nginx'), ('Content-Type', 'text/html; charset=utf-8'), ('X-Frame-Options', 'DENY'), ('Via', '1.1 vegur'), ('Via', '1.1 varnish'), ('Accept-Ranges', 'bytes'), ('Date', 'Sat, 02 May 2020 10:02:45 GMT'), ('Via', '1.1 varnish'), ('Age', '1565'), ('X-Served-By', 'cache-bwi5125-BWI, cache-hkg17926-HKG'), ('X-Cache', 'HIT, HIT'), ('X-Cache-Hits', '2, 1285'), ('X-Timer', 'S1588413766.708830,VS0,VE0'), ('Vary', 'Cookie'), ('Strict-Transport-Security', 'max-age=63072000; includeSubDomains')]
nginx

1.3 Request

import urllib.request

request = urllib.request.Request('https://python.org')
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

from urllib import request,parse

url = 'http://httpbin.org/post'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
    'Host': 'hhtpbin.org'
}
dict = {
    'name': 'Germey'
}
data = bytes(parse.urlencode(dict),encoding='utf8')
req = request.Request(url=url,data=data,headers=headers,method='POST')
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "name": "Germey"
  }, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Content-Length": "11", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "hhtpbin.org", 
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36", 
    "X-Amzn-Trace-Id": "Root=1-5ead4564-b4b9718ffe96ccdf583114bb"
  }, 
  "json": null, 
  "origin": "111.17.130.11", 
  "url": "http://hhtpbin.org/post"
}

from urllib import request,parse

url = 'http://httpbin.org/post'
dict = {
    'name': 'Germey'
}
data = bytes(parse.urlencode(dict),encoding='utf8')
req = request.Request(url=url,data=data,headers=headers,method='POST')
req.add_header('Host', 'hhtpbin.org')
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "name": "Germey"
  }, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Content-Length": "11", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "hhtpbin.org", 
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36", 
    "X-Amzn-Trace-Id": "Root=1-5ead4564-c648029e662a405e2540ff3d"
  }, 
  "json": null, 
  "origin": "111.17.130.11", 
  "url": "http://hhtpbin.org/post"
}

1.4 Handler

1.4.1 代理

可以伪装本机的ip地址，对方服务器得到的是代理服务器的ip

在爬虫中可以不断切换代理IP，这样服务器得到的是不同的请求，就不会封掉当前的IP

import urllib.request

# 本机
proxy_handler = urllib.request.ProxyHandler({
    'http':'http://127.0.0.1:1080',
    'https':'http://127.0.0.1:1080'
})
opener = urllib.request.build_opener(proxy_handler)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))

1.4.2 cookie

维持当前的会话状态
比如清除已登录网站的cookie，就需要重新进行登录
通过在请求中加入cookie，可以请求需要登录才能访问的数据

# cookie的获取
import http.cookiejar,urllib.request

cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('https://www.baidu.com/')
for item in cookie:
    print(item.name,"=",item.value)

BAIDUID = 82E7A394D22559989695621BB18C3B19:FG=1
BIDUPSID = 82E7A394D22559983DB33D72EDF03F0C
PSTM = 1588413911
BD_NOT_HTTPS = 1

# 保存cookie到本地文件
import http.cookiejar,urllib.request

filename = "cookie.txt"
cookie = http.cookiejar.MozillaCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('https://www.baidu.com/')
cookie.save(ignore_discard=True,ignore_expires=True)

# 从本地文件读取cookie
import http.cookiejar,urllib.request

filename = "cookie.txt"
cookie = http.cookiejar.MozillaCookieJar()
cookie.load(filename,ignore_discard=True,ignore_expires=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('https://www.baidu.com/')
print(response.read().decode('utf-8'))

2. error模块

from urllib import request,error
try:
    response = request.urlopen('https://nicahead.github.io/2020/04/27/curl%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8222/')
except error.URLError as e:
    print(e.reason)

Not Found

# HTTPError是URLError的子类，可以先捕捉HTTPError再捕捉URLError
from urllib import request,error
try:
    response = request.urlopen('https://nicahead.github.io/2020/04/27/curl%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8222/')
except error.HTTPError as e:
    print(e.reason,e.code,e.headers,sep='\n')
except error.URLError as e:
    print(e.reason)

Not Found
404
Connection: close
Content-Length: 3988
Server: GitHub.com
Content-Type: text/html; charset=utf-8
Strict-Transport-Security: max-age=31556952
ETag: "5ea663e8-f94"
Access-Control-Allow-Origin: *
X-Proxy-Cache: MISS
X-GitHub-Request-Id: A674:672F:4F7276:641AD3:5EAD4580
Accept-Ranges: bytes
Date: Sat, 02 May 2020 10:03:49 GMT
Via: 1.1 varnish
Age: 4
X-Served-By: cache-hkg17926-HKG
X-Cache: HIT
X-Cache-Hits: 1
X-Timer: S1588413829.150841,VS0,VE1
Vary: Accept-Encoding
X-Fastly-Request-ID: 94e582ab8d2e4cc87477b89ecb48e3e07e30823e

# 可以对异常的类型进行判断
import urllib.request
import urllib.error
import socket

try:
    response = urllib.request.urlopen('http://httpbin.org/get',timeout=0.1)
    print(response.read().decode('utf-8'))
except urllib.error.URLError as e:
    print(type(e.reason))
    if isinstance(e.reason,socket.timeout):
        print('socket.timeout')

<class 'socket.timeout'>
socket.timeout

3. parse模块

3.1 urlparse

解析url,划分为6个字段

urlib.parse.urlparse(urlstring,scheme='',allow_fragments=True)

# 对url进行切分
from urllib.parse import urlparse

result = urlparse('http://www.baidu.com/index.html;user?id=5#comment')
print(type(result),result)

<class 'urllib.parse.ParseResult'> ParseResult(scheme='http', netloc='www.baidu.com', path='/index.html', params='user', query='id=5', fragment='comment')

# 假如url中没有指定scheme，就会赋值为这个值，如果有，则可以忽略定义的值
from urllib.parse import urlparse

result = urlparse('www.baidu.com/index.html;user?id=5#comment',scheme = 'https')
print(result)

ParseResult(scheme='https', netloc='', path='www.baidu.com/index.html', params='user', query='id=5', fragment='comment')

# allow_fragments=True则会将其放到fragment里面，否则，会放到query里面
from urllib.parse import urlparse

result = urlparse('www.baidu.com/index.html;user?id=5#comment',allow_fragments=False)
print(result)

ParseResult(scheme='', netloc='', path='www.baidu.com/index.html', params='user', query='id=5#comment', fragment='')

3.2 urlunparse

组装url，urlunparse的逆操作

from urllib.parse import urlunparse

data = ['http','www.baidu.com','index.html','user','id=5','comment']
print(urlunparse(data))

http://www.baidu.com/index.html;user?id=5#comment

3.3 urljoin

拼接url，以后面的为基准。对于前面提到的切分成的六个部分。如果后面有，就用后面url的，后面没有就用前面url的

from urllib.parse import urljoin

print(urljoin('http://www.baidu.com','FAQ.html'))
print(urljoin('http://www.baidu.com#comment','https://www.zhihu.com/FAQ.html'))

http://www.baidu.com/FAQ.html
https://www.zhihu.com/FAQ.html

3.4 urlencode

可以将字典对象转化为get请求的参数

from urllib.parse import urlencode

params = {
    'name':'getmey',
    'age':22
}
base_url = 'http://www.baidu.com?'
url = base_url + urlencode(params)
print(url)