'''
# demo1 GET
import urllib.request as request
# urlopen
response = request.urlopen(url)
#print(response.read().decode('utf-8')) #解码 如中文 下载页面则不需要解码
#print(type(response)) #类型
print(response.status)
print(response.getheaders())
print(response.getheader('Server'))
'''
''' demo2 POST
import urllib.parse
import urllib.request
data = bytes(urllib.parse.urlencode({'word':'hello'}),encoding='utf-8')
response = urllib.request.urlopen('http://httpbin.org/post',data=data,timeout=1)
print(response.read())
'''
'''
# demo3
import socket
import urllib.request
import urllib.error
try:
response = urllib.request.urlopen('http://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
if isintance(e.reason,socket.timeout):
print('Time Out')
'''
'''
# demo4
from urllib import request
url = r'https://www.python.org'#转义
req = request.Request(url) #此方法可自定义其它参数 可对反爬
response = request.urlopen(req)
print(response.read().decode('utf-8'))
'''
'''
# demo5
# get cookie
import http.cookiejar, urllib.request
cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
for item in cookie:
print(item.name+"="+item.value)
'''
'''
# demo6 get cookie and save
import http.cookiejar, urllib.request
filename = 'cookie.txt'
cookie = http.cookiejar.MozillaCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)
'''
'''
# demo7 get cookie and save
import http.cookiejar, urllib.request
filename = 'cookie.txt'
cookie = http.cookiejar.LWPCookieJar()
# 加载本地cookie
cookie.load(filename,ignore_discard=True, ignore_expires=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))
'''
'''
# demo8 user-agent
from urllib import request
import random
import re
url = r'http://www.baidu.com'
agent1 = 'a'
agent2 = 'b'
agent = [agent1,agent2]
header = {
"user-agent": random.choice(agent)
}
#print(random.choice(agent))
req = request.Request(url,headers=header)
response = request.urlopen(req)
'''
'''
#demo9 自定义 opener
from urllib import request
#构建HTTP处理对象专门HTTP请求
handler = request.HTTPHandler()
#创建自定义opener
opener = request.build_opener(handler)
#创建自定义对象
req = request.Request(url)
#发送请求获取响应
#reponse = opener.open(req).read().decode() #只能发一次请求
#设置全局opener 完成多次请求 用urlopen发送请求也可以自定义opener
request.install_opener(opener)
reponse = request.urlopen(req).read()
print(reponse)
'''
'''
#demo10 proxy
from urllib import request
import random
#多个代理IP及端口 免费/收费
proxylist = [
{"http",ip1},
{"http",ip2},
{"http",ip...},
]
proxy = random.choice(proxylist)
#构建代理对象
handler = request.ProxyHandler(proxy)
#创建自定义opener
opener = request.build_opener(handler)
req = request.Request(url)
res = opener.open(req).read().decode()
'''
'''
#demo 11 GET请求
from urllib import request,parse
#分析网址传值 注意中文编码
wd = {"wd":"中文"}
#构造URL编码
wdd = parse.urlencode(wd)
url = "http://www.baidu.com/s?"+wdd
#同上
'''
''' #requests 基本用法 import requests #content二进制适合获取视频,图片 text字符串会中文乱码 response = requests.get('http://www.baidu.com').content.decode() #与上同样 response = requests.request('get','http://www.baidu.com').content.decode() print(response) ''' ''' #传参 import requests wd = {"wd":"中文"} url = r'http://www.baidu.com/s?' #伪浏览器 agent1 = 'a' agent2 = 'b' agent = [agent1,agent2] header = { "user-agent": random.choice(agent) } #get params=wd 中文省转义 response = requests.get(url,params=wd,headers=header) #post formdata = { "":"" ... } response = requests.post(url,headers=header,data=formdata) print(response.json()) #多个代理IP及端口 免费/收费 proxylist = { "http",ip1, "https",ip2, "http",ip... } proxy = random.choice(proxylist) #proxies 单个多个都行 response = requests.get(url,proxies=proxylist) print(response) ''' ''' #获取cookie import requests req = requests.get("http://www.baidu.com") #获取返回cookiejar对象 cookiejar = req.cookies #将cookiejar转成字典 cookiedict = requests.utils.dict_from_cookiejar(cookiejar) print(cookiedict) ''' ''' #session 模拟登录 import requests header = { "user-agent": "xxx" } ses = requests.session() data = {"user":"xxx","pwd":"xxx"} #获取COOKIE ses.post(url,data=data) #请求页面 response = ses.get(newurl) print(response) '''