urllib.request
import urllib.request
url='http://www.baidu.com'
response=urllib.request.urlopen(url)
print(response.read().decode())
print(response.geturl())
print(dict(response.getheaders()))
print(response.getcode())
print(response.readlines())
with open('baidu.html','w',encoding='utf8') as fp:
fp.write(response.read().decode())
with open('baidu1.html','wb') as fp:
response=urllib.request.urlopen(image_url)
with open('1.jpg','wb') as fp:
fp.write(response.read())
urllib.request.urlretrieve(image_url,'2.jpg')
urllib.parse
import urllib.parse
import urllib.request
url='http://www.baidu.com/index.html?name=狗蛋&pwd=123456'
ret=urllib.parse.quote(url)
print(ret)
ret=urllib.parse.unquote(ret)
print(ret)
url1='http://www.baidu.com/index.html'
name='goudan'
age=18
sex='nv'
height='180'
data={
'name':name,
'age':age,
'sex':sex,
'height':height,
'weight':180,
}
lt = []
for k , v in data.items():
lt.append(k + '=' + str(v))
query_string='&'.join(lt)
url1=url1+'?'+query_string
query_string=urllib.parse.urlencode(data)
url1=url1+'?'+query_string
response=urllib.request.urlopen(url1)
print(response.read())
get请求
import urllib.parse
import urllib.request
word="zyglz"
url='http://www.baidu.com/s?'
data={
'ie':'utf-8',
'wd':word,
}
query_string=urllib.parse.urlencode(data)
url +=query_string
print(url)
response=urllib.request.urlopen(url)
filename=word +'.html'
with open(filename,'wb')as fp:
fp.write(response.read())
请求头部
import urllib.parse
import urllib.request
url='http://www.baidu.com/'
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
request=urllib.request.Request(url=url,headers=headers)
response=urllib.request.urlopen(request)
print(response.read().decode())
post请求
import urllib.parse
import urllib.request
url='https://fanyi.baidu.com/v2transapi'
word='wolf'
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
'Cookie':'PSTM=1547277712; BIDUPSID=0E95D79222272DA55C610BA0B5F94018; BAIDUID=A91FE67E81B916D9B10F1356BA4542AD:FG=1; BDUSS=ZoaU9FSTJoNlpBLTJmYn5aZDdqNjQ5V2dSUXJPdlZtU1lWdElYT3pIODRKR0ZjQVFBQUFBJCQAAAAAAAAAAAEAAABG~UECaG9vY2hvbjEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADiXOVw4lzlcLV; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; locale=zh; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1549804685; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1549804685; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; from_lang_often=%5B%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D',
}
form_data={
'from':'en',
'to':'zh',
'query':word,
'transtype':'realtime',
'simple_means_flag':3,
'sign':'275695.55262',
'token':'ec367a8c3f87c38b837b3e51934a4351',
}
form_data=urllib.parse.urlencode(form_data).encode()
request=urllib.request.Request(url=url,headers=headers)
response=urllib.request.urlopen(request,form_data)
print(response.read().decode('gbk'))
handler与opener
import urllib.parse
import urllib.request
url='http://www.baidu.com'
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
handler=urllib.request.HTTPHandler()
opener=urllib.request.build_opener(handler)
request=urllib.request.Request(url,headers=headers)
resopnse=opener.open(request)
print(resopnse.read().decode())
代理设置
import urllib.parse
import urllib.request
handler=urllib.request.ProxyHandler({'http':'112.85.175.77:9999'})
opener=urllib.request.build_opener(handler)
url='http://www.baidu.com/s?wd=ip'
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
request=urllib.request.Request(url,headers=headers)
response=opener.open(request)
with open('ip.html','wb') as fp:
fp.write(response.read())
cookiejar
import urllib.request
import urllib.parse
import http.cookiejar
post_url='http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019131833818'
cj=http.cookiejar.CookieJar()
handler=urllib.request.HTTPCookieProcessor(cj)
opener=urllib.request.build_opener(handler)
form_data={
'email':'hoochon@163.com',
'icode':'',
'origURL':'http://www.renren.com/home',
'domain':'renren.com',
'key_id':'1',
'captcha_type':'web_login',
'password':'************************************',
'rkey':'39b392090c635431e86ef76d46f31f40',
'f':'http%3A%2F%2Fwww.renren.com%2F1868758764',
}
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
request=urllib.request.Request(url=post_url,headers=headers)
form_data=urllib.parse.urlencode(form_data).encode()
response=opener.open(request,data=form_data)
print(response.read().decode())
print('*' *50)
get_url='http://www.renren.com/1868758764/profile'
request=urllib.request.Request(url=get_url,headers=headers)
response=opener.open(request)
print(response.read().decode())