requests.get()
import requests
response = requests.get('http://www.baidu.com')
print("打印状态码_tatus_code",response.status_code)
print("打印请求_url :",response.url)
print("打印头信息_headers:",response.headers)
print("打印cookie信息_cookies:",response.cookies)
print("请求内容_text:",response.text)
print("以字节流形式打印_content:",response.content)
打印状态码_tatus_code 200
打印请求_url : http://www.baidu.com/
打印头信息_headers: {'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-
alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Sat, 16 May 2020 07:11:48 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:57 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}打印cookie信息_cookies: <RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>
请求内容_text: <!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatib
le content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>ç¾åº¦ä¸ä¸ï¼ä½ å°±ç¥é</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=ç¾åº¦ä¸ä¸ class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>æ°é»</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>å°å¾</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>è§é¢</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>è´´å§</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>ç»å½</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">ç»å½</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">æ´å¤äº§å</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>å³äºç¾åº¦</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>©2017 Baidu <a href=http://www.baidu.com/duty/>使ç¨ç¾åº¦åå¿è¯»</a> <a href=http://jianyi.baidu.com/ class=cp-feedback>æè§åé¦</a> 京ICPè¯030173å· <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>
以字节流形式打印_content: b'<!DOCTYPE html>\r\n<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/htm
l;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b\xef\xbc\x8c\xe4\xbd\xa0\xe5\xb0\xb1\xe7\x9f\xa5\xe9\x81\x93</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>\xe6\x96\xb0\xe9\x97\xbb</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>\xe5\x9c\xb0\xe5\x9b\xbe</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>\xe8\xa7\x86\xe9\xa2\x91</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>\xe8\xb4\xb4\xe5\x90\xa7</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>\xe7\x99\xbb\xe5\xbd\x95</a> </noscript> <script>document.write(\'<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=\'+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ \'" name="tj_login" class="lb">\xe7\x99\xbb\xe5\xbd\x95</a>\');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">\xe6\x9b\xb4\xe5\xa4\x9a\xe4\xba\xa7\xe5\x93\x81</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>\xe5\x85\xb3\xe4\xba\x8e\xe7\x99\xbe\xe5\xba\xa6</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>©2017 Baidu <a href=http://www.baidu.com/duty/>\xe4\xbd\xbf\xe7\x94\xa8\xe7\x99\xbe\xe5\xba\xa6\xe5\x89\x8d\xe5\xbf\x85\xe8\xaf\xbb</a> <a href=http://jianyi.baidu.com/ class=cp-feedback>\xe6\x84\x8f\xe8\xa7\x81\xe5\x8f\x8d\xe9\xa6\x88</a> \xe4\xba\xacICP\xe8\xaf\x81030173\xe5\x8f\xb7 <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>\r\n'
不带参数GET请求
mport requests
response = requests.get('https://bj.fang.lianjia.com/')
print(response.text)
带参的GET请求
- 直接将参数放在url中
import requests
response = requests.get(http://httpbin.org/get?name=gemey&age=22)
print(response.text)
- 参数填写在dict中,发起请求时params参数指定为dict
import requests
data = {'dfsdf': 'hgkhg'}
response = requests.get('http://httpbin.org/get', params=data)
print(response.text)
请求添加头信息
import requests
header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}
response = requests.get('http://www.baidu.com',headers=headers)
使用代理
使用requests库爬取了IP代理网站的IP与端口和类型
因为是免费的,使用的代理地址很快就失效了。
import requests
import re
def get_html(url):
proxy = {'http': '120.25.253.234:812',
'https' '163.125.222.244:8123'}
header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}
req = requests.get(url, headers=heads,proxies=proxy)
html = req.text
return html
证书验证设置
import requests
response = requests.get('https://bj.fang.lianjia.com/',verify=False) #证书验证设为FALSE
print(response.status_code)
超时异常捕获
import requests
from requests.exceptions import ReadTimeout
try:
res = requests.get('https://bj.fang.lianjia.com/', timeout=0.1)
print(res.status_code)
except ReadTimeout:
print(timeout)
异常处理
import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException
try:
response = requests.get('https://bj.fang.lianjia.com/',timeout=0.5)
print(response.status_code)
except ReadTimeout:
print('timeout')
except HTTPError:
print('httperror')
except RequestException:
print('reqerror')
requests.POST()
带数据的post
```python
import requests
data = {'name':'tom','age':'22'}
response = requests.post('https://bj.fang.lianjia.com/', data=data)
带header的post
import requests
header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}
data = {'name':'tom','age':'22'}
response = requests.post('https://bj.fang.lianjia.com/', headers=header,data=data)
带json的post
# -*- coding:utf-8 -*-
import requests
import json
host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
data = {"sites": [{ "name":"weibo" , "url":"www.weibo.com" }]}
r = requests.post(url,json=data)
response = r.json()
带参数的post
# -*- coding:utf-8 -*-
import requests
import json
host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
params = {'key1':'params1','key2':'params2'}
r = requests.post(url,params=params)
response = r.json()
普通文件上传
# -*- coding:utf-8 -*-
import requests
import json
host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
#普通上传
files = {
'file':open('test.txt','rb')
}
r = requests.post(url,files=files)
print (r.text)
定制化文件上传
# -*- coding:utf-8 -*-
import requests
import json
host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
#自定义文件名,文件类型、请求头
files = {
'file':('test.png',open('test.png','rb'),'image/png')
}
r = requests.post(url,files=files)
print (r.text)heman793
多文件上传
# -*- coding:utf-8 -*-
import requests
import json
host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
#多文件上传
files = [
('file1',('test.txt',open('test.txt', 'rb'))),
('file2', ('test.png', open('test.png', 'rb')))
]
r = requests.post(url,files=files)
print (r.text)